Skip to content

Commit 9f6e89d

Browse files
timsaucerclaude
andcommitted
refactor(spark): derive schema, cast decision, and cast target from one authority
Enumerating "maps to a Spark type" and "readable by ArrowColumnVector" by hand let them drift: a type could map to a Spark DataType yet have no vectorized accessor, crashing at task time (FixedSizeList, and latent for LargeList / FixedSizeBinary / Date64). Replace the three hand-kept switches with a single authority. - sparkConsumable(ArrowType): the read gate, mirroring Spark 4.0's ArrowColumnVector.initAccessor accessor set (verified against the class), deliberately the narrower gate. - sparkTarget(Field): the nearest consumable field, recursing into children (identity when already consumable). One place normalizes unsigned->signed, Float16->Float32, non-us timestamp->us, Date64->Date32, Time->int, FixedSizeBinary->Binary, FixedSizeList/LargeList->List. toSparkType, needsCast, and castTargetString now all derive from sparkTarget, so the gates cannot disagree and adding a type is one case. toSparkSchema asserts at plan time that every target is consumable, so an unsupported type (e.g. Interval) fails in planning naming the column, instead of an opaque executor UNSUPPORTED_ARROWTYPE. Closes three latent bugs alongside FixedSizeList: LargeList, FixedSizeBinary, and Date64 previously mapped to a Spark type without a cast. Tests: SchemaConverterTest gains cases for LargeList/FixedSizeBinary/Date64, a "sparkTarget is always consumable" invariant, and a fail-fast case; the example `types` table gains labels (LargeList<Utf8>), digest (FixedSizeBinary), and day (Date64) columns, asserted end-to-end in AdbcSourceTest and the pyspark harness. The E2E session enables the Java 8 date API so DateType decodes without needing sun.util.calendar opened. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 390691b commit 9f6e89d

5 files changed

Lines changed: 366 additions & 150 deletions

File tree

‎examples/adbc-datafusion-driver/pyspark_e2e.py‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,17 @@ def read(table):
118118
# cast columns are flagged (so filter pushdown stays off them); pass-through ones are not.
119119
cast_cols = {f.name for f in dft.schema.fields if f.metadata.get(CAST_META_KEY)}
120120
print("cast columns:", sorted(cast_cols))
121-
assert cast_cols == {"channel", "big", "event_time", "score", "tags", "vec"}, cast_cols
121+
assert cast_cols == {
122+
"channel",
123+
"big",
124+
"event_time",
125+
"score",
126+
"tags",
127+
"vec",
128+
"labels",
129+
"digest",
130+
"day",
131+
}, cast_cols
122132

123133
# count() prunes the projection to empty; the connector must not emit a bare SELECT * (which
124134
# would return the raw, uncast schema and fail the reader on the ns timestamp / unsigned ids).
@@ -163,6 +173,20 @@ def read(table):
163173
assert r2["vec"] == [30, 40]
164174
assert r3["vec"] == [50, 60]
165175

176+
# LargeList<Utf8> -> Array<String>
177+
assert r1["labels"] == ["a", "b"]
178+
assert r2["labels"] == []
179+
assert r3["labels"] == ["c"]
180+
181+
# FixedSizeBinary -> Binary
182+
assert bytes(r1["digest"]) == b"\x01\x02\x03\x04"
183+
assert bytes(r3["digest"]) == b"\xff\xfe\xfd\xfc"
184+
185+
# Date64 -> Date32 (day-aligned)
186+
assert r1["day"] == datetime.date(2020, 9, 13)
187+
assert r2["day"] == datetime.date(2021, 1, 7)
188+
assert r3["day"] == datetime.date(2021, 5, 3)
189+
166190
# nested List<Struct<key,val>> passes through
167191
assert [(x["key"], x["val"]) for x in r1["attrs"]] == [("a", "1")]
168192
assert r2["attrs"] == []

‎examples/adbc-datafusion-driver/src/provider.rs‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ use std::sync::Arc;
4848

4949
use async_trait::async_trait;
5050
use datafusion::arrow::array::{
51-
ArrayRef, BinaryArray, FixedSizeListBuilder, Float16Array, Int64Array, ListBuilder,
52-
StringArray, StringBuilder, StructBuilder, TimestampNanosecondArray, UInt16Array,
53-
UInt16Builder, UInt64Array,
51+
ArrayRef, BinaryArray, Date64Array, FixedSizeBinaryBuilder, FixedSizeListBuilder, Float16Array,
52+
Int64Array, LargeListBuilder, ListBuilder, StringArray, StringBuilder, StructBuilder,
53+
TimestampNanosecondArray, UInt16Array, UInt16Builder, UInt64Array,
5454
};
5555
use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
5656
use datafusion::arrow::record_batch::RecordBatch;
@@ -149,6 +149,12 @@ struct Row {
149149
/// A fixed-size list of two unsigned ints -- Spark cannot read FixedSizeList and must have it
150150
/// cast to a variable List (with the element widened). Always length 2.
151151
vec: [u16; 2],
152+
/// A LargeList of strings -- maps to ArrayType but has no accessor, so it casts to List.
153+
labels: &'static [&'static str],
154+
/// FixedSizeBinary(4) -- maps to BinaryType but has no accessor, so it casts to Binary.
155+
digest: [u8; 4],
156+
/// Date64 (ms since epoch, day-aligned) -- no accessor, casts to Date32.
157+
day_ms: i64,
152158
}
153159

154160
// Values are chosen to exercise the widening edges: channel spans past i16::MAX, big includes
@@ -166,6 +172,9 @@ const TYPE_ROWS: [Row; 3] = [
166172
score: 1.5,
167173
tags: &[1, 2],
168174
vec: [10, 20],
175+
labels: &["a", "b"],
176+
digest: [0x01, 0x02, 0x03, 0x04],
177+
day_ms: 1_599_955_200_000, // 2020-09-13 (18518 days)
169178
},
170179
Row {
171180
id: 2,
@@ -178,6 +187,9 @@ const TYPE_ROWS: [Row; 3] = [
178187
score: 2.5,
179188
tags: &[],
180189
vec: [30, 40],
190+
labels: &[],
191+
digest: [0x00, 0x00, 0x00, 0x00],
192+
day_ms: 1_609_977_600_000, // 2021-01-07 (18634 days)
181193
},
182194
Row {
183195
id: 3,
@@ -190,6 +202,9 @@ const TYPE_ROWS: [Row; 3] = [
190202
score: 3.5,
191203
tags: &[3],
192204
vec: [50, 60],
205+
labels: &["c"],
206+
digest: [0xff, 0xfe, 0xfd, 0xfc],
207+
day_ms: 1_620_000_000_000, // 2021-05-03 (18750 days)
193208
},
194209
];
195210

@@ -217,6 +232,9 @@ impl TypesTableProvider {
217232
Field::new("score", proto[7].data_type().clone(), true),
218233
Field::new("tags", proto[8].data_type().clone(), true),
219234
Field::new("vec", proto[9].data_type().clone(), true),
235+
Field::new("labels", proto[10].data_type().clone(), true),
236+
Field::new("digest", proto[11].data_type().clone(), true),
237+
Field::new("day", proto[12].data_type().clone(), true),
220238
]));
221239

222240
// One self-contained single-row batch per partition (see the module docs on why we do
@@ -265,7 +283,7 @@ impl TableProvider for TypesTableProvider {
265283
}
266284
}
267285

268-
/// Build the ten single-row column arrays for one [`Row`], in schema order.
286+
/// Build the single-row column arrays for one [`Row`], in schema order.
269287
fn row_columns(row: &Row) -> Vec<ArrayRef> {
270288
let mut tags = ListBuilder::new(UInt16Builder::new());
271289
for t in row.tags {
@@ -279,6 +297,15 @@ fn row_columns(row: &Row) -> Vec<ArrayRef> {
279297
}
280298
vec.append(true);
281299

300+
let mut labels = LargeListBuilder::new(StringBuilder::new());
301+
for l in row.labels {
302+
labels.values().append_value(l);
303+
}
304+
labels.append(true);
305+
306+
let mut digest = FixedSizeBinaryBuilder::new(row.digest.len() as i32);
307+
digest.append_value(row.digest).expect("digest bytes");
308+
282309
let attr_fields = vec![
283310
Field::new("key", DataType::Utf8, true),
284311
Field::new("val", DataType::Utf8, true),
@@ -307,5 +334,8 @@ fn row_columns(row: &Row) -> Vec<ArrayRef> {
307334
Arc::new(Float16Array::from(vec![f16::from_f32(row.score)])),
308335
Arc::new(tags.finish()) as ArrayRef,
309336
Arc::new(vec.finish()) as ArrayRef,
337+
Arc::new(labels.finish()) as ArrayRef,
338+
Arc::new(digest.finish()) as ArrayRef,
339+
Arc::new(Date64Array::from(vec![row.day_ms])),
310340
]
311341
}

0 commit comments

Comments
 (0)