diff --git a/native/core/src/execution/columnar_to_row.rs b/native/core/src/execution/columnar_to_row.rs index 74defde6729..5d241ed2f69 100644 --- a/native/core/src/execution/columnar_to_row.rs +++ b/native/core/src/execution/columnar_to_row.rs @@ -884,6 +884,10 @@ pub struct ColumnarToRowContext { _batch_size: usize, /// Whether all columns are fixed-width (enables fast path). all_fixed_width: bool, + /// Flat path scratch: per-row lengths, row starts (num_rows + 1), var-length cursors. + row_lens: Vec, + row_starts: Vec, + cursors: Vec, } impl ColumnarToRowContext { @@ -917,6 +921,9 @@ impl ColumnarToRowContext { fixed_width_size, _batch_size: batch_size, all_fixed_width, + row_lens: Vec::new(), + row_starts: Vec::new(), + cursors: Vec::new(), } } @@ -1911,11 +1918,393 @@ fn write_nested_variable_to_buffer( } } +/// Whether a type can be written by the column-wise flat path: a fixed-width type, or a +/// variable-length type whose bytes are a single contiguous value. +fn is_flat(data_type: &DataType) -> bool { + is_fixed_width(data_type) + || matches!( + data_type, + DataType::Utf8 + | DataType::LargeUtf8 + | DataType::Binary + | DataType::LargeBinary + | DataType::FixedSizeBinary(_) + | DataType::Decimal128(_, _) + ) +} + +/// Big-endian minimal two's complement bytes of `value`, as `i128_to_spark_decimal_bytes` +/// returns them, without allocating. Returns the buffer and the start of the value in it. +#[inline] +fn decimal_bytes(value: i128) -> ([u8; 16], usize) { + let bytes = value.to_be_bytes(); + let sign_byte = if value < 0 { 0xFF } else { 0x00 }; + let mut start = 0; + while start < 15 && bytes[start] == sign_byte && ((bytes[start + 1] ^ sign_byte) & 0x80) == 0 { + start += 1; + } + (bytes, start) +} + +/// Adds each valid row's padded value length to `row_lens`. +#[inline] +fn add_var_lens usize>( + row_lens: &mut [usize], + nulls: Option<&arrow::buffer::NullBuffer>, + len_at: F, +) { + match nulls { + None => { + for (i, l) in row_lens.iter_mut().enumerate() { + *l += round_up_to_8(len_at(i)); + } + } + Some(nb) => { + for (i, l) in row_lens.iter_mut().enumerate() { + if nb.is_valid(i) { + *l += round_up_to_8(len_at(i)); + } + } + } + } +} + +impl ColumnarToRowContext { + /// Whether every column in the schema can use the column-wise flat path. + pub fn is_flat(&self) -> bool { + self.schema.iter().all(is_flat) + } + + /// Casts each array to the schema type, as `convert` does. + pub fn prepare(&self, arrays: &[ArrayRef]) -> CometResult> { + if arrays.len() != self.schema.len() { + return Err(CometError::Internal(format!( + "Column count mismatch: expected {}, got {}", + self.schema.len(), + arrays.len() + ))); + } + arrays + .iter() + .zip(self.schema.iter()) + .map(|(arr, schema_type)| Self::maybe_cast_to_schema_type(arr, schema_type)) + .collect() + } + + /// Computes the start of each row for the flat path and returns the total size in bytes. + /// `row_starts` gets `num_rows + 1` entries, the last being the total. + pub fn plan_flat(&mut self, arrays: &[ArrayRef], num_rows: usize) -> CometResult { + let fixed = self.fixed_width_size; + let mut row_lens = std::mem::take(&mut self.row_lens); + row_lens.clear(); + row_lens.resize(num_rows, fixed); + for array in arrays { + let nulls = array.nulls(); + match array.data_type() { + DataType::Utf8 => { + let o = array.as_string::().value_offsets(); + add_var_lens(&mut row_lens, nulls, |i| (o[i + 1] - o[i]) as usize) + } + DataType::LargeUtf8 => { + let o = array.as_string::().value_offsets(); + add_var_lens(&mut row_lens, nulls, |i| (o[i + 1] - o[i]) as usize) + } + DataType::Binary => { + let o = array.as_binary::().value_offsets(); + add_var_lens(&mut row_lens, nulls, |i| (o[i + 1] - o[i]) as usize) + } + DataType::LargeBinary => { + let o = array.as_binary::().value_offsets(); + add_var_lens(&mut row_lens, nulls, |i| (o[i + 1] - o[i]) as usize) + } + DataType::FixedSizeBinary(w) => { + let w = *w as usize; + add_var_lens(&mut row_lens, nulls, |_| w) + } + DataType::Decimal128(p, _) if *p > MAX_LONG_DIGITS => { + let v = array.as_primitive::().values(); + add_var_lens(&mut row_lens, nulls, |i| 16 - decimal_bytes(v[i]).1) + } + _ => {} + } + } + let starts = &mut self.row_starts; + starts.clear(); + starts.reserve(num_rows + 1); + let mut total = 0usize; + for &l in &row_lens { + starts.push(total); + total += l; + } + starts.push(total); + self.row_lens = row_lens; + Ok(total) + } + + /// The row starts computed by the last `plan_flat`. + pub fn row_starts(&self) -> &[usize] { + &self.row_starts + } + + /// Writes the rows planned by `plan_flat` into `out`, which must be zeroed and exactly the + /// planned size. Works column by column. + // Each loop indexes several per-row arrays (starts, cursors, values) by row. + #[allow(clippy::needless_range_loop)] + pub fn write_flat( + &mut self, + arrays: &[ArrayRef], + num_rows: usize, + out: &mut [u8], + ) -> CometResult<()> { + let starts = &self.row_starts; + debug_assert_eq!(starts.len(), num_rows + 1); + debug_assert_eq!(out.len(), starts[num_rows]); + let bitset = self.null_bitset_width; + let fixed = self.fixed_width_size; + let mut cursors = std::mem::take(&mut self.cursors); + cursors.clear(); + cursors.extend(starts[..num_rows].iter().map(|s| s + fixed)); + + for (col, array) in arrays.iter().enumerate() { + let slot = bitset + col * 8; + let nulls = array.nulls().filter(|n| n.null_count() > 0); + if let Some(nb) = nulls { + let word = (col / 64) * 8; + let mask = 1u64 << (col % 64); + for i in 0..num_rows { + if nb.is_null(i) { + let p = starts[i] + word; + let w = u64::from_le_bytes(out[p..p + 8].try_into().unwrap()) | mask; + out[p..p + 8].copy_from_slice(&w.to_le_bytes()); + } + } + } + let valid = |i: usize| nulls.is_none_or(|nb| nb.is_valid(i)); + + macro_rules! fixed_col { + ($t:ty, $f:expr) => {{ + let v = array.as_primitive::<$t>().values(); + for i in 0..num_rows { + if valid(i) { + let p = starts[i] + slot; + let x: i64 = $f(v[i]); + out[p..p + 8].copy_from_slice(&x.to_le_bytes()); + } + } + }}; + } + macro_rules! var_col { + ($get:expr) => {{ + for i in 0..num_rows { + if valid(i) { + let bytes: &[u8] = $get(i); + let len = bytes.len(); + // Like the row path, an empty value leaves the slot zero. + if len == 0 { + continue; + } + let dst = cursors[i]; + out[dst..dst + len].copy_from_slice(bytes); + let rel = (dst - starts[i]) as u64; + let p = starts[i] + slot; + out[p..p + 8] + .copy_from_slice(&((rel << 32) | len as u64).to_le_bytes()); + cursors[i] = dst + round_up_to_8(len); + } + } + }}; + } + + use arrow::datatypes::*; + match array.data_type() { + DataType::Boolean => { + let a = array.as_boolean(); + for i in 0..num_rows { + if valid(i) && a.value(i) { + out[starts[i] + slot] = 1; + } + } + } + DataType::Int8 => fixed_col!(Int8Type, |x: i8| x as i64), + DataType::Int16 => fixed_col!(Int16Type, |x: i16| x as i64), + DataType::Int32 => fixed_col!(Int32Type, |x: i32| x as i64), + DataType::Int64 => fixed_col!(Int64Type, |x: i64| x), + DataType::Float32 => fixed_col!(Float32Type, |x: f32| x.to_bits() as i64), + DataType::Float64 => fixed_col!(Float64Type, |x: f64| x.to_bits() as i64), + DataType::Date32 => fixed_col!(Date32Type, |x: i32| x as i64), + DataType::Timestamp(TimeUnit::Microsecond, _) => { + fixed_col!(TimestampMicrosecondType, |x: i64| x) + } + DataType::Time64(TimeUnit::Nanosecond) => { + fixed_col!(Time64NanosecondType, |x: i64| x) + } + DataType::Decimal128(p, _) if *p <= MAX_LONG_DIGITS => { + fixed_col!(Decimal128Type, |x: i128| x as i64) + } + DataType::Decimal128(_, _) => { + let v = array.as_primitive::().values(); + for i in 0..num_rows { + if valid(i) { + let (bytes, start) = decimal_bytes(v[i]); + let len = 16 - start; + let dst = cursors[i]; + out[dst..dst + len].copy_from_slice(&bytes[start..]); + let rel = (dst - starts[i]) as u64; + let p = starts[i] + slot; + out[p..p + 8] + .copy_from_slice(&((rel << 32) | len as u64).to_le_bytes()); + cursors[i] = dst + round_up_to_8(len); + } + } + } + DataType::Utf8 => { + let a = array.as_string::(); + var_col!(|i| a.value(i).as_bytes()) + } + DataType::LargeUtf8 => { + let a = array.as_string::(); + var_col!(|i| a.value(i).as_bytes()) + } + DataType::Binary => { + let a = array.as_binary::(); + var_col!(|i| a.value(i)) + } + DataType::LargeBinary => { + let a = array.as_binary::(); + var_col!(|i| a.value(i)) + } + DataType::FixedSizeBinary(_) => { + let a = array.as_fixed_size_binary(); + var_col!(|i| a.value(i)) + } + other => { + return Err(CometError::Internal(format!( + "Unexpected type in flat columnar-to-row path: {other:?}" + ))) + } + } + } + self.cursors = cursors; + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; use std::sync::Arc; + fn flat_matches_existing(arrays: Vec) { + let schema: Vec = arrays.iter().map(|a| a.data_type().clone()).collect(); + let n = arrays[0].len(); + let mut reference = ColumnarToRowContext::new(schema.clone(), n); + let (ptr, offsets, lengths) = reference.convert(&arrays, n).unwrap(); + let total = offsets + .last() + .map(|o| *o as usize + *lengths.last().unwrap() as usize); + let expected = unsafe { std::slice::from_raw_parts(ptr, total.unwrap_or(0)) }.to_vec(); + let expected_starts: Vec = offsets.iter().map(|o| *o as usize).collect(); + + let mut ctx = ColumnarToRowContext::new(schema, n); + assert!(ctx.is_flat()); + let prepared = ctx.prepare(&arrays).unwrap(); + let size = ctx.plan_flat(&prepared, n).unwrap(); + let mut out = vec![0u8; size]; + ctx.write_flat(&prepared, n, &mut out).unwrap(); + assert_eq!(&ctx.row_starts()[..n], expected_starts.as_slice()); + if let Some(pos) = out.iter().zip(&expected).position(|(a, b)| a != b) { + let row = expected_starts.partition_point(|s| *s <= pos) - 1; + let start = expected_starts[row]; + panic!( + "byte {pos} differs (row {row}, offset in row {}): got {:?} expected {:?}; schema {:?}", + pos - start, + &out[start..start + 48.min(out.len() - start)], + &expected[start..start + 48.min(expected.len() - start)], + arrays.iter().map(|a| a.data_type()).collect::>() + ); + } + assert_eq!(out, expected); + } + + #[test] + fn test_flat_path_matches_general_path() { + let n = 300; + let ints: ArrayRef = Arc::new(Int32Array::from_iter((0..n).map(|i| { + if i % 7 == 0 { + None + } else { + Some(i * 3 - 100) + } + }))); + let longs: ArrayRef = Arc::new(Int64Array::from_iter_values((0..n).map(|i| i as i64 * -7))); + let bools: ArrayRef = Arc::new(BooleanArray::from_iter((0..n).map(|i| { + if i % 5 == 0 { + None + } else { + Some(i % 2 == 0) + } + }))); + let doubles: ArrayRef = Arc::new(Float64Array::from_iter((0..n).map(|i| { + if i % 11 == 0 { + None + } else { + Some(i as f64 / 3.0) + } + }))); + let strings: ArrayRef = Arc::new(StringArray::from_iter((0..n).map(|i| { + if i % 4 == 0 { + None + } else { + Some("x".repeat(i as usize % 23)) + } + }))); + let large: ArrayRef = Arc::new(LargeBinaryArray::from_iter((0..n).map(|i| { + if i % 6 == 0 { + None + } else { + Some(vec![i as u8; i as usize % 17]) + } + }))); + let small_dec: ArrayRef = Arc::new( + Decimal128Array::from_iter((0..n).map(|i| { + if i % 9 == 0 { + None + } else { + Some(i as i128 * 12345 - 99999) + } + })) + .with_precision_and_scale(12, 2) + .unwrap(), + ); + let wide_dec: ArrayRef = Arc::new( + Decimal128Array::from_iter((0..n).map(|i| { + if i % 8 == 0 { + None + } else { + Some((i as i128 - 150) * 10i128.pow((i % 30) as u32)) + } + })) + .with_precision_and_scale(38, 10) + .unwrap(), + ); + let date: ArrayRef = Arc::new(Date32Array::from_iter_values(0..n)); + let all = vec![ + ints, longs, bools, doubles, strings, large, small_dec, wide_dec, date, + ]; + flat_matches_existing(all.clone()); + // Sliced inputs exercise non-zero array offsets. + flat_matches_existing(all.iter().map(|a| a.slice(13, 200)).collect()); + // Fixed-width only, compared against the existing fixed-width fast path. + flat_matches_existing(vec![ + Arc::clone(&all[0]), + Arc::clone(&all[1]), + Arc::clone(&all[2]), + Arc::clone(&all[6]), + ]); + // More than 64 columns spill the null bitset into a second word. + flat_matches_existing((0..70).map(|i| Arc::clone(&all[i % all.len()])).collect()); + } + #[test] fn test_bitset_width_calculation() { assert_eq!(ColumnarToRowContext::calculate_bitset_width(0), 0); diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index d915abfed0b..8adc7e31540 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -2614,6 +2614,224 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_columnarToRowConvert( }) } +/// Converts a batch to Spark `UnsafeRow`s in a new JVM `byte[]`, writing the start of each row +/// and the total length (`num_rows + 1` entries) into `offsets_out`. +/// +/// # Safety +/// This function is inherently unsafe since it deals with raw pointers passed from JNI. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_comet_Native_columnarToRowConvertToArray( + e: EnvUnowned, + _class: JClass, + c2r_handle: jlong, + array_addrs: JLongArray, + schema_addrs: JLongArray, + num_rows: jint, + offsets_out: jni::objects::JIntArray, +) -> jni::sys::jobject { + try_unwrap_or_throw(&e, |env| { + let ctx = (c2r_handle as *mut ColumnarToRowContext) + .as_mut() + .ok_or_else(|| CometError::Internal("Null columnar to row context".to_string()))?; + let n = num_rows as usize; + let imported = { + let array_addrs = unsafe { array_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + let schema_addrs = unsafe { schema_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + unsafe { import_c2r_arrays(&array_addrs, &schema_addrs)? } + }; + let too_large = |total: usize| { + CometError::Internal(format!( + "Batch too large for columnar-to-row conversion: {total} bytes" + )) + }; + let (bytes, starts): (jni::objects::JByteArray, Vec) = if ctx.is_flat() { + let arrays = ctx.prepare(&imported)?; + let total = ctx.plan_flat(&arrays, n)?; + if total > (i32::MAX - 15) as usize { + return Err(too_large(total)); + } + let bytes = env.new_byte_array(total)?; + { + let mut elems = unsafe { bytes.get_elements_critical(env, ReleaseMode::CopyBack)? }; + let out: &mut [u8] = + unsafe { std::slice::from_raw_parts_mut(elems.as_mut_ptr() as *mut u8, total) }; + ctx.write_flat(&arrays, n, out)?; + } + let starts = ctx.row_starts().iter().map(|&s| s as i32).collect(); + (bytes, starts) + } else { + let (ptr, offsets, lengths) = ctx.convert(&imported, n)?; + let total = offsets + .last() + .zip(lengths.last()) + .map(|(o, l)| *o as usize + *l as usize) + .unwrap_or(0); + if total > (i32::MAX - 15) as usize { + return Err(too_large(total)); + } + let mut starts = Vec::with_capacity(n + 1); + starts.extend_from_slice(offsets); + starts.push(total as i32); + let slice = unsafe { std::slice::from_raw_parts(ptr, total) }; + (env.byte_array_from_slice(slice)?, starts) + }; + offsets_out.set_region(env, 0, &starts)?; + Ok(bytes.into_raw()) + }) +} + +/// Converts a batch whose columns are passed as raw Arrow buffer addresses, without the C Data +/// interface. `desc` holds four longs per column: validity, offsets and data addresses (0 when +/// absent) and flags (bit 0: 64-bit offsets). The JVM batch must stay alive for the call; the +/// arrays built here borrow its memory and are dropped before returning. +/// +/// # Safety +/// The addresses must point to Arrow buffers of `num_rows` values of the schema's types. +#[no_mangle] +pub unsafe extern "system" fn Java_org_apache_comet_Native_columnarToRowConvertRaw( + e: EnvUnowned, + _class: JClass, + c2r_handle: jlong, + desc: JLongArray, + num_rows: jint, + offsets_out: jni::objects::JIntArray, +) -> jni::sys::jobject { + try_unwrap_or_throw(&e, |env| { + let ctx = (c2r_handle as *mut ColumnarToRowContext) + .as_mut() + .ok_or_else(|| CometError::Internal("Null columnar to row context".to_string()))?; + let n = num_rows as usize; + let arrays = { + let desc = unsafe { desc.get_elements(env, ReleaseMode::NoCopyBack)? }; + unsafe { borrow_raw_c2r_arrays(ctx.schema(), &desc, n)? } + }; + let total = ctx.plan_flat(&arrays, n)?; + if total > (i32::MAX - 15) as usize { + return Err(CometError::Internal(format!( + "Batch too large for columnar-to-row conversion: {total} bytes" + ))); + } + let bytes = env.new_byte_array(total)?; + { + let mut elems = unsafe { bytes.get_elements_critical(env, ReleaseMode::CopyBack)? }; + let out: &mut [u8] = + unsafe { std::slice::from_raw_parts_mut(elems.as_mut_ptr() as *mut u8, total) }; + ctx.write_flat(&arrays, n, out)?; + } + drop(arrays); + // Row starts fit in i32 because the total does. + let starts: Vec = ctx.row_starts().iter().map(|&s| s as i32).collect(); + offsets_out.set_region(env, 0, &starts)?; + Ok(bytes.into_raw()) + }) +} + +/// Builds arrays that borrow raw JVM Arrow buffers, as described for `columnarToRowConvertRaw`. +/// +/// # Safety +/// See `Java_org_apache_comet_Native_columnarToRowConvertRaw`. +unsafe fn borrow_raw_c2r_arrays( + schema: &[arrow::datatypes::DataType], + desc: &[i64], + n: usize, +) -> CometResult> { + use arrow::buffer::Buffer; + use arrow::datatypes::DataType; + use std::ptr::NonNull; + if desc.len() != schema.len() * 4 { + return Err(CometError::Internal(format!( + "{} descriptor entries for {} columns", + desc.len(), + schema.len() + ))); + } + let owner: Arc = Arc::new(()); + let borrow = |addr: i64, len: usize| -> CometResult { + let ptr = NonNull::new(addr as *mut u8) + .ok_or_else(|| CometError::Internal("Null buffer address".to_string()))?; + Ok(unsafe { Buffer::from_custom_allocation(ptr, len, Arc::clone(&owner)) }) + }; + let mut arrays = Vec::with_capacity(schema.len()); + for (col, dt) in schema.iter().enumerate() { + let (validity, offsets, data, flags) = ( + desc[col * 4], + desc[col * 4 + 1], + desc[col * 4 + 2], + desc[col * 4 + 3], + ); + let large = flags & 1 != 0; + let nulls = if validity != 0 && n > 0 { + let nb = arrow::buffer::NullBuffer::new(arrow::buffer::BooleanBuffer::new( + borrow(validity, n.div_ceil(8))?, + 0, + n, + )); + (nb.null_count() > 0).then_some(nb) + } else { + None + }; + let (data_type, buffers) = match dt { + DataType::Utf8 | DataType::Binary => { + let width = if large { 8 } else { 4 }; + let offsets_buf = borrow(offsets, (n + 1) * width)?; + let end = if large { + offsets_buf.typed_data::()[n] as usize + } else { + offsets_buf.typed_data::()[n] as usize + }; + let data_buf = if end == 0 { + Buffer::from_vec(Vec::::new()) + } else { + borrow(data, end)? + }; + let data_type = match (dt, large) { + (DataType::Utf8, false) => DataType::Utf8, + (DataType::Utf8, true) => DataType::LargeUtf8, + (_, false) => DataType::Binary, + (_, true) => DataType::LargeBinary, + }; + (data_type, vec![offsets_buf, data_buf]) + } + DataType::Boolean => (dt.clone(), vec![borrow(data, n.div_ceil(8).max(1))?]), + other => { + let width = other.primitive_width().ok_or_else(|| { + CometError::Internal(format!("Unsupported raw C2R type {other:?}")) + })?; + (dt.clone(), vec![borrow(data, (n * width).max(1))?]) + } + }; + let data = unsafe { + arrow::array::ArrayData::builder(data_type) + .len(n) + .nulls(nulls) + .buffers(buffers) + .build_unchecked() + }; + arrays.push(arrow::array::make_array(data)); + } + Ok(arrays) +} + +/// Imports exported columns for columnar-to-row conversion. +/// +/// # Safety +/// Each address must point to an exported `FFI_ArrowArray` / `FFI_ArrowSchema`, whose ownership +/// moves to this call. +unsafe fn import_c2r_arrays( + array_addrs: &[i64], + schema_addrs: &[i64], +) -> CometResult> { + let mut arrays = Vec::with_capacity(array_addrs.len()); + for (a, s) in array_addrs.iter().zip(schema_addrs) { + let ffi_array = unsafe { std::ptr::read(*a as *mut FFI_ArrowArray) }; + let ffi_schema = unsafe { std::ptr::read(*s as *mut FFI_ArrowSchema) }; + let array_data = from_ffi(ffi_array, &ffi_schema) + .map_err(|e| CometError::Internal(format!("Failed to import array: {}", e)))?; + arrays.push(decode_string_arrays(&arrow::array::make_array(array_data))?); + } + Ok(arrays) +} + /// Imports the columns at `array_addrs` and `schema_addrs` and converts their first `num_rows` /// rows to Spark `UnsafeRow`s in `ctx`'s buffer. Returns the buffer address and the offset and /// length of each row in it, valid until the next conversion. Core of diff --git a/spark/src/main/scala/org/apache/comet/Native.scala b/spark/src/main/scala/org/apache/comet/Native.scala index c24bd762c3e..6b050756009 100644 --- a/spark/src/main/scala/org/apache/comet/Native.scala +++ b/spark/src/main/scala/org/apache/comet/Native.scala @@ -429,6 +429,33 @@ class Native extends NativeBase { schemaAddrs: Array[Long], numRows: Int): NativeColumnarToRowInfo + /** + * Convert Arrow columnar data to Spark UnsafeRow format in a new JVM byte array. + * + * @param offsetsOut + * receives the start of each row in the returned array, then its total length, so at least + * numRows + 1 entries. + * @return + * the rows, back to back. + */ + @native def columnarToRowConvertToArray( + c2rHandle: Long, + arrayAddrs: Array[Long], + schemaAddrs: Array[Long], + numRows: Int, + offsetsOut: Array[Int]): Array[Byte] + + /** + * Like columnarToRowConvertToArray, but reads each column from raw Arrow buffer addresses + * instead of the C Data interface. `desc` holds validity, offsets and data addresses and flags + * (bit 0: 64-bit offsets) for each column. + */ + @native def columnarToRowConvertRaw( + c2rHandle: Long, + desc: Array[Long], + numRows: Int, + offsetsOut: Array[Int]): Array[Byte] + /** * Close and release the native columnar to row converter. * diff --git a/spark/src/main/scala/org/apache/comet/NativeColumnarToRowConverter.scala b/spark/src/main/scala/org/apache/comet/NativeColumnarToRowConverter.scala index b6aad967504..44fdd42bb9f 100644 --- a/spark/src/main/scala/org/apache/comet/NativeColumnarToRowConverter.scala +++ b/spark/src/main/scala/org/apache/comet/NativeColumnarToRowConverter.scala @@ -19,13 +19,16 @@ package org.apache.comet +import org.apache.arrow.vector.{BaseLargeVariableWidthVector, BaseVariableWidthVector, FieldVector} +import org.apache.arrow.vector.types.Types.MinorType import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.UnsafeRow -import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.types._ import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.unsafe.Platform import org.apache.comet.serde.QueryPlanSerde -import org.apache.comet.vector.NativeUtil +import org.apache.comet.vector.{CometPlainVector, NativeUtil} /** * Native converter that converts Arrow columnar data to Spark UnsafeRow format. @@ -44,7 +47,11 @@ import org.apache.comet.vector.NativeUtil * @param batchSize * Maximum number of rows per batch (used for buffer pre-allocation) */ -class NativeColumnarToRowConverter(schema: StructType, batchSize: Int) extends AutoCloseable { +class NativeColumnarToRowConverter( + schema: StructType, + batchSize: Int, + heapRows: Boolean = !java.lang.Boolean.getBoolean("comet.c2r.legacy")) + extends AutoCloseable { private val nativeLib = new Native() private val nativeUtil = new NativeUtil() @@ -66,6 +73,48 @@ class NativeColumnarToRowConverter(schema: StructType, batchSize: Int) extends A // Reusable UnsafeRow for iteration private val unsafeRow = new UnsafeRow(schema.fields.length) + // Heap-rows mode reuses one set of FFI structs for every batch. + private val numCols = schema.fields.length + private lazy val (ffiArrays, ffiSchemas) = nativeUtil.allocateArrowStructs(numCols) + private lazy val ffiArrayAddrs = ffiArrays.map(_.memoryAddress()) + private lazy val ffiSchemaAddrs = ffiSchemas.map(_.memoryAddress()) + + // Raw mode: the Arrow minor types each column may arrive as, for reading its buffers directly. + private val rawTypes: Array[Set[MinorType]] = + schema.fields.map(f => NativeColumnarToRowConverter.rawMinorTypes(f.dataType)) + private val rawEligible = heapRows && rawTypes.forall(_.nonEmpty) + private val rawDesc = new Array[Long](numCols * 4) + + /** Fills rawDesc from the batch's buffers, or returns false when a column can't be read raw. */ + private def fillRawDesc(batch: ColumnarBatch): Boolean = { + val numRows = batch.numRows() + var c = 0 + while (c < numCols) { + batch.column(c) match { + case v: CometPlainVector if v.getValueVector.getField.getDictionary == null => + val vv = v.getValueVector + if (vv.getValueCount != numRows || !rawTypes(c).contains(vv.getMinorType)) return false + val fv = vv.asInstanceOf[FieldVector] + rawDesc(c * 4) = fv.getValidityBufferAddress + vv match { + case bv: BaseVariableWidthVector => + rawDesc(c * 4 + 1) = bv.getOffsetBufferAddress + rawDesc(c * 4 + 3) = 0 + case lv: BaseLargeVariableWidthVector => + rawDesc(c * 4 + 1) = lv.getOffsetBufferAddress + rawDesc(c * 4 + 3) = 1 + case _ => + rawDesc(c * 4 + 1) = 0 + rawDesc(c * 4 + 3) = 0 + } + rawDesc(c * 4 + 2) = fv.getDataBufferAddress + case _ => return false + } + c += 1 + } + true + } + /** * Converts a ColumnarBatch to an iterator of InternalRows. * @@ -87,6 +136,26 @@ class NativeColumnarToRowConverter(schema: StructType, batchSize: Int) extends A return Iterator.empty } + if (rawEligible && fillRawDesc(batch)) { + // Each batch gets its own offsets, since its iterator may outlive the next convert(). + val offsets = new Array[Int](numRows + 1) + val bytes = nativeLib.columnarToRowConvertRaw(c2rHandle, rawDesc, numRows, offsets) + return new HeapRowIterator(bytes, offsets, numRows, numCols) + } + + if (heapRows) { + val exported = nativeUtil.exportBatch(ffiArrayAddrs, ffiSchemaAddrs, batch) + val offsets = new Array[Int](exported + 1) + val bytes = + nativeLib.columnarToRowConvertToArray( + c2rHandle, + ffiArrayAddrs, + ffiSchemaAddrs, + exported, + offsets) + return new HeapRowIterator(bytes, offsets, exported, numCols) + } + // Export the batch to Arrow FFI and get memory addresses val (arrayAddrs, schemaAddrs, exportedNumRows) = nativeUtil.exportBatchToAddresses(batch) @@ -110,6 +179,10 @@ class NativeColumnarToRowConverter(schema: StructType, batchSize: Int) extends A nativeLib.columnarToRowClose(c2rHandle) c2rHandle = 0 } + if (heapRows && numCols > 0) { + ffiArrays.foreach(_.close()) + ffiSchemas.foreach(_.close()) + } nativeUtil.close() } } @@ -143,3 +216,52 @@ private class NativeRowIterator(info: NativeColumnarToRowInfo, unsafeRow: Unsafe unsafeRow.copy() } } + +object NativeColumnarToRowConverter { + + /** The Arrow minor types a column of `dataType` can be read from raw, or empty if none. */ + private[comet] def rawMinorTypes(dataType: DataType): Set[MinorType] = dataType match { + case BooleanType => Set(MinorType.BIT) + case ByteType => Set(MinorType.TINYINT) + case ShortType => Set(MinorType.SMALLINT) + case IntegerType => Set(MinorType.INT) + case LongType => Set(MinorType.BIGINT) + case FloatType => Set(MinorType.FLOAT4) + case DoubleType => Set(MinorType.FLOAT8) + case DateType => Set(MinorType.DATEDAY) + case TimestampType => Set(MinorType.TIMESTAMPMICROTZ) + case TimestampNTZType => Set(MinorType.TIMESTAMPMICRO) + case _: DecimalType => Set(MinorType.DECIMAL) + case _: StringType => Set(MinorType.VARCHAR, MinorType.LARGEVARCHAR) + case BinaryType => Set(MinorType.VARBINARY, MinorType.LARGEVARBINARY) + case _ => Set.empty + } +} + +/** + * Iterator over rows that native conversion wrote into a JVM byte array, one array per batch. + * Each row is a new UnsafeRow over that array, so it stays valid after later batches without + * copying its bytes. + */ +private class HeapRowIterator( + bytes: Array[Byte], + offsets: Array[Int], + numRows: Int, + numFields: Int) + extends Iterator[InternalRow] { + + private var currentIdx = 0 + + override def hasNext: Boolean = currentIdx < numRows + + override def next(): InternalRow = { + if (!hasNext) { + throw new NoSuchElementException("No more rows") + } + val start = offsets(currentIdx) + val row = new UnsafeRow(numFields) + row.pointTo(bytes, Platform.BYTE_ARRAY_OFFSET + start, offsets(currentIdx + 1) - start) + currentIdx += 1 + row + } +} diff --git a/spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala b/spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala index 8dd49d8fb1f..7dd7f00efdf 100644 --- a/spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala +++ b/spark/src/main/scala/org/apache/comet/vector/NativeUtil.scala @@ -123,14 +123,22 @@ class NativeUtil extends AutoCloseable { arrayAddrs: Array[Long], schemaAddrs: Array[Long], batch: ColumnarBatch): Int = { - val numRows = mutable.ArrayBuffer.empty[Int] + // Tracks the row counts without boxing: one count, plus the full list only on a mismatch. + var firstNumRows = -1 + var numRowsMismatch = false + def recordNumRows(n: Int): Unit = { + if (firstNumRows < 0) firstNumRows = n + else if (n != firstNumRows) numRowsMismatch = true + } - (0 until batch.numCols()).foreach { index => + val numCols = batch.numCols() + var index = 0 + while (index < numCols) { batch.column(index) match { case a: CometVector => val valueVector = a.getValueVector - numRows += valueVector.getValueCount + recordNumRows(valueVector.getValueCount) val provider = if (valueVector.getField.getDictionary != null) { a.getDictionaryProvider @@ -157,7 +165,7 @@ class NativeUtil extends AutoCloseable { val materialised = ConstantColumnVectors .materialize(cv, cv.dataType(), batch.numRows(), s"_const_$index", allocator, "UTC") - numRows += materialised.getValueCount + recordNumRows(materialised.getValueCount) val arrowSchema = ArrowSchema.wrap(schemaAddrs(index)) val arrowArray = ArrowArray.wrap(arrayAddrs(index)) @@ -167,14 +175,21 @@ class NativeUtil extends AutoCloseable { "Comet execution only takes Arrow Arrays, but got " + s"${c.getClass}") } + index += 1 } - if (numRows.distinct.length > 1) { + if (numRowsMismatch) { + val counts = (0 until numCols).map { i => + batch.column(i) match { + case a: CometVector => a.getValueVector.getValueCount + case _ => batch.numRows() + } + } throw new SparkException( - s"Number of rows in each column should be the same, but got [${numRows.distinct}]") + s"Number of rows in each column should be the same, but got [${counts.distinct}]") } - numRows.headOption.getOrElse(batch.numRows()) + if (firstNumRows < 0) batch.numRows() else firstNumRows } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometC2RIsolatedBench.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometC2RIsolatedBench.scala index 07d9f475754..c2b6748e24f 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometC2RIsolatedBench.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometC2RIsolatedBench.scala @@ -129,6 +129,60 @@ object CometC2RIsolatedBench { i => InternalRow((0 until 16).map(c => i.toLong + c): _*), row => row.getLong(0) + row.getLong(15)) + // Strings long enough that copying the bytes, not per-value overhead, dominates. + private val longStrings = Scenario( + "long, 2 x string(~100 bytes)", + new StructType() + .add("a", LongType) + .add("b", StringType) + .add("c", StringType), + i => + InternalRow( + i.toLong, + UTF8String.fromString(("b" * 90) + i), + UTF8String.fromString(("c" * 95) + i)), + row => row.getLong(0) + row.getLong(1)) + + // Nested types: the JVM path builds each array/struct/map through ColumnarArray / + // ColumnarRow accessors and writes it with UnsafeProjection's nested writers. The slot + // read in `consume` is the raw offset-and-size word, which avoids allocating in the sink. + private val intArrays = Scenario( + "long, array(8)", + new StructType() + .add("a", LongType) + .add("b", ArrayType(IntegerType)), + i => + InternalRow( + i.toLong, + org.apache.spark.sql.catalyst.util.ArrayData.toArrayData((0 until 8).map(_ + i).toArray)), + row => row.getLong(0) + row.getLong(1)) + + private val structs = Scenario( + "long, struct", + new StructType() + .add("a", LongType) + .add( + "b", + new StructType() + .add("x", IntegerType) + .add("y", DoubleType) + .add("z", StringType)), + i => InternalRow(i.toLong, InternalRow(i, i.toDouble, UTF8String.fromString(s"value_$i"))), + row => row.getLong(0) + row.getLong(1)) + + private val maps = Scenario( + "long, map(4)", + new StructType() + .add("a", LongType) + .add("b", MapType(StringType, IntegerType)), + i => + InternalRow( + i.toLong, + org.apache.spark.sql.catalyst.util.ArrayBasedMapData( + (0 until 4).map(k => UTF8String.fromString(s"k$k")).toArray, + (0 until 4).map(_ + i).toArray)), + row => row.getLong(0) + row.getLong(1)) + private def makeBatches(scenario: Scenario, batchSize: Int): Array[ColumnarBatch] = { val rows = (0 until totalRows).iterator.map(scenario.makeRow) CometArrowConverters @@ -174,7 +228,8 @@ object CometC2RIsolatedBench { if (sink == Long.MinValue) println(sink) } - benchmark.addCase("JVM direct converter (prototype)") { _ => + val directSupported = DirectColumnarToRowConverter.supportsSchema(scenario.schema) + if (directSupported) benchmark.addCase("JVM direct converter (prototype)") { _ => val converter = new DirectColumnarToRowConverter(scenario.schema) var sink = 0L var b = 0 @@ -210,6 +265,25 @@ object CometC2RIsolatedBench { if (sink == Long.MinValue) println(sink) } + benchmark.addCase("Native converter (heap rows)") { _ => + val converter = + new NativeColumnarToRowConverter(scenario.schema, batchSize, heapRows = true) + var sink = 0L + try { + var b = 0 + while (b < batches.length) { + val it = converter.convert(batches(b)) + while (it.hasNext) { + sink += scenario.consume(it.next()) + } + b += 1 + } + } finally { + converter.close() + } + if (sink == Long.MinValue) println(sink) + } + benchmark.run() // GC pressure comparison: the JVM path allocates per object-typed value (Decimal, @@ -226,22 +300,41 @@ object CometC2RIsolatedBench { } if (sink == Long.MinValue) println(sink) } - val directAlloc = measureAllocatedBytes { - val converter = new DirectColumnarToRowConverter(scenario.schema) + val directAlloc = + if (!directSupported) -1L + else + measureAllocatedBytes { + val converter = new DirectColumnarToRowConverter(scenario.schema) + var sink = 0L + for (batch <- batches) { + converter.setBatch(batch) + val numRows = batch.numRows() + var i = 0 + while (i < numRows) { + sink += scenario.consume(converter.convertRow(i)) + i += 1 + } + } + if (sink == Long.MinValue) println(sink) + } + val nativeAlloc = measureAllocatedBytes { + val converter = new NativeColumnarToRowConverter(scenario.schema, batchSize) var sink = 0L - for (batch <- batches) { - converter.setBatch(batch) - val numRows = batch.numRows() - var i = 0 - while (i < numRows) { - sink += scenario.consume(converter.convertRow(i)) - i += 1 + try { + for (batch <- batches) { + val it = converter.convert(batch) + while (it.hasNext) { + sink += scenario.consume(it.next()) + } } + } finally { + converter.close() } if (sink == Long.MinValue) println(sink) } - val nativeAlloc = measureAllocatedBytes { - val converter = new NativeColumnarToRowConverter(scenario.schema, batchSize) + val heapAlloc = measureAllocatedBytes { + val converter = + new NativeColumnarToRowConverter(scenario.schema, batchSize, heapRows = true) var sink = 0L try { for (batch <- batches) { @@ -258,7 +351,8 @@ object CometC2RIsolatedBench { println( f"JVM heap allocation per row: JVM path ${jvmAlloc.toDouble / totalRows}%.1f bytes, " + f"direct path ${directAlloc.toDouble / totalRows}%.1f bytes, " + - f"native path ${nativeAlloc.toDouble / totalRows}%.1f bytes%n") + f"native path ${nativeAlloc.toDouble / totalRows}%.1f bytes, " + + f"native heap-rows path ${heapAlloc.toDouble / totalRows}%.1f bytes%n") batches.foreach(_.close()) } @@ -270,9 +364,21 @@ object CometC2RIsolatedBench { tpchDecimalsWithStrings, decimalsAllFixedWidth, highPrecisionDecimals, - widePrimitives) + widePrimitives, + longStrings, + intArrays, + structs, + maps) + // Optional filters: args(0) is a comma-separated list of scenario-name substrings, + // args(1) a comma-separated list of batch sizes. + val selected = args.headOption.filter(_.nonEmpty) match { + case Some(f) => scenarios.filter(s => f.split(",").exists(s.name.contains(_))) + case None => scenarios + } + val batchSizes = + if (args.length > 1) args(1).split(",").map(_.toInt).toSeq else Seq(8192, 512, 32) - for (scenario <- scenarios; batchSize <- Seq(8192, 512, 32)) { + for (scenario <- selected; batchSize <- batchSizes) { runForBatchSize(scenario, batchSize) } } diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala index 1a05bdbcd73..a892e804c54 100644 --- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala +++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala @@ -81,9 +81,23 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { * Helper method to add the standard benchmark cases for columnar to row conversion. Reduces * code duplication across benchmark methods. */ + private def withLegacyNative[T](legacy: Boolean)(body: => T): T = { + val key = "comet.c2r.legacy" + val old = System.getProperty(key) + System.setProperty(key, legacy.toString) + try body + finally if (old == null) System.clearProperty(key) else System.setProperty(key, old) + } + private def addC2RBenchmarkCases(benchmark: Benchmark, query: String): Unit = { - def addCase(name: String, expected: Class[_ <: SparkPlan], conf: (String, String)*): Unit = { - withSQLConf(conf: _*) { + def addCase(name: String, expected: Class[_ <: SparkPlan], conf: (String, String)*): Unit = + addCaseWith(name, expected, legacy = false, conf: _*) + def addCaseWith( + name: String, + expected: Class[_ <: SparkPlan], + legacy: Boolean, + conf: (String, String)*): Unit = { + withLegacyNative(legacy)(withSQLConf(conf: _*) { // Validate the actual noop write, whose plan can differ from the SELECT's plan. // Keep execution and listener synchronization outside the timed benchmark cases. val plans = ArrayBuffer.empty[SparkPlan] @@ -117,11 +131,11 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { s"$name did not execute the expected columnar-to-row conversion.\n" + plans.mkString("\n")) benchmark.out.println(s"Verified $name") - } + }) benchmark.addCase(name) { _ => - withSQLConf(conf: _*) { + withLegacyNative(legacy)(withSQLConf(conf: _*) { spark.sql(query).noop() - } + }) } } @@ -137,14 +151,57 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED.key -> "false") + addCaseWith( + "Comet Native, legacy (CometNativeColumnarToRowExec)", + classOf[CometNativeColumnarToRowExec], + legacy = true, + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED.key -> "true") + addCase( - "Comet Native (CometNativeColumnarToRowExec)", + "Comet Native, optimized (CometNativeColumnarToRowExec)", classOf[CometNativeColumnarToRowExec], CometConf.COMET_ENABLED.key -> "true", CometConf.COMET_EXEC_ENABLED.key -> "true", CometConf.COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED.key -> "true") } + /** TPC-H lineitem-like columns: compact decimals, a date and two short flag strings. */ + def tpchLikeBenchmark(values: Int, filter: String, label: String): Unit = { + val benchmark = + new Benchmark( + s"Columnar to Row - TPC-H-like decimals ($label)", + values.toLong, + output = output) + + withTempPath { dir => + withTempTable("parquetV1Table") { + val df = spark + .range(values.toLong) + .selectExpr( + "id", + "cast(id % 5000000 / 100 as decimal(12,2)) as l_quantity", + "cast(id % 9000000 / 100 as decimal(12,2)) as l_extendedprice", + "cast(id % 10 / 100 as decimal(12,2)) as l_discount", + "cast(id % 8 / 100 as decimal(12,2)) as l_tax", + "date_add(to_date('1992-01-01'), cast(id % 2500 as int)) as l_shipdate", + "if(id % 2 = 0, 'A', 'R') as l_returnflag", + "if(id % 3 = 0, 'F', 'O') as l_linestatus") + + prepareTable(dir, df) + spark.udf.register("plus_one", (x: Long) => x + 1) + val query = + if (filter == "udf") { + "SELECT plus_one(id), l_quantity, l_extendedprice, l_discount, l_tax, " + + "l_shipdate, l_returnflag, l_linestatus FROM parquetV1Table" + } else s"SELECT * FROM parquetV1Table $filter" + addC2RBenchmarkCases(benchmark, query) + benchmark.run() + } + } + } + /** * Benchmark columnar to row conversion for primitive types. */ @@ -478,6 +535,23 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase { override def runCometBenchmark(mainArgs: Array[String]): Unit = { val numRows = 1024 * 1024 // 1M rows + val only = sys.props.get("c2r.only") + def runBenchmark(name: String)(f: => Unit): Unit = + if (only.forall(o => o.split(",").exists(name.contains(_)))) super.runBenchmark(name)(f) + + runBenchmark("Columnar to Row Conversion - TPC-H-like") { + tpchLikeBenchmark(numRows * 4, "", "all rows") + } + + runBenchmark("Columnar to Row Conversion - TPC-H-like, JVM UDF projection") { + tpchLikeBenchmark(numRows * 4, "udf", "Scala UDF projection consumer") + } + + runBenchmark("Columnar to Row Conversion - TPC-H-like, selective filter") { + // Keeps 1 row in 100, so each batch reaching the conversion has about 80 rows. + tpchLikeBenchmark(numRows * 4, "WHERE id % 100 = 0", "1% selective filter") + } + runBenchmark("Columnar to Row Conversion - Fixed Width Only") { fixedWidthOnlyBenchmark(numRows) }