From f4ca5934a0a9da54182ad9e173a58f7b77f89d92 Mon Sep 17 00:00:00 2001 From: damn-am-star Date: Fri, 9 Oct 2026 09:41:07 -0400 Subject: [PATCH 1/5] Create Upstream file with introductory content Added introductory content for the Upstream file. --- Upstream | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Upstream diff --git a/Upstream b/Upstream new file mode 100644 index 0000000000..d54c59e7fb --- /dev/null +++ b/Upstream @@ -0,0 +1,20 @@ +Browse. Chat. Meet. +Scroll down +Use Adam4Adam +on your smartphone +Age +Ethnicity +Body Type +Height +and more +Browse. + +Browse unlimited profiles in five different grids. Not only do you find out who's near you, who's new, and who's visiting, we've also got 27 free filters you can use to find Mr. Right or Mr. Right Now! +27 FREE FILTERS +Chat. +27 FREE FILTERS +Chat on and on and on on Adam4Adam, because there are no restrictions here. Send and receive unlimited messages, send photos directly into the conversation or send a smile to users that you like. +27 FREE FILTERS +Meet. +27 FREE FILTERS +We've redesigned our website, so not only is Adam4Adam easier to use, it's also easier to meet guys as well! The only thing that's missing, is you! Join now! From 24377022fb7302c0033871e80d02c31e28867aec Mon Sep 17 00:00:00 2001 From: damn-am-star Date: Fri, 9 Oct 2026 09:42:03 -0400 Subject: [PATCH 2/5] Update phrases in Upstream file --- Upstream | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 Upstream diff --git a/Upstream b/Upstream new file mode 100644 index 0000000000..58ea9e70a0 --- /dev/null +++ b/Upstream @@ -0,0 +1,20 @@ +Watch. See. Meet. +Scroll down +Use Adam4Adam +on your smartphone +Age +Ethnicity +Body Type +Height +and more +Browse. + +Browse unlimited profiles in five different grids. Not only do you find out who's near you, who's new, and who's visiting, we've also got 27 free filters you can use to find Mr. Right or Mr. Right Now! +27 FREE FILTERS +Chat. +27 FREE FILTERS +Chat on and on and on on Adam4Adam, because there are no restrictions here. Send and receive unlimited messages, send photos directly into the conversation or send a smile to users that you like. +27 FREE FILTERS +Meet. +27 FREE FILTERS +We've redesigned our website, so not only is Adam4Adam easier to use, it's also easier to meet guys as well! The only thing that's missing, is you! Join now! From 63214f16f9ed5ed6450f4c3817fd2e05d898e438 Mon Sep 17 00:00:00 2001 From: damn-am-star Date: Fri, 9 Oct 2026 22:36:13 -0400 Subject: [PATCH 3/5] Add initial implementation of Arrow crate --- Arrow | 446 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 Arrow diff --git a/Arrow b/Arrow new file mode 100644 index 0000000000..7e377f5a41 --- /dev/null +++ b/Arrow @@ -0,0 +1,446 @@ +Skip to main content +arrow/ +lib.rs +Search + +Summary +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A complete, safe, native Rust implementation of [Apache Arrow](https://arrow.apache.org), a cross-language +//! development platform for in-memory data. +//! +//! Please see the [arrow crates.io](https://crates.io/crates/arrow) +//! page for feature flags and tips to improve performance. +//! +//! # Platform Support +//! +//! Only little-endian platforms are officially supported and tested in CI. +//! Big-endian platforms are not tested in CI and may not work correctly. +//! Fixes for big-endian platforms are welcome and handled on a best-effort basis, +//! but compatibility is not guaranteed. +//! +//! # Columnar Format +//! +//! The [`array`] module provides statically typed implementations of all the array types as defined +//! by the [Arrow Columnar Format](https://arrow.apache.org/docs/format/Columnar.html) +//! +//! For example, an [`Int32Array`](array::Int32Array) represents a nullable array of `i32` +//! +//! ```rust +//! # use arrow::array::{Array, Int32Array}; +//! let array = Int32Array::from(vec![Some(1), None, Some(3)]); +//! assert_eq!(array.len(), 3); +//! assert_eq!(array.value(0), 1); +//! assert_eq!(array.is_null(1), true); +//! +//! let collected: Vec<_> = array.iter().collect(); +//! assert_eq!(collected, vec![Some(1), None, Some(3)]); +//! assert_eq!(array.values(), &[1, 0, 3]) +//! ``` +//! +//! It is also possible to write generic code for different concrete types. +//! For example, since the following function is generic over all primitively +//! typed arrays, when invoked the Rust compiler will generate specialized implementations +//! with optimized code for each concrete type. +//! +//! ```rust +//! # use std::iter::Sum; +//! # use arrow::array::{Float32Array, PrimitiveArray, TimestampNanosecondArray}; +//! # use arrow::datatypes::ArrowPrimitiveType; +//! # +//! fn sum(array: &PrimitiveArray) -> T::Native +//! where +//! T: ArrowPrimitiveType, +//! T::Native: Sum +//! { +//! array.iter().map(|v| v.unwrap_or_default()).sum() +//! } +//! +//! assert_eq!(sum(&Float32Array::from(vec![1.1, 2.9, 3.])), 7.); +//! assert_eq!(sum(&TimestampNanosecondArray::from(vec![1, 2, 3])), 6); +//! ``` +//! +//! And the following uses [`ArrayAccessor`] to implement a generic function +//! over all arrays with comparable values. +//! +//! [`ArrayAccessor`]: array::ArrayAccessor +//! +//! ```rust +//! # use arrow::array::{ArrayAccessor, ArrayIter, Int32Array, StringArray}; +//! # use arrow::datatypes::ArrowPrimitiveType; +//! # +//! fn min(array: T) -> Option +//! where +//! T::Item: Ord +//! { +//! ArrayIter::new(array).filter_map(|v| v).min() +//! } +//! +//! assert_eq!(min(&Int32Array::from(vec![4, 2, 1, 6])), Some(1)); +//! assert_eq!(min(&StringArray::from(vec!["b", "a", "c"])), Some("a")); +//! ``` +//! +//! **For more examples, and details consult the [arrow_array] docs.** +//! +//! # Type Erasure / Trait Objects +//! +//! It is common to write code that handles any type of array, without necessarily +//! knowing its concrete type. This is done using the [`Array`] trait and using +//! [`DataType`] to determine the appropriate `downcast_ref`. +//! +//! [`DataType`]: datatypes::DataType +//! +//! ```rust +//! # use arrow::array::{Array, Float32Array}; +//! # use arrow::array::StringArray; +//! # use arrow::datatypes::DataType; +//! # +//! fn impl_string(array: &StringArray) {} +//! fn impl_f32(array: &Float32Array) {} +//! +//! fn impl_dyn(array: &dyn Array) { +//! match array.data_type() { +//! // downcast `dyn Array` to concrete `StringArray` +//! DataType::Utf8 => impl_string(array.as_any().downcast_ref().unwrap()), +//! // downcast `dyn Array` to concrete `Float32Array` +//! DataType::Float32 => impl_f32(array.as_any().downcast_ref().unwrap()), +//! _ => unimplemented!() +//! } +//! } +//! ``` +//! +//! You can use the [`AsArray`] extension trait to facilitate downcasting: +//! +//! [`AsArray`]: crate::array::AsArray +//! +//! ```rust +//! # use arrow::array::{Array, Float32Array, AsArray}; +//! # use arrow::array::StringArray; +//! # use arrow::datatypes::DataType; +//! # +//! fn impl_string(array: &StringArray) {} +//! fn impl_f32(array: &Float32Array) {} +//! +//! fn impl_dyn(array: &dyn Array) { +//! match array.data_type() { +//! DataType::Utf8 => impl_string(array.as_string()), +//! DataType::Float32 => impl_f32(array.as_primitive()), +//! _ => unimplemented!() +//! } +//! } +//! ``` +//! +//! It is also common to want to write a function that returns one of a number of possible +//! array implementations. [`ArrayRef`] is a type-alias for [`Arc`](array::Array) +//! which is frequently used for this purpose +//! +//! ```rust +//! # use std::str::FromStr; +//! # use std::sync::Arc; +//! # use arrow::array::{ArrayRef, Int32Array, PrimitiveArray}; +//! # use arrow::datatypes::{ArrowPrimitiveType, DataType, Int32Type, UInt32Type}; +//! # use arrow::compute::cast; +//! # +//! fn parse_to_primitive<'a, T, I>(iter: I) -> PrimitiveArray +//! where +//! T: ArrowPrimitiveType, +//! T::Native: FromStr, +//! I: IntoIterator, +//! { +//! PrimitiveArray::from_iter(iter.into_iter().map(|val| T::Native::from_str(val).ok())) +//! } +//! +//! fn parse_strings<'a, I>(iter: I, to_data_type: DataType) -> ArrayRef +//! where +//! I: IntoIterator, +//! { +//! match to_data_type { +//! DataType::Int32 => Arc::new(parse_to_primitive::(iter)) as _, +//! DataType::UInt32 => Arc::new(parse_to_primitive::(iter)) as _, +//! _ => unimplemented!() +//! } +//! } +//! +//! let array = parse_strings(["1", "2", "3"], DataType::Int32); +//! let integers = array.as_any().downcast_ref::().unwrap(); +//! assert_eq!(integers.values(), &[1, 2, 3]) +//! ``` +//! +//! # Compute Kernels +//! +//! The [`compute`] module provides optimised implementations of many common operations, +//! for example the `parse_strings` operation above could also be implemented as follows: +//! +//! ``` +//! # use std::sync::Arc; +//! # use arrow::error::Result; +//! # use arrow::array::{ArrayRef, StringArray, UInt32Array}; +//! # use arrow::datatypes::DataType; +//! # +//! fn parse_strings<'a, I>(iter: I, to_data_type: &DataType) -> Result +//! where +//! I: IntoIterator, +//! { +//! let array = StringArray::from_iter(iter.into_iter().map(Some)); +//! arrow::compute::cast(&array, to_data_type) +//! } +//! +//! let array = parse_strings(["1", "2", "3"], &DataType::UInt32).unwrap(); +//! let integers = array.as_any().downcast_ref::().unwrap(); +//! assert_eq!(integers.values(), &[1, 2, 3]) +//! ``` +//! +//! This module also implements many common vertical operations: +//! +//! * All mathematical binary operators, such as [`sub`](compute::kernels::numeric::sub) +//! * All boolean binary operators such as [`equality`](compute::kernels::cmp::eq) +//! * [`cast`](compute::kernels::cast::cast) +//! * [`filter`](compute::kernels::filter::filter) +//! * [`take`](compute::kernels::take::take) +//! * [`sort`](compute::kernels::sort::sort) +//! * some string operators such as [`substring`](compute::kernels::substring::substring) and [`length`](compute::kernels::length::length) +//! +//! ``` +//! # use arrow::compute::kernels::cmp::gt; +//! # use arrow_array::cast::AsArray; +//! # use arrow_array::Int32Array; +//! # use arrow_array::types::Int32Type; +//! # use arrow_select::filter::filter; +//! let array = Int32Array::from_iter(0..100); +//! // Create a 32-bit integer scalar (single) value: +//! let scalar = Int32Array::new_scalar(60); +//! // find all rows in the array that are greater than 60 +//! let predicate = gt(&array, &scalar).unwrap(); +//! // copy all matching rows into a new array +//! let filtered = filter(&array, &predicate).unwrap(); +//! +//! let expected = Int32Array::from_iter(61..100); +//! assert_eq!(&expected, filtered.as_primitive::()); +//! ``` +//! +//! As well as some horizontal operations, such as: +//! +//! * [`min`](compute::kernels::aggregate::min) and [`max`](compute::kernels::aggregate::max) +//! * [`sum`](compute::kernels::aggregate::sum) +//! +//! # Tabular Representation +//! +//! It is common to want to group one or more columns together into a tabular representation. This +//! is provided by [`RecordBatch`] which combines a [`Schema`](datatypes::Schema) +//! and a corresponding list of [`ArrayRef`]. +//! +//! +//! ``` +//! # use std::sync::Arc; +//! # use arrow::array::{Float32Array, Int32Array}; +//! # use arrow::record_batch::RecordBatch; +//! # +//! let col_1 = Arc::new(Int32Array::from_iter([1, 2, 3])) as _; +//! let col_2 = Arc::new(Float32Array::from_iter([1., 6.3, 4.])) as _; +//! +//! let batch = RecordBatch::try_from_iter([("col1", col_1), ("col_2", col_2)]).unwrap(); +//! ``` +//! +//! # Pretty Printing +//! +//! See the [`util::pretty`] module (requires the `prettyprint` crate feature) +//! +//! # IO +//! +//! This crate provides readers and writers for various formats to/from [`RecordBatch`] +//! +//! * JSON: [`Reader`](json::reader::Reader) and [`Writer`](json::writer::Writer) +//! * CSV: [`Reader`](csv::reader::Reader) and [`Writer`](csv::writer::Writer) +//! * IPC: [`Reader`](ipc::reader::StreamReader) and [`Writer`](ipc::writer::FileWriter) +//! +//! Support for [Apache Parquet] is published as a [separate parquet crate](https://crates.io/crates/parquet) +//! +//! Support for [Apache Avro] is published as a [separate arrow-avro crate](https://crates.io/crates/arrow-avro) +//! +//! # Serde Compatibility +//! +//! [`arrow_json::reader::Decoder`] provides a mechanism to convert arbitrary, serde-compatible +//! structures into [`RecordBatch`]. +//! +//! Whilst likely less performant than implementing a custom builder, as described in +//! [arrow_array::builder], this provides a simple mechanism to get up and running quickly +//! +//! ``` +//! # use std::sync::Arc; +//! # use arrow_json::ReaderBuilder; +//! # use arrow_schema::{DataType, Field, Schema}; +//! # use serde::Serialize; +//! # use arrow_array::cast::AsArray; +//! # use arrow_array::types::{Float32Type, Int32Type}; +//! # +//! #[derive(Serialize)] +//! struct MyStruct { +//! int32: i32, +//! string: String, +//! } +//! +//! let schema = Schema::new(vec![ +//! Field::new("int32", DataType::Int32, false), +//! Field::new("string", DataType::Utf8, false), +//! ]); +//! +//! let rows = vec![ +//! MyStruct{ int32: 5, string: "bar".to_string() }, +//! MyStruct{ int32: 8, string: "foo".to_string() }, +//! ]; +//! +//! let mut decoder = ReaderBuilder::new(Arc::new(schema)).build_decoder().unwrap(); +//! decoder.serialize(&rows).unwrap(); +//! +//! let batch = decoder.flush().unwrap().unwrap(); +//! +//! // Expect batch containing two columns +//! let int32 = batch.column(0).as_primitive::(); +//! assert_eq!(int32.values(), &[5, 8]); +//! +//! let string = batch.column(1).as_string::(); +//! assert_eq!(string.value(0), "bar"); +//! assert_eq!(string.value(1), "foo"); +//! ``` +//! +//! # Crate Topology +//! +//! The [`arrow`] project is implemented as multiple sub-crates, which are then re-exported by +//! this top-level crate. +//! +//! Crate authors can choose to depend on this top-level crate, or just +//! the sub-crates they need. +//! +//! The current list of sub-crates is: +//! +//! * [`arrow-arith`][arrow_arith] - arithmetic kernels +//! * [`arrow-array`][arrow_array] - type-safe arrow array abstractions +//! * [`arrow-buffer`][arrow_buffer] - buffer abstractions for arrow arrays +//! * [`arrow-cast`][arrow_cast] - cast kernels for arrow arrays +//! * [`arrow-csv`][arrow_csv] - read/write CSV to arrow format +//! * [`arrow-data`][arrow_data] - the underlying data of arrow arrays +//! * [`arrow-ipc`][arrow_ipc] - read/write IPC to arrow format +//! * [`arrow-json`][arrow_json] - read/write JSON to arrow format +//! * [`arrow-ord`][arrow_ord] - ordering kernels for arrow arrays +//! * [`arrow-row`][arrow_row] - comparable row format +//! * [`arrow-schema`][arrow_schema] - the logical types for arrow arrays +//! * [`arrow-select`][arrow_select] - selection kernels for arrow arrays +//! * [`arrow-string`][arrow_string] - string kernels for arrow arrays +//! +//! Some functionality is also distributed independently of this crate: +//! +//! * [`arrow-flight`] - support for [Arrow Flight RPC] +//! * [`parquet`](https://docs.rs/parquet) - support for [Apache Parquet] +//! * [`arrow-avro`](https://docs.rs/arrow-avro) - support for [Apache Avro] +//! +//! # Security +//! +//! This project follows the [Apache Arrow Security Model]. +//! +//! Unexpected behavior (e.g., panics, crashes, or infinite loops) triggered by +//! malformed input is considered a **bug**, not a security vulnerability, +//! unless it is **exploitable** by an attacker to +//! +//! * Execute arbitrary code (Remote Code Execution); +//! * Exfiltrate sensitive information from process memory (Information Disclosure); +//! +//! If you think you have found a security vulnerability, please follow the +//! reporting instructions in the [security policy]. +//! +//! [security policy]: https://github.com/apache/arrow-rs/blob/main/SECURITY.md +//! +//! # Safety +//! +//! Like many crates, this crate makes use of `unsafe` where prudent. However, it endeavors to be +//! sound. Specifically, **it should not be possible to trigger undefined behavior using safe APIs.** +//! +//! Undefined behavior using safe APIs is considered a bug, not a security +//! vulnerability, unless it can be exploited. Please see the [security policy] +//! for details. +//! +//! For more information on the use of `unsafe`, see [here](https://github.com/apache/arrow-rs/tree/main/arrow#safety). +//! +//! [Apache Arrow Security Model]: https://arrow.apache.org/docs/dev/format/Security.html +//! +//! # Higher-level Processing +//! +//! This crate aims to provide reusable, low-level primitives for operating on columnar data. For +//! more sophisticated query processing workloads, consider checking out [DataFusion]. This +//! orchestrates the primitives exported by this crate into an embeddable query engine, with +//! SQL and DataFrame frontends, and heavily influences this crate's roadmap. +//! +//! [`arrow`]: https://github.com/apache/arrow-rs +//! [`array`]: mod@array +//! [`Array`]: array::Array +//! [`ArrayRef`]: array::ArrayRef +//! [`ArrayData`]: array::ArrayData +//! [`make_array`]: array::make_array +//! [`Buffer`]: buffer::Buffer +//! [`RecordBatch`]: record_batch::RecordBatch +//! [`arrow-flight`]: https://docs.rs/arrow-flight/latest/arrow_flight/ +//! [`parquet`]: https://docs.rs/parquet/latest/parquet/ +//! [Arrow Flight RPC]: https://arrow.apache.org/docs/format/Flight.html +//! [Arrow JSON Test Format]: https://github.com/apache/arrow/blob/master/docs/source/format/Integration.rst#json-test-data-format +//! [Apache Parquet]: https://parquet.apache.org/ +//! [Apache Avro]: https://avro.apache.org/ +//! [DataFusion]: https://github.com/apache/arrow-datafusion +//! [issue tracker]: https://github.com/apache/arrow-rs/issues + +#![doc( + html_logo_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_white-bg.svg", + html_favicon_url = "https://arrow.apache.org/img/arrow-logo_chevrons_black-txt_transparent-bg.svg" +)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![deny(clippy::redundant_clone)] +#![warn(missing_debug_implementations)] +#![warn(missing_docs)] +#![allow(rustdoc::invalid_html_tags)] +pub use arrow_array::{downcast_dictionary_array, downcast_primitive_array}; + +pub use arrow_buffer::{alloc, buffer}; + +/// Arrow crate version +pub const ARROW_VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub mod array; +pub mod compute; +#[cfg(feature = "csv")] +pub use arrow_csv as csv; +pub mod datatypes; +pub mod error; +#[cfg(feature = "ffi")] +pub use arrow_array::ffi; +#[cfg(feature = "ffi")] +pub use arrow_array::ffi_stream; +#[cfg(feature = "ipc")] +pub use arrow_ipc as ipc; +#[cfg(feature = "json")] +pub use arrow_json as json; +#[cfg(feature = "pyarrow")] +pub use arrow_pyarrow as pyarrow; + +/// Contains the `RecordBatch` type and associated traits +pub mod record_batch { + pub use arrow_array::{ + RecordBatch, RecordBatchIterator, RecordBatchOptions, RecordBatchReader, RecordBatchWriter, + }; +} +pub use arrow_array::temporal_conversions; +pub use arrow_row as row; +pub mod tensor; +pub mod util; From a1bafd52d166552a988224d075905111726fdd0a Mon Sep 17 00:00:00 2001 From: damn-am-star Date: Sat, 10 Oct 2026 11:07:04 -0400 Subject: [PATCH 4/5] Add LICENSE file for PassportCard --- LICENSE => PassportCard | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename LICENSE => PassportCard (99%) diff --git a/LICENSE b/PassportCard similarity index 99% rename from LICENSE rename to PassportCard index 261eeb9e9f..c816e4402f 100644 --- a/LICENSE +++ b/PassportCard @@ -4,7 +4,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - 1. Definitions. + 10. Def "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. From 5b68cd798a019c15edf2bd5b38709cae7c737574 Mon Sep 17 00:00:00 2001 From: damn-am-star Date: Mon, 12 Oct 2026 19:41:05 -0400 Subject: [PATCH 5/5] Add files via upload --- CLAUDE.md | 210 +----------------------------------------------------- 1 file changed, 1 insertion(+), 209 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8084538afc..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,209 +1 @@ -# CLAUDE.md - -## Project Overview - -Delta-kernel-rs is a Rust library for building Delta Lake connectors. It encapsulates the -Delta protocol so connectors can read and write Delta tables without understanding protocol -internals. Kernel never does I/O directly -- it defines _what_ to do via its APIs -(`Snapshot`, `Scan`, `Transaction`) and delegates _how_ to the `Engine` trait. - -Current capabilities: table reads with predicates, data skipping, deletion vectors, change -data feed, checkpoints (V1 & V2), log compaction, blind append writes, table creation -(including clustered tables), and catalog-managed table support. - -## Build & Test Commands - -```bash -# Build -cargo build --workspace --all-features - -# Run all tests (prefer nextest over cargo test) -cargo nextest run --workspace --all-features - -# Run tests for a specific crate -cargo nextest run -p delta_kernel --all-features - -# Run a single test in a specific crate (fastest -- only compiles that crate) -cargo nextest run -p delta_kernel --lib --all-features test_name_here - -# Run a test by name, searching all crates (slow -- compiles everything) -cargo nextest run --workspace --all-features test_name_here - -# Format, lint, and doc check (always run after code changes) -cargo fmt \ - && cargo clippy --workspace --benches --tests --all-features -- -D warnings \ - && cargo doc --workspace --all-features --no-deps - -# Quick pre-push check (mimics CI) -cargo fmt \ - && cargo clippy --workspace --benches --tests --all-features -- -D warnings \ - && cargo doc --workspace --all-features --no-deps \ - && cargo nextest run --workspace --all-features -``` - -### Crate Names for `-p` Flag - -| Crate | Directory | Description | -|------------------------|-----------------|------------------------------------------------| -| `delta_kernel` | `kernel/` | Core library | -| `delta_kernel_ffi` | `ffi/` | C/C++ FFI bindings | -| `delta_kernel_derive` | `derive-macros/`| Proc macros | -| `acceptance` | `acceptance/` | Acceptance tests (DAT) | -| `test_utils` | `test-utils/` | Shared test utilities | -| `feature_tests` | `feature-tests/`| Feature flag tests | -| `uc-catalog` | `uc-catalog/` | Unity Catalog integration (UCCatalog, UCCommitter) | -| `uc-client` | `uc-client/` | Unity Catalog REST client | - -### Feature Flags - -- `default-engine` / `default-engine-rustls` / `default-engine-native-tls` -- async - Arrow/Tokio engine (pick one TLS backend) -- `arrow`, `arrow-XX`, `arrow-YY` -- Arrow version selection (kernel tracks the latest two - major Arrow releases; `arrow` defaults to latest). Kernel itself does not depend on Arrow, - but default-engine does. -- `arrow-conversion`, `arrow-expression` -- Arrow interop (auto-enabled by default engine) -- `catalog-managed` -- catalog-managed table support (experimental) -- `clustered-table` -- clustered table write support (experimental) -- `internal-api` -- unstable APIs like `parallel_scan_metadata`. Items are marked with the - `#[internal_api]` proc macro attribute. -- `test-utils`, `integration-test` -- development only - -## Architecture at a Glance - -**Snapshot** is the entry point for everything -- an immutable view of a table at a specific -version. From it you build a `Scan` (reads) or `Transaction` (writes). - -**Read path:** `Snapshot` -> `ScanBuilder` -> `Scan` -> data. Three execution paths: -`execute()` (simple), `scan_metadata()` (advanced/distributed), -`parallel_scan_metadata()` (two-phase distributed log replay). - -**Write path:** `Snapshot` -> `Transaction` -> `commit()`. Kernel provides `WriteContext`, -assembles commit actions, enforces protocol compliance, delegates atomic commit to a -`Committer`. - -**Engine trait:** five handlers (`StorageHandler`, `JsonHandler`, `ParquetHandler`, -`EvaluationHandler`, optional `MetricsReporter`). `DefaultEngine` lives in -`kernel/src/engine/default/`. - -**EngineData:** opaque columnar data interface. IMPORTANT: never access `EngineData` columns -directly -- always use the visitor pattern (`visit_rows` with typed `GetData` accessors). - -## Testing - -- **Unit tests** test internal APIs and module internals. It is fine to use public APIs - like `create_table` in a unit test as setup (e.g. to create a table for testing reads, - writes, or state loading). -- **Integration tests** exercise only public APIs end-to-end. -- Consider how the feature interacts with Delta table features (see Protocol TLDR below). -- Consider write paths: normal commits, checkpointing, CRC files, log compaction files. -- Consider read paths: loading a snapshot from scratch at latest version, at a specific - version (time travel), and updating from an existing snapshot. -- Consider table state: only deltas (`00.json` to `0N.json`), after a checkpoint, after - a CRC (`0N.crc`) file, after log compaction, etc. -- Prefer descriptive test names over doc comments. Encode the scenario and expected - behavior in the test name. Only add a test doc comment when the intent is too - verbose or complex to express succinctly in the name. -- Use `rstest` to parameterize tests that share the same logic but differ in setup - or inputs. Prefer `#[case]` over duplicating test functions. When parameters are - independent and form a cartesian product, prefer `#[values]` over enumerating - every combination with `#[case]`. -- Reuse helpers from `test_utils` instead of writing custom ones when possible. -- **`add_commit` and table setup in tests:** `add_commit` takes a `table_root` string and - resolves it to an absolute object-store path. The `table_root` must be a proper URL string - with a trailing slash (e.g. `"memory:///"`, `"file:///tmp/my_table/"`). Avoid using the - `Url` type directly -- most test helpers and kernel APIs accept `impl AsRef`, so pass - URL strings instead. When using local storage, use an un-prefixed store - (`LocalFileSystem::new()`) with a `file:///` URL string. Do NOT use - `LocalFileSystem::new_with_prefix()` with `add_commit` -- the prefix causes double-nesting - because `add_commit` already resolves the full path from the URL. For in-memory tests, use - `InMemory::new()` with `"memory:///"`. Always use the same `table_root` URL string for - both `add_commit` (writing log files) and `snapshot`/`Snapshot::try_new` (reading the - table). Always include a trailing slash in directory URLs to ensure correct path joining. - -## Protocol TLDR - -The [Delta protocol spec](https://raw.githubusercontent.com/delta-io/delta/master/PROTOCOL.md) -is the source of truth. Key concepts: - -- **Actions** -- atomic units of a transaction: Metadata, Add File, Remove File, Add CDC - File, Protocol, CommitInfo, Domain Metadata, Sidecar, Checkpoint Metadata -- **Log structure** -- JSON commit files, checkpoints (V1 parquet, V2 multi-part), log - compaction files, version checksum (CRC) files, `_last_checkpoint` -- **Protocol versioning** -- (readerVersion, writerVersion) pair. At (3, 7) switches to - explicit table features via `readerFeatures`/`writerFeatures` arrays. Features cannot be - removed once added. -- **Data skipping** -- per-file column statistics (min, max, null count, row count) with - tight/wide bounds -- **Schemas** -- JSON serialization format for StructType/StructField/DataType -- **Stats and partition values** -- per-file column statistics (min, max, nullCount) and - partition values are stored as JSON strings in Add file actions. The stats JSON structure - mirrors the table schema. See the protocol spec sections on "Per-file Statistics" and - "Partition Value Serialization" for the exact formats. - -**Table features**: - -- Writer: `appendOnly`, `invariants`, `checkConstraints`, `generatedColumns`, - `allowColumnDefaults`, `changeDataFeed`, `identityColumns`, `rowTracking`, - `domainMetadata`, `icebergCompatV1`, `icebergCompatV2`, `clustering`, - `inCommitTimestamp` -- Reader + writer: `columnMapping`, `deletionVectors`, `timestampNtz`, - `v2Checkpoint`, `vacuumProtocolCheck`, `variantType`, `variantType-preview`, - `typeWidening` - -Keep this list updated when new protocol features are added to kernel. - -## Common Gotchas - -- **EngineData is opaque:** Never downcast to `ArrowEngineData` or any concrete type - in production code (ok in tests). Never assume one batch per file -- always iterate. -- **Column mapping:** Physical column names can differ from logical names. Always use - the schema from `Snapshot::schema()` for user data columns. Metadata/system schema - column names (defined by the protocol) are not subject to column mapping. -- **Transforms:** Generic recursive schema and expression transform traits and helpers - are in `kernel/src/transforms/`. - -## Code Style / Documentation - -- MUST include doc comments for all public functions, structs, enums, and methods. -- MUST document function parameters, return values, and errors. -- Keep comments up-to-date with code changes. -- Include examples in doc comments for complex functions only. -- NEVER use emoji or unicode in comments that emulates emoji (e.g. special arrows, - checkmarks). Use ASCII equivalents (`->`, `=>`, etc.) instead. -- Comments should be concise and non-repetitive -- find the right place to say it once. -- Comments should never include temporal references -- only refer to current code and - design, not past iterations. -- Doc comments focus on "what" (contract with caller) more than "how" (implementation), - unless the "how" meaningfully impacts the "what". -- Code comments state intent and explain "why" -- don't restate what the code self-documents. -- Place `use` imports at the top of the file (for non-test code) or at the top of the - `mod tests` block (for test code) -- never inside function bodies. -- NEVER panic in production code -- use errors instead. Panicking - (including `unwrap()`, `expect()`, `panic!()`, `unreachable!()`, etc) is acceptable in test code only. - -## Pull Requests - -**Title:** use conventional commit format, lowercase after prefix, no period at the end. -Allowed types: `feat`, `fix`, `refactor`, `chore`, `docs`, `perf`, `test`, `ci`. -If the pull request contains a breaking change, the type must have a `!` suffix. -Examples: `feat: add checkpoint stream support`, `fix: handle empty log segment`, -`refactor: extract common log replay logic` -Breaking change examples: `feat!: make_physical takes column mapping and sets parquet field ids`, -`chore!: remove the arrow-55 feature` - -**Description:** follow the template in `.github/PULL_REQUEST_TEMPLATE.md`. Error on the -side of simplicity -- don't list every change. Focus on key API changes, functionality, -and data flow. Keep it concise. - -## Deep Context - -Read these when relevant to the task at hand: -- `CLAUDE/architecture.md` -- kernel architecture: snapshot loading, read/write paths, - engine trait system, EngineData, key modules, catalog-managed tables -- Always cross-check protocol behavior against the - [Delta protocol spec](https://raw.githubusercontent.com/delta-io/delta/master/PROTOCOL.md) - -**Keeping docs current:** If you notice anything inaccurate in these docs -- renamed -structs, traits, functions, modules, crates, APIs, stale data flows, wrong file paths -- -inform the user so they can be updated. After major changes, update this file, -`CLAUDE/architecture.md`, `ffi/CLAUDE.md`, and any relevant `/CLAUDE.md` files. +@AGENTS.md