Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
522 commits
Select commit Hold shift + click to select a range
90894d6
feat: added get_earliest_delta_file function (#2582)
thinh2 Jun 4, 2026
c67799f
feat!: add commit timestamp to the history_manager search result (#2663)
thinh2 Jun 4, 2026
a4ff559
test: add CRC support to TestTableBuilder via Snapshot::write_checksu…
DrakeLin Jun 5, 2026
a4871e2
feat!: Define and use CollectInto trait (#2699)
scovich Jun 5, 2026
3ca5f50
chore!: Use type constructors to avoid Box::new and DataType::from bo…
scovich Jun 5, 2026
ae7a2b8
ci: fixup title and body validator actions for merge/workflow (#2697)
nicklan Jun 5, 2026
0c0414a
feat: add basic plan IR behind declarative-plans feature (#2613)
OussamaSaoudi Jun 5, 2026
3638ba8
feat!: replace ScanBuilder stats setters with StatsOptions struct (#2…
DrakeLin Jun 5, 2026
b23ec52
feat: introduce FFI types for plan execution (#2680)
kyli87 Jun 5, 2026
0ee0f73
feat: add MeteredJsonHandler and MeteredParquetHandler (#2694)
DrakeLin Jun 5, 2026
aab11d7
refactor!: bridge Operation::QueryPlan onto the new IR (#2637)
OussamaSaoudi Jun 5, 2026
d09dabd
ci: Fix dupliate CI runs (#2704)
scottsand-db Jun 5, 2026
d6fab83
refactor!: extract default engine into separate crate (#2397)
DrakeLin Jun 6, 2026
721faee
fix: set stats.tightBound to false for DV updates (#2681)
dengsh12 Jun 6, 2026
bba26ff
ci: various improvements (#2708)
scottsand-db Jun 7, 2026
3f5b1bb
ci: bump codecov-action to v7.0.0 to fix coverage uploads (#2711)
scottsand-db Jun 8, 2026
e22f470
feat: use incremental CRC replay in Snapshot construction (#2678)
scottsand-db Jun 8, 2026
c7e39b8
test: parsed-stats integration test covering nested structs via table…
rajayush143 Jun 8, 2026
e3cc537
feat: add FFI-backed PlanExecutor impl (#2685)
kyli87 Jun 8, 2026
cffe836
feat!: add failure metric events (#2702)
scottsand-db Jun 8, 2026
d177f27
feat: support incremental CRC reconstruction on snapshot `builder_fro…
scottsand-db Jun 9, 2026
2ba2ffc
ci: fix flaky metrics deadlock test (#2716)
scottsand-db Jun 9, 2026
1c5cb1d
feat: support local DAT corpus override for acceptance workloads (#2705)
rliao147 Jun 9, 2026
0c9bb9b
feat: add proto wire format for declarative-plans IR (#2629)
OussamaSaoudi Jun 9, 2026
6a84c01
refactor: update benchmark output format, update tables (#2703)
Jameson-Crate Jun 10, 2026
07beb9e
feat!: history_manager: add get_earliest_recreatable version method (…
thinh2 Jun 10, 2026
b436f4a
feat: add a public history_manager::get_earliest_commit (#2713)
thinh2 Jun 10, 2026
d0a9072
feat!: add `Transaction::commit` metric events (#2719)
scottsand-db Jun 10, 2026
61d1626
refactor: unify column mapping validations to `MakePhysical` (#2599)
dengsh12 Jun 10, 2026
3ae4eaa
feat!: Introduce struct patch builder (#2665)
scovich Jun 10, 2026
1efb814
refactor!: Project node takes a single expression instead of a list (…
OussamaSaoudi Jun 10, 2026
3d090fd
fix: specify selection-vector contracts at the engine boundary (#2728)
scottsand-db Jun 11, 2026
682eb14
feat!: Introduce schema struct patching (#2672)
scovich Jun 11, 2026
8a1830b
refactor!: Checkpoint writer eagerly creates output schema (#2726)
scovich Jun 11, 2026
5c2951c
test: addressed failing invalid-handle-code (#2706)
rajayush143 Jun 11, 2026
322ecb6
fix: restore CRC-rooted P&M replay removed by #2678 (#2741)
scottsand-db Jun 12, 2026
f2641a2
chore: make delta_kernel_default_engine publishable (#2561)
DrakeLin Jun 12, 2026
e852422
chore(deps): bump rustls-webpki from 0.103.10 to 0.103.13 (#2735)
dependabot[bot] Jun 12, 2026
b38f7bd
refactor: Eagerly materialize the physical no-partition-schema (#2743)
scovich Jun 12, 2026
d185750
refactor!: Add and use Schema::field_at and friends (#2744)
scovich Jun 12, 2026
909841e
refactor!: improve plan execution FFI and error handling (#2729)
kyli87 Jun 12, 2026
e0a18bb
feat!: add recreatable-commit resolution to history manager (#2714)
thinh2 Jun 12, 2026
8755228
feat!: add `table_type` to snapshot-load and scan metric events (#2721)
scottsand-db Jun 13, 2026
4666763
feat!: Define and use a new projection struct patch (#2730)
scovich Jun 13, 2026
4563fa1
ci: run body validator on synchronize and reopened (#2748)
scottsand-db Jun 13, 2026
e1a163d
perf: terminate log listing before paging through staged commits (#2738)
DrakeLin Jun 13, 2026
055781a
feat!: add caller-supplied correlation id to operation metric events …
sanujbasu Jun 16, 2026
e40cd2e
ci: fail bench job on 15% regressions and tighten the bench comment (…
scottsand-db Jun 16, 2026
c09a451
feat: automatically insert partition columns when needed (#2715)
dengsh12 Jun 16, 2026
0883c3d
refactor: extract workloads crate (#2761)
Jameson-Crate Jun 16, 2026
7a6d45c
test: non-blocking invalid-handle-code in CI (#2745)
rajayush143 Jun 16, 2026
729391f
feat: add internal kernel sql parser for literals (#2670)
Jameson-Crate Jun 16, 2026
50a2b87
fix!: strum-ify metric label enums (and fix missing dv metric) (#2732)
scottsand-db Jun 16, 2026
f322e30
test: inject sparse nulls into table-builder data generation (#2657)
DrakeLin Jun 17, 2026
d766f60
docs: add issue for heap size estimation optimize (#2758)
dengsh12 Jun 17, 2026
22f5141
feat: add CommitRange for iterating Delta log actions across a commit…
thinh2 Jun 17, 2026
ee03090
fix: update benchmark ci to checkout merge head (#2771)
Jameson-Crate Jun 17, 2026
4706489
chore: default plan executor errors instead of panics (#2787)
scovich Jun 17, 2026
ebef942
refactor!: rename plan node NodeKind as Operator (#2789)
scovich Jun 17, 2026
589e95e
feat: add FFI function for get_earliest_delta (#2763)
thinh2 Jun 18, 2026
8d96fe5
perf: reuse loaded parquet footer instead of re-fetching (#2783)
scottsand-db Jun 18, 2026
38d10d9
feat(ffi): opaque-predicate file pruning via Arrow (#2627)
chiinlquah Jun 18, 2026
bafa8db
fix: timestamp offset fix (#2760)
Jameson-Crate Jun 18, 2026
5993863
feat: ffi metrics reporting + refactor for better reloading (#2746)
nicklan Jun 18, 2026
6544819
feat: support timestamp-to-version conversion for staged commits (#2784)
rliao147 Jun 22, 2026
05dbe81
fix: update workload download to use local ca (#2766)
Jameson-Crate Jun 22, 2026
cfedb38
perf: parse MapToStruct partition values with arrow parsers (#2750)
DrakeLin Jun 22, 2026
b7b23e6
test: CREATE TABLE tests to validate column mapping preservation (#2501)
sanujbasu Jun 23, 2026
79a77a2
feat: FFI implementation for latest_version_as_of, first_version_afte…
thinh2 Jun 23, 2026
5d99205
test: add ALTER TABLE tests validating existing column mapping annota…
sanujbasu Jun 23, 2026
44c2d4b
feat: add schema creation helper for column defaults (#2686)
Jameson-Crate Jun 23, 2026
723c9e3
chore: release 0.25.0 (#2788)
dengsh12 Jun 23, 2026
18828b9
feat!: Add proper declarative plan agg node (#2790)
scovich Jun 23, 2026
fbbf055
fix: update use statements in default engine doc strings (#2801)
adamreeve Jun 23, 2026
d86fc98
fix: update snapshot builder info to not include fields (#2807)
Jameson-Crate Jun 23, 2026
11ae3c1
test: add VersionTarget::AtTimestamp and IncrementalFrom to TestTable…
DrakeLin Jun 24, 2026
9cd65d9
test: assert row_id is unique for row tracking enabled feature sets (…
aaronlinear Jun 24, 2026
39e6415
feat: accept millisecond-precision timestamps in arrow-to-kernel conv…
sanujbasu Jun 24, 2026
9f9395d
feat!: conversion utilities for plan IR to proto bytes (#2802)
kyli87 Jun 25, 2026
92369de
feat: expose the FFI function for CommitRange builder (#2775)
thinh2 Jun 25, 2026
4ab973a
feat!: read interval types in core kernel (#2768)
rajayush143 Jun 25, 2026
e9c1a8a
feat: add java_package option to declarative-plan protos (#2817)
OussamaSaoudi Jun 25, 2026
dd80cde
feat: add column defaults struct (#2797)
Jameson-Crate Jun 25, 2026
450eca9
test: extend TableConfig with numIndexedCols and dataSkippingStatsCol…
DrakeLin Jun 25, 2026
d7b0cd5
feat: allow maintenance properties during CREATE TABLE (#2816)
chiinlquah Jun 26, 2026
4119daf
refactor!: identify plan nodes by index, drop RefId and output (#2829)
OussamaSaoudi Jun 26, 2026
681bbef
test: cover all-null file pruning across operators and log-replay sou…
sanujbasu Jun 27, 2026
a0ba980
feat: add without_row_transforms scan opt-out (#2756)
DrakeLin Jun 27, 2026
e40de65
feat: emit partitionValues_parsed via PartitionValuesOptions in scan …
DrakeLin Jun 27, 2026
9c3b36d
docs: minor test comment update (#2838)
sanujbasu Jun 29, 2026
90382e1
feat: schema! macro for easier schema creation (#2795)
scovich Jun 29, 2026
8007375
feat: add with_correlation_id to create-table and alter-table builder…
sanujbasu Jun 30, 2026
3182fde
feat: add schema and transaction methods for column defaults (#2808)
Jameson-Crate Jun 30, 2026
675f668
feat!: expose checkpoint writes via checkpoint_snapshot + FfiCheckpoi…
kim-ale Jun 30, 2026
89a3adb
chore: upgrade to strum-28 (#2849)
scovich Jun 30, 2026
c976c26
refactor: extract shared table-feature promotion helpers (#2844)
rliao147 Jun 30, 2026
ba8f013
test: verify all primitive values in parsed_stats test (#2835)
rajayush143 Jun 30, 2026
5f5f8f2
fix: accept LargeBinary and BinaryView for variant fields (#2737)
DrakeLin Jun 30, 2026
54007ac
feat!: carry scan sub-step latencies as nanoseconds (#2850)
scottsand-db Jul 1, 2026
f5aff96
test: ensuring dv works with row tracking feature (#2806)
aaronlinear Jul 1, 2026
88483fd
chore!: Add arrow-59, drop arrow-57 (#2847)
scovich Jul 1, 2026
0af4c38
feat!: Introduce fluent PlanBuilder to easily construct Plans (#2772)
OussamaSaoudi Jul 2, 2026
cdc45df
feat: add commit range iterator FFI bindings (#2776)
thinh2 Jul 2, 2026
acb98c9
fix: pair struct fields by name instead of by position (#2822)
DrakeLin Jul 2, 2026
ad88549
feat!: make default engine read I/O concurrency configurable (#2813)
chiinlquah Jul 3, 2026
1c23490
fix: parse empty-string partition values with spark cast semantics (#…
DrakeLin Jul 3, 2026
0e26fcd
feat!: added delete functionality to StorageHandler (#2823)
aaronlinear Jul 3, 2026
e07c6ee
feat!: Simplify agg alias handling (#2868)
scovich Jul 3, 2026
32a6f0d
chore!: Define XXX_FIELD to go with XXX_NAME (#2870)
scovich Jul 3, 2026
4a15dba
feat: expose create-table data layout/partitioned write context via F…
chiinlquah Jul 6, 2026
cd4e712
feat!: embed proto serialized schema inside of CParquetFooter (#2865)
kyli87 Jul 6, 2026
714905a
refactor: Define and use FoldWithOption extension trait (#2878)
scovich Jul 6, 2026
ecbf634
feat: parse the full v2Checkpoint object from _last_checkpoint (#2777)
OussamaSaoudi Jul 6, 2026
f995e3f
feat: support more cases for `snap.write_checksum` (#2853)
scottsand-db Jul 8, 2026
8bf28e1
docs: add incremental scan user-guide page (#2863)
DrakeLin Jul 8, 2026
20c4d1a
fix: tolerate stale column mapping annotations when disabled (#2886)
sanujbasu Jul 9, 2026
a2188d2
fix: rename IntoArray::into_array to avoid std name collision on Rust…
rliao147 Jul 9, 2026
2e1f36e
feat!: Add reader/writer features for AMT (#2871)
emkornfield Jul 9, 2026
e3603f2
feat: add checkpoint-shape resolution via PlanExecutor (#2779)
OussamaSaoudi Jul 10, 2026
7e7f8ac
feat!: Improve the column_name! macro and friends (#2891)
scovich Jul 10, 2026
e1739b4
feat: add generic REST/HTTP ObjectStore for the default engine (#2661)
chiinlquah Jul 13, 2026
fc3f2de
feat: add SQL predicate tokenizer for CHECK constraints (#2882)
wangm22 Jul 13, 2026
84e2c4a
docs: recommend TestTableBuilder for integration test setup (#2873)
DrakeLin Jul 13, 2026
ad86b1a
feat: expose if a snapshot is built as latest from intent level (#2905)
dengsh12 Jul 13, 2026
dd68e9f
feat!: support iceberg compat v3 with column defaults (#2846)
Jameson-Crate Jul 13, 2026
b14b4b8
chore: release v0.26.0 (#2921)
thinh2 Jul 14, 2026
0833777
feat: add predicate pushdown to incremental scan (#2860)
DrakeLin Jul 14, 2026
a460e58
feat: Add method to calculate base stats field IDs (#2923)
emkornfield Jul 14, 2026
07b2778
feat: strip newly-introduced column mapping metadata on none-mode wri…
sanujbasu Jul 15, 2026
3af0b08
feat!: support a fallback on PlanBasedEngine (#2924)
kyli87 Jul 15, 2026
ba3f9de
feat: add UC Delta-Tables API wire models and UpdateTableClient trait…
rliao147 Jul 15, 2026
b03af65
feat: send User-Agent header on UC Delta-Tables API requests (#2909)
rliao147 Jul 15, 2026
42717c1
feat: add UC OSS-server live integration test CI for get_config and l…
rliao147 Jul 16, 2026
2bddf72
feat!: read interval types in default engine (#2769)
rajayush143 Jul 16, 2026
0d1bebf
feat!: register intervalType reader-writer table feature (#2880)
rajayush143 Jul 16, 2026
e0faf1b
feat: expose several internal apis for connectors to do log cleanup (…
dengsh12 Jul 16, 2026
a614cfb
refactor: extract log_tail_from_commits helper in UC kernel client (#…
rliao147 Jul 17, 2026
a4f8e26
feat: add UC Delta-Tables credential-vending wire types and client me…
rliao147 Jul 17, 2026
beab39d
fix!: change PlanBasedEngine FFI to not free fallback engine (#2943)
kyli87 Jul 17, 2026
c591294
feat: add single-comparison parser for CHECK constraint predicates (#…
wangm22 Jul 19, 2026
29a43b1
feat: expose incremental scan via FFI (#2864)
DrakeLin Jul 20, 2026
f88ecc5
feat!: column defaults flag removal (#2922)
Jameson-Crate Jul 20, 2026
67e718a
feat!: re-introduce CRC caching (#2908)
scottsand-db Jul 20, 2026
c6cc57d
feat!: emit P&MLoad metric on every snapshot-load with a source label…
scottsand-db Jul 20, 2026
7a8eb0c
feat!: added write support with nullCount stats (#2845)
rajayush143 Jul 20, 2026
bae8ed0
fix: clarify schema JSON recursion limit errors (#2940)
dengsh12 Jul 20, 2026
3ba5d43
feat: add benchmark registry (#2918)
Jameson-Crate Jul 20, 2026
5700575
fix: bump parquet to 58.1 and preserve exact zero null counts (#2952)
DrakeLin Jul 20, 2026
c6328eb
feat: add UC Delta-Tables create-table wire types and staging E2E tes…
rliao147 Jul 20, 2026
b677b14
feat: Declarative P&M query (#2874)
scovich Jul 21, 2026
be9a9b6
refactor: rename NullGuarded to Checkpoint data skipping creator (#2960)
DrakeLin Jul 21, 2026
d794da6
refactor: extract StatColumns and emit prefixes from the checkpoint c…
DrakeLin Jul 22, 2026
f3384a6
fix: stop tracing events from warning on metrics (#2799)
Jameson-Crate Jul 22, 2026
9f15138
feat!: add Cast expression node (#2962)
DrakeLin Jul 22, 2026
0cef488
feat!: add geometry and geography schema types (#2930)
thinh2 Jul 22, 2026
72d97de
feat: Add CDF file listing implementation for read time CDF (#2803)
Omar-Saleh Jul 22, 2026
3aaf823
fix: source checkpoint partition columns from physical schema (#2976)
DrakeLin Jul 22, 2026
675ee48
fix!: interval table feature cleanup (#2959)
rajayush143 Jul 23, 2026
03e2537
refactor: expose treemap to bool functions from internal api (#2986)
hntd187 Jul 23, 2026
c15d4e6
feat: Enhance ToSchema macro with additional field tags (#2932)
emkornfield Jul 23, 2026
8547538
perf: skip non-add checkpoint row groups during scans (#2991)
DrakeLin Jul 24, 2026
24390ba
feat(ffi): expose with_stats with_partition_values scan builder APIs …
kyli87 Jul 24, 2026
9f411b4
feat: enable checkpoint row group skipping for partition columns (#2978)
DrakeLin Jul 24, 2026
bdd5b5b
feat: Add base Structs for modelling AMT (#2933)
emkornfield Jul 24, 2026
cdbbd05
feat(ffi): expose snapshot file stats from CRC (#2977)
chiinlquah Jul 24, 2026
64a36ca
feat!: add cooperative cancellation to scan_metadata (#2937)
sanujbasu Jul 24, 2026
770b259
feat: use stale CRC in DomainMetadata queries (#2961)
scottsand-db Jul 24, 2026
65288d4
fix: make generic errors more specific in snapshot creation (#2987)
Jameson-Crate Jul 24, 2026
7fd9a24
feat: Add checkpoint structure/schema for adaptive metadata tree. (#2…
emkornfield Jul 24, 2026
3a0cbb4
feat: validation for required addFile fields at commit time (#2856)
dengsh12 Jul 24, 2026
7f581f2
feat: add retry when data skipping before dedup fails (#2944)
dengsh12 Jul 24, 2026
75d66b3
fix!: reclassify MalformedJson error during schema parsing to Schema …
kyli87 Jul 24, 2026
28b245f
feat(ffi): expose snapshot_publish_with_committer for catalog-managed…
chiinlquah Jul 25, 2026
2403501
feat: enable reading tables with geospatial columns (#2931)
thinh2 Jul 25, 2026
0093ca8
feat!: label snapshot-load events as fresh vs incremental (#2916)
scottsand-db Jul 27, 2026
4aa6203
fix: normalize decimal scalar parsing failures to ParseError (#2954)
SAY-5 Jul 27, 2026
f5dcef0
perf: omit raw checkpoint stats when structured stats are compatible …
DrakeLin Jul 27, 2026
619bc52
fix: accept `timestampWithoutTimezone` feature alias for legacy table…
BoazC-MSFT Jul 27, 2026
4f0a13a
feat!: scalars and partition values (#2854)
rajayush143 Jul 27, 2026
e46f68f
feat!: add interval support to FFI visitors (#2903)
rajayush143 Jul 27, 2026
81355d4
feat!: push checkpoint partition casts to parquet handlers (#2983)
DrakeLin Jul 28, 2026
c85608d
fix: include offending feature in InvalidProtocol feature-validation …
sanujbasu Jul 28, 2026
9844985
feat: add declarative metadata scan plan (#2887)
scovich Jul 28, 2026
adf73f1
feat: expose parquet stats collection (#3022)
DrakeLin Jul 29, 2026
1288af2
docs: document plan expression semantics for executor authors (#3017)
scottsand-db Jul 29, 2026
5b75854
fix: three plan/executor improvements (#3029)
scottsand-db Jul 29, 2026
776d4ed
feat!: migrate UC integration to Delta Tables LoadTable and UpdateTab…
rliao147 Jul 30, 2026
3adc58d
feat(ffi): Expose declarative metadata scan plan (#3023)
kyli87 Jul 30, 2026
6563c71
feat!: create Unity Catalog managed tables via the Delta Tables API (…
rliao147 Jul 30, 2026
3b3d688
fix: correctly generate `extendedFileMetadata` for removeFile (#3006)
dengsh12 Jul 30, 2026
ca8b206
refactor: consolidate test engine wrappers behind DelegatingEngine (#…
scottsand-db Jul 30, 2026
bfa69d0
ci: split no-declarative-plans tests into a parallel job (#3038)
scottsand-db Jul 30, 2026
469b22e
feat: SetTxn queries: fix retention bug, root in CRC (#3001)
scottsand-db Jul 31, 2026
8ed72c9
chore: scaffold datafusion-executor crate (#2928)
aaronlinear Jul 31, 2026
8f0c3c5
feat(ffi): track peak native memory via optional tracking allocator (…
chiinlquah Jul 31, 2026
d8dfa2f
feat: add DataFusionExecutor struct (#2929)
aaronlinear Jul 31, 2026
bdcb019
feat: kernel to datafusion scalar conversion (#2957)
aaronlinear Aug 1, 2026
322f09d
feat!: validate add file partition columns (#2939)
dengsh12 Aug 1, 2026
ab38344
feat: add UC Delta-Tables reportMetrics wire types and client method …
rliao147 Aug 1, 2026
fb30159
feat: add FileStatsAccumulator to merge per-row-group stats (#3010)
arnav-databricks Aug 1, 2026
73dfcde
feat!: respect scan metadata output options in declarative plans (#3015)
OussamaSaoudi Aug 3, 2026
90422f0
refactor: extract unit test utils from `kernel/src/utils.rs` (#3018)
dengsh12 Aug 3, 2026
a178327
feat!: add column defaults ack method (#3036)
Jameson-Crate Aug 3, 2026
24aec0a
feat: expression conversion (#2958)
aaronlinear Aug 4, 2026
5f32a2a
fix: dv update respect shorten selection vector (#3051)
dengsh12 Aug 4, 2026
8177af9
feat!: tag UC client User Agent and rename unity-catalog-delta-rest-c…
rliao147 Aug 4, 2026
bf11f2f
feat: add create_staging_table and create_table to UCClient (#3034)
rliao147 Aug 4, 2026
dc3677a
feat!(ffi): expose clustering column infos read path (#2573)
chiinlquah Aug 4, 2026
eca870c
fix!: deterministically select among complete checkpoints at one vers…
sanujbasu Aug 5, 2026
063c1fd
fix!: remove `interval-type-in-dev` feature flag (#3041)
rajayush143 Aug 5, 2026
51ce398
fix: truncate timestamp stats to milliseconds in JSON encoding (#3073)
rliao147 Aug 7, 2026
efcee2b
docs: add crate metadata and READMEs for the UC crates (#3061)
rliao147 Aug 7, 2026
ebff0dc
chore: version the UC crates independently at 0.1.0 (#3062)
rliao147 Aug 7, 2026
47da584
feat: Utilities for converting between AMT and Delta DV structs (#3021)
emkornfield Aug 7, 2026
014f341
feat: Define and use Predicate TRUE/FALSE/NULL constants (#3065)
scovich Aug 7, 2026
d1f7f52
feat: make the PlanBasedEngine fallback optional (#3069)
chiinlquah Aug 8, 2026
d0215b7
fix: dv resolution when multiple batches are produced (#3077)
hntd187 Aug 10, 2026
de609db
feat!: gate vendored protoc behind its own feature (#3054)
scottsand-db Aug 10, 2026
4dfd13a
docs: explain why timestamp stats are floored, not rounded or ceiled …
rliao147 Aug 10, 2026
fb8e997
feat!: rename Load to DynamicScan and require file metadata (#3024)
OussamaSaoudi Aug 10, 2026
5fcc122
perf: speed up the Miri CI job and widen its coverage (#3035)
scottsand-db Aug 10, 2026
feab5df
refactor: Move plan construction to `Scan` (#3039)
OussamaSaoudi Aug 10, 2026
5cde0ce
fix: block dataChange removeFile for appendOnly tables (#3042)
dengsh12 Aug 10, 2026
64fe8e9
feat!: preserve typed schema-field metadata across FFI (#3086)
chiinlquah Aug 11, 2026
c3e8f42
feat!: Refine agg definitions and sync engine impl (#3081)
scovich Aug 11, 2026
44a454d
docs: explain why kernel fails on a dangling _last_checkpoint hint (#…
thinh2 Aug 11, 2026
3d9fbf1
refactor!: Make the col! macro a preferred alias of column_expr! (#3071)
scovich Aug 11, 2026
dc5ec4e
feat!: 3-arg Min/MaxNotNullBy (#3082)
scovich Aug 11, 2026
e5f227b
chore: Use col! instead of column_expr! (#3074)
scovich Aug 11, 2026
1122a73
feat!: Define and implement SUM/COUNT aggregates (#3083)
scovich Aug 11, 2026
c4c0307
fix: remove version from kernel in datafusion executor (#3108)
aaronlinear Aug 13, 2026
3e7b2c9
feat: initial predicate conversion (#2971)
aaronlinear Aug 13, 2026
41ee7dc
fix: logical comparison for NULL instead of physical comparison (#3085)
aaronlinear Aug 13, 2026
acc0b93
fix: read tables with orphaned columnMapping reader feature (#3097)
sanujbasu Aug 13, 2026
dad0a1e
feat: typed Struct and StructPatch expression conversion (#2972)
aaronlinear Aug 13, 2026
5c1369b
test: strengthen scan metadata struct columns coverage (#3053)
DrakeLin Aug 13, 2026
bade178
feat: convert MapToStruct expressions to DataFusion (#2996)
aaronlinear Aug 13, 2026
cd2ca4e
feat: lower ParseJson to a kernel-delegating scalar UDF (#3056)
aaronlinear Aug 13, 2026
0950f13
feat: validate required fields for dv update (#3026)
dengsh12 Aug 13, 2026
b8f31c3
fix: supurious delta acceptance failure due to network conn issues (#…
jtuglu1 Aug 13, 2026
83c52e2
feat!: Derive macro for converting POD structs to Scalar::Struct (#3095)
scovich Aug 13, 2026
a8b6e8c
chore: release v0.27.0 (#3107)
chiinlquah Aug 13, 2026
96bf8d8
feat!: Add derive macro for populating a struct from a Scalar (#3101)
scovich Aug 13, 2026
0aefffb
chore: Use col and col_name macros everywhere they make sense (#3121)
scovich Aug 13, 2026
7f4b02c
fix: import CollectInto from crate::utils to fix default-feature buil…
chiinlquah Aug 14, 2026
728aeb9
chore: release v0.27.1 (#3125)
chiinlquah Aug 14, 2026
3529fd1
feat!: add reader timezone to map-to-struct
DrakeLin Aug 14, 2026
a512f72
feat: honor reader timezone in partition values
DrakeLin Aug 14, 2026
3400443
feat!: add element-at expression
DrakeLin Aug 14, 2026
eab4ed5
feat!: add reader timezone to cast expressions
DrakeLin Aug 14, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
69 changes: 69 additions & 0 deletions .github/CLAUDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# .github/CLAUDE.md

Workflow-authoring rules for GitHub Actions.

## Triggers: restrict `push` to long-lived branches

The CI trigger events:

- `pull_request`: a commit is pushed to a branch with an open PR. Runs CI on the
PR.
- `merge_group`: the merge queue builds a temporary `gh-readonly-queue/*` branch
and tests the change as it would land on `main`.
- `push`: a branch ref is updated. We only want this for the long-lived branches
(`main` and `release/**`), i.e. the post-merge run once a PR lands.

We want CI on `pull_request`, on `merge_group`, and on `push` to a long-lived
branch: one run per stage. A bare `push` trigger instead duplicates two of these:

- `push` + `pull_request`: a branch pushed directly to this repo (a maintainer
branch, not a fork) fires both. Fork PRs avoid this because the `push` lands on
the fork, not on this repo.
- `push` + `merge_group`: the merge queue's push to the `gh-readonly-queue/*`
branch fires both.

Restricting `push` to long-lived branches drops the redundant `push` in each
case:

```yaml
on:
push:
branches: [main, "release/**"]
pull_request:
merge_group:
```

## Never `paths`/`paths-ignore` a workflow whose jobs are required checks

Do NOT add `paths` or `paths-ignore` filters to a workflow whose jobs are required
status checks (most of `build.yml`). When a path filter skips the whole workflow,
the required checks never report, and the PR stays blocked on "Expected, waiting
for status" indefinitely. A skipped workflow is not the same as a passing one.

## Supply chain security: `--locked`

Every `cargo` command in CI that resolves dependencies MUST use `--locked` to
enforce the committed `Cargo.lock`. This prevents CI from silently picking up a
newer (potentially compromised) transitive dependency. If `Cargo.lock` is out of
sync with `Cargo.toml`, the build fails immediately, forcing dependency changes to
be explicit and reviewable. See the top-level comment in `build.yml` for full
rationale. Commands exempt from `--locked`: `cargo +nightly fmt` (no dep
resolution), `cargo msrv verify/show` (wrapper tool), `cargo miri setup` (tooling
setup).

## Action safety

When writing any github action, consider safety including mitigating common attack
vectors such as expression injection and pull request target attacks.

```yaml
# The code below is vulnerable to expression injection
run: |
echo "Comment: ${{ github.event.comment.body }}"

# To mitigate instead use environment variables
env:
COMMENT_BODY: ${{ github.event.comment.body }}
run: |
echo "Comment: $COMMENT_BODY"
```
61 changes: 61 additions & 0 deletions .github/SECURITY_MODEL.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Benchmark workflow security model

Scope: [`benchmark.yml`](workflows/benchmark.yml) (runs bench on the PR merge
commit -- PR head merged into base) and
[`benchmark-post-comment.yml`](workflows/benchmark-post-comment.yml) (posts the
result comment in base-branch context). Other workflows in this repo may follow
different patterns -- this document is specific to the benchmark pair.

## Running untrusted PR code

The `run-benchmark` job in `benchmark.yml` checks out the PR merge commit (PR
head merged into base) and runs its build + bench harness. To bound what that
code can do:

- Workflow-level `permissions: {}` is the default; each job opts in to the
minimum scope it needs. `run-benchmark` declares only `contents: read`,
so any leaked GITHUB_TOKEN has no write power. Fork PRs additionally get
a read-only token from GitHub by default under the `pull_request` event.
- `actions/checkout` uses `persist-credentials: false` so the token isn't
left in the local git config after checkout.
- `cargo install critcmp` runs *before* the PR merge-commit checkout, so the
PR's `.cargo/config.toml` cannot redirect the registry for that install.

Do not add other secrets or grant write permissions to `run-benchmark`.

## Rust-cache poisoning

`Swatinem/rust-cache` is configured with
`save-if: github.event_name == 'push' && github.ref == 'refs/heads/main'`.
PR runs read-restore from main's cache; they never save. The
`github.event_name == 'push'` clause is the load-bearing half -- the
benchmark workflow has no `push` trigger outside of `push: main` for the
cache-warming job, so PR runs cannot match this condition and therefore
cannot write the cache.

Do not relax the `github.event_name == 'push'` clause. If you do, a PR-head
run can land compile artifacts in main's cache scope (poisoning subsequent
runs) or evict main's entries through the shared cache budget.

## Cross-PR comment forgery

`benchmark-post-comment.yml` runs in base-branch context with
`pull-requests: write`. It must derive the target PR number from a source
the fork cannot forge. The choice depends on which event triggered the
upstream bench workflow:

- **`pull_request` upstream:** use `gh pr view --repo "$REPO" "owner:branch"`,
where `owner:branch` is built from `workflow_run.head_repository.owner.login`
and `workflow_run.head_branch`. Both fields are populated by GitHub from the
trusted workflow_run event payload, not from anything the fork wrote.
- **`issue_comment` upstream:** trust the `bench-pr-number` artifact. For
`issue_comment` events GitHub serves `benchmark.yml` from the default
branch (not the PR head), so the "Stash PR number" + "Upload PR number"
steps run trusted code. The upload happens *before* `actions/checkout`
brings PR code onto the runner, so once the artifact is on GitHub's
storage it is immutable for the rest of the run -- the bench script
cannot alter it, and `contents: read` denies any API-level mutation.

`workflow_run` itself runs the post-comment workflow from the default
branch (a GitHub guarantee), so a fork PR cannot modify the resolver to
trust the wrong source.
101 changes: 101 additions & 0 deletions .github/actions/install-and-cache/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# This is copied from https://github.com/tecolicom/actions-install-and-cache
# which is Copyright 2022 Office TECOLI, LLC

name: install-and-cache generic backend
description: 'GitHub Action to run installer and cache the result'
branding:
color: orange
icon: type

inputs:
run: { required: true, type: string }
path: { required: true, type: string }
cache: { required: false, type: string, default: yes }
key: { required: false, type: string }
sudo: { required: false, type: string }
verbose: { required: false, type: string, default: false }

outputs:
cache-hit:
value: ${{ steps.cache.outputs.cache-hit }}

runs:
using: composite
steps:

- id: setup
shell: bash
run: |
: setup install-and-cache
define() { IFS='\n' read -r -d '' ${1} || true ; }
define script <<'EOS_cad8_c24e_'
${{ inputs.run }}
EOS_cad8_c24e_
directory="${{ inputs.path }}"
given_key="${{ inputs.key }}"
archive= key=
case "${{ inputs.cache }}" in
yes|workflow)
cache="${{ inputs.cache }}"
uname -mrs
hash=$( (uname -mrs ; cat <<< "$script" ; echo $directory) | \
(md5sum||md5) | awk '{print $1}' )
key="${hash}${given_key:+-$given_key}"
[ "$cache" == 'workflow' ] && \
key+="-${{ github.run_id }}-${{ github.run_attempt }}"
archive=$HOME/archive-$hash.tz
;;
*)
cache=no
;;
esac
# use "--recursive-unlink" option if GNU tar is found
if tar --version | grep GNU > /dev/null
then
tar="tar --recursive-unlink"
elif gtar --version | grep GNU > /dev/null
then
tar="gtar --recursive-unlink"
else
tar=tar
fi
sed 's/^ *//' << END >> $GITHUB_OUTPUT
cache=$cache
archive=$archive
key=$key
tar=$tar
END

- id: cache
if: steps.setup.outputs.cache != 'no'
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: ${{ steps.setup.outputs.archive }}
key: ${{ steps.setup.outputs.key }}

- id: extract
if: steps.setup.outputs.cache != 'no' && steps.cache.outputs.cache-hit == 'true'
shell: bash
run: |
: extract
archive="${{ steps.setup.outputs.archive }}"
verbose="${{ inputs.verbose }}"
tar="${{ steps.setup.outputs.tar }}"
ls -l $archive
if [ -s $archive ]
then
opt=-Pxz
[[ $verbose == yes || $verbose == true ]] && opt+=v
sudo $tar -C / $opt -f $archive
else
echo "$archive is empty"
fi

- id: install-and-archive
if: steps.cache.outputs.cache-hit != 'true'
uses: tecolicom/actions-install-and-archive@9d5afb27f9900f2df47fe40de58fbd837032bddf # v1.3
with:
run: ${{ inputs.run }}
archive: ${{ steps.setup.outputs.archive }}
path: ${{ inputs.path }}
sudo: ${{ inputs.sudo }}
47 changes: 47 additions & 0 deletions .github/actions/use-homebrew-tools/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# This is copied from https://github.com/tecolicom/actions-use-homebrew-tools/
# which is Copyright 2022 Office TECOLI, LLC

name: install-and-cache homebrew tools
description: 'GitHub Action to install and cache homebrew tools'
branding:
color: orange
icon: type

inputs:
tools: { required: false, type: string }
key: { required: false, type: string }
path: { required: false, type: string }
cache: { required: false, type: string, default: yes }
verbose: { required: false, type: boolean, default: false }

outputs:
cache-hit:
value: ${{ steps.update.outputs.cache-hit }}

runs:
using: composite
steps:

- id: setup
shell: bash
run: |
: setup use-homebrew-tools
given_key="${{ inputs.key }}"
brew_version="$(brew --version)"
echo "$brew_version"
version_key="$( echo "$brew_version" | (md5sum||md5) | awk '{print $1}' )"
key="${given_key:+$given_key-}${version_key}"
sed 's/^ *//' << END >> $GITHUB_OUTPUT
command=brew install
prefix=$(brew --prefix)
key=$key
END

- id: update
uses: ./.github/actions/install-and-cache
with:
run: ${{ steps.setup.outputs.command }} ${{ inputs.tools }}
path: ${{ steps.setup.outputs.prefix }} ${{ inputs.path }}
key: ${{ steps.setup.outputs.key }}
cache: ${{ inputs.cache }}
verbose: ${{ inputs.verbose }}
18 changes: 18 additions & 0 deletions .github/scripts/seed_uc_table.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env bash
# Seed a catalog-managed Delta table in the running UC server via its `bin/uc` CLI,
# so the gated `load_table` live test has a table to read.
#
# Usage: seed_uc_table.sh <catalog> <schema> <table>
# Overridable via env: UC_DIR.
set -euo pipefail

UC_DIR="${UC_DIR:-$HOME/unitycatalog}"
catalog="${1:?catalog required}"
schema="${2:?schema required}"
table="${3:?table required}"

cd "$UC_DIR"
bin/uc table create \
--full_name "$catalog.$schema.$table" \
--columns "id INT, name STRING" \
--table_type MANAGED
27 changes: 27 additions & 0 deletions .github/scripts/setup_unitycatalog.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Clone and build the Unity Catalog server at a pinned commit so the gated
# unity-catalog-delta-client-default live integration tests can run against a real server in CI.
#
# Pinned to a commit that serves the Delta-Tables API the kernel UC crate targets. The published
# `:latest` artifacts/images predate these endpoints, so building from this commit is required.
# Bump UC_COMMIT when the server contract advances.
#
# Overridable via env: UC_REPO, UC_COMMIT, UC_DIR.
set -euo pipefail

UC_REPO="${UC_REPO:-https://github.com/unitycatalog/unitycatalog.git}"
UC_COMMIT="${UC_COMMIT:-0f1445227bd251b386420c90136515daefa4e03d}"
UC_DIR="${UC_DIR:-$HOME/unitycatalog}"

if [[ ! -d "$UC_DIR/.git" ]]; then
git init -q "$UC_DIR"
git -C "$UC_DIR" remote add origin "$UC_REPO"
fi

git -C "$UC_DIR" fetch --depth 1 origin "$UC_COMMIT"
git -C "$UC_DIR" checkout -q --force FETCH_HEAD

cd "$UC_DIR"
# Build the server jar up front so the launch step starts fast and a cache can capture it.
# `bin/start-uc-server` will reuse this jar instead of rebuilding when present.
build/sbt -info package
68 changes: 68 additions & 0 deletions .github/scripts/uc_server.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# Start or stop the Unity Catalog server built by setup_unitycatalog.sh, for the gated
# unity-catalog-delta-client-default live integration tests.
#
# Usage: uc_oss_server.sh {start|stop}
# Overridable via env: UC_DIR, UC_HOST, UC_PID_FILE, UC_LOG_FILE.
set -euo pipefail

UC_DIR="${UC_DIR:-$HOME/unitycatalog}"
UC_HOST="${UC_HOST:-http://localhost:8080}"
PID_FILE="${UC_PID_FILE:-$UC_DIR/uc-server.pid}"
LOG_FILE="${UC_LOG_FILE:-$UC_DIR/uc-server.log}"
# Used to poll the UC server to see when it is ready to accept requests.
HEALTH_URL="$UC_HOST/api/2.1/unity-catalog/catalogs"

start() {
cd "$UC_DIR"

# Set a storage root so the server can allocate locations for UC catalog-managed tables; a fresh
# server has none, so the staging-tables create endpoint 400s. Point it at a local dir for CI,
# kept outside UC_DIR so created tables don't end up in the build cache.
local managed_dir="${UC_MANAGED_DIR:-${TMPDIR:-/tmp}/uc-managed-tables}"
local props="$UC_DIR/etc/conf/server.properties"
mkdir -p "$managed_dir"
sed -i '/^storage-root\.tables=/d' "$props"
echo "storage-root.tables=file://$managed_dir" >>"$props"

echo "Starting Unity Catalog server (logs: $LOG_FILE)"
# Launch in a new session so stop() can kill the whole process group (the forked JVM child).
setsid bash -c 'echo $$ >"'"$PID_FILE"'"; exec bin/start-uc-server >"'"$LOG_FILE"'" 2>&1' &

# Wait up to ~5 minutes: covers an incremental sbt build on first boot plus JVM startup.
for i in $(seq 1 150); do
if curl -fsS "$HEALTH_URL" >/dev/null 2>&1; then
echo "Unity Catalog server is up (after $((i * 2))s)."
return 0
fi
if [[ -f "$PID_FILE" ]] && ! kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then
echo "Server process exited before becoming healthy. Logs:" >&2
cat "$LOG_FILE" >&2
return 1
fi
sleep 2
done

echo "Server did not become healthy within the timeout. Logs:" >&2
cat "$LOG_FILE" >&2
return 1
}

stop() {
if [[ -f "$PID_FILE" ]]; then
local pid
pid="$(cat "$PID_FILE")"
# Kill the whole process group (negative pid) to take down the forked JVM child.
kill -- -"$pid" 2>/dev/null || kill "$pid" 2>/dev/null || true
rm -f "$PID_FILE"
fi
}

case "${1:-}" in
start) start ;;
stop) stop ;;
*)
echo "usage: $0 {start|stop}" >&2
exit 2
;;
esac
Loading
Loading