Skip to content

Commit c815686

Browse files
committed
chore(deps): bump DataFusion to 55.0.0
Bump the workspace `datafusion`, `datafusion-proto`, `datafusion-spark`, and `datafusion-substrait` deps from 54.1.0 to 55.0.0, and `arrow` from 58 to 59 to match what DataFusion 55 pulls in. `object_store` stays on 0.13, which is still what DataFusion 55 depends on. Upstream changes that required code updates: - The `datafusion.proto` schema moved from `datafusion/proto/proto/` to `datafusion/proto-models/proto/`, so the `download-maven-plugin` URL and both `<sha512>` checksums are updated. - `ListingTableScanNode.target_partitions` was removed (now `reserved 9`); the partition count comes from the session config instead. - The concrete `Default*Cache` types were collapsed into a single generic `DefaultCache<K, V>`. Rather than name the new type, `cache_manager` now only sets the cache *limits* and leaves the slots `None`, which is what `CacheManager::try_new` already used to build the built-in caches on both 54 and 55 — so this is behavior-preserving. - `ExecutionPlan` gained a required `apply_expressions`; `JavaScanExec` is a leaf scan with no physical expressions, so it returns `Continue`. - `datafusion_substrait::serializer::deserialize_bytes` is now synchronous and takes `&[u8]`.
1 parent 0d2c72b commit c815686

9 files changed

Lines changed: 287 additions & 247 deletions

File tree

‎Cargo.lock‎

Lines changed: 245 additions & 214 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,15 @@ repository = "https://github.com/apache/datafusion-java"
3737
# and add per-crate flags (optional, features, default-features) at the use
3838
# site.
3939
[workspace.dependencies]
40-
arrow = { version = "58", features = ["ffi"] }
40+
arrow = { version = "59", features = ["ffi"] }
4141
async-trait = "0.1"
42-
datafusion = { version = "54.1.0" }
43-
datafusion-proto = "54.1.0"
44-
datafusion-spark = "54.1.0"
45-
datafusion-substrait = "54.1.0"
42+
datafusion = { version = "55.0.0" }
43+
datafusion-proto = "55.0.0"
44+
datafusion-spark = "55.0.0"
45+
datafusion-substrait = "55.0.0"
4646
futures = "0.3"
4747
jni = "0.21"
48-
# Pinned to the major DataFusion 54.1 pulls in transitively (0.13.x) so we
48+
# Pinned to the major DataFusion 55.0 pulls in transitively (0.13.x) so we
4949
# share the same `dyn ObjectStore` vtable and don't double-link.
5050
object_store = { version = "0.13", default-features = false }
5151
prost = "0.14"

‎core/pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -129,18 +129,18 @@ under the License.
129129
<url>https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-common/proto/datafusion_common.proto</url>
130130
<outputDirectory>${project.build.directory}/proto/datafusion/proto-common/proto</outputDirectory>
131131
<outputFileName>datafusion_common.proto</outputFileName>
132-
<sha512>e6a20115badec14f641b9237b0faa3f553162992444208322558ea3fcfaa24ca15334e4945827259dcb7c1de7f9aed659d496974eb8eeb1cfcf54ad4d7f6d20f</sha512>
132+
<sha512>0e6db862693eb687b5a77ca80dc0465d64a40d396cfaf39d864f6062b174c3818adbf68f04c4a7b03b7aad6aadcf319303dcd1c8c41929e36e00cea5e9e2dc08</sha512>
133133
</configuration>
134134
</execution>
135135
<execution>
136136
<id>fetch-datafusion-proto</id>
137137
<phase>generate-sources</phase>
138138
<goals><goal>wget</goal></goals>
139139
<configuration>
140-
<url>https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto/proto/datafusion.proto</url>
140+
<url>https://raw.githubusercontent.com/apache/datafusion/${datafusion.version}/datafusion/proto-models/proto/datafusion.proto</url>
141141
<outputDirectory>${project.build.directory}/proto/datafusion/proto/proto</outputDirectory>
142142
<outputFileName>datafusion.proto</outputFileName>
143-
<sha512>bdae5361d32b4d4f0418677ce650ee8cc212fddfe9390b2c033ee4ab3bb3c969a6148b7b318b9bb268ee9384f18d4254ebe6e0701f15639059a8f2800d6e7847</sha512>
143+
<sha512>05f98d5245ac46b81f9a897d16add727538697dce252796586020220c240679673ec6489242ad0d30240d911cd18caa867577baddd018495ac0e5a86086d3728</sha512>
144144
</configuration>
145145
</execution>
146146
</executions>

‎core/src/test/java/org/apache/datafusion/proto/SessionContextProtoTest.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,11 @@ void fromProtoListingScanMatchesSql() throws Exception {
151151
.build())
152152
.setParquet(
153153
DatafusionCommon.ParquetFormat.getDefaultInstance())
154-
.setTargetPartitions(1)
154+
// No target_partitions: DataFusion 55
155+
// removed the field from
156+
// ListingTableScanNode (now
157+
// `reserved 9`) and takes the partition
158+
// count from the session config.
155159
.build())
156160
.build())
157161
.addExpr(

‎docs/source/contributor-guide/updating-datafusion-version.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ $EDITOR pom.xml # set <datafusion.version>
4646
# <sha512> elements in pom.xml.
4747
NEW=$(grep -m1 -oE '<datafusion.version>[^<]+' pom.xml | cut -d'>' -f2)
4848
curl -sL "https://raw.githubusercontent.com/apache/datafusion/$NEW/datafusion/proto-common/proto/datafusion_common.proto" | shasum -a 512 | awk '{print $1}'
49-
curl -sL "https://raw.githubusercontent.com/apache/datafusion/$NEW/datafusion/proto/proto/datafusion.proto" | shasum -a 512 | awk '{print $1}'
49+
curl -sL "https://raw.githubusercontent.com/apache/datafusion/$NEW/datafusion/proto-models/proto/datafusion.proto" | shasum -a 512 | awk '{print $1}'
5050
$EDITOR pom.xml # paste the two hashes into the <sha512> elements
5151

5252
# Drop the local download cache so the next build re-downloads against

‎native/src/cache_manager.rs‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,18 @@
2020
//!
2121
//! Each of the three caches is independent; an unset proto field leaves the
2222
//! corresponding upstream default in place (no cache for list-files / stats,
23-
//! a `DefaultFilesMetadataCache` with the default limit for file metadata).
24-
//! When a setter *is* present, the JNI layer always installs a fresh
25-
//! `Default*Cache` impl -- the v1 contract is "configure the built-in
26-
//! caches", not "swap in a custom one".
23+
//! a default file-metadata cache with the default limit).
24+
//!
25+
//! We only ever set the *limits* (and the list-files TTL), never the cache
26+
//! slots themselves: `CacheManager::try_new` constructs the built-in
27+
//! `DefaultCache` for any slot left `None` whose limit is non-zero. That is
28+
//! the v1 contract -- "configure the built-in caches", not "swap in a custom
29+
//! one" -- and it keeps us off the concrete cache types, which DataFusion 55
30+
//! collapsed into a single generic `DefaultCache<K, V>`.
2731
28-
use std::sync::Arc;
2932
use std::time::Duration;
3033

3134
use datafusion::execution::cache::cache_manager::CacheManagerConfig;
32-
use datafusion::execution::cache::file_statistics_cache::DefaultFileStatisticsCache;
33-
use datafusion::execution::cache::{DefaultFilesMetadataCache, DefaultListFilesCache};
3435

3536
use crate::proto_gen::CacheManagerOptionsProto;
3637
use datafusion_jni_common::errors::JniResult;
@@ -56,9 +57,7 @@ pub(crate) fn build_config(
5657
let mut config = CacheManagerConfig::default();
5758

5859
if let Some(max_bytes) = opts.file_metadata_cache_max_bytes {
59-
let max = max_bytes as usize;
60-
config.file_metadata_cache = Some(Arc::new(DefaultFilesMetadataCache::new(max)));
61-
config.metadata_cache_limit = max;
60+
config.metadata_cache_limit = max_bytes as usize;
6261
}
6362

6463
if let Some(lfc) = &opts.list_files_cache {
@@ -69,23 +68,19 @@ pub(crate) fn build_config(
6968
let max = lfc.max_bytes.map(|v| v as usize).unwrap_or(default_limit);
7069
let ttl = lfc.ttl_millis.map(Duration::from_millis);
7170

72-
config.list_files_cache = Some(Arc::new(DefaultListFilesCache::new(max, ttl)));
7371
config.list_files_cache_limit = max;
7472
config.list_files_cache_ttl = ttl;
7573
}
7674

7775
// DataFusion 54 renamed `table_files_statistics_cache` to
7876
// `file_statistics_cache` and made the *limit* the on/off switch:
79-
// `CacheManager::try_new` now builds a `DefaultFileStatisticsCache`
80-
// whenever `file_statistics_cache_limit > 0`, even when the cache slot is
81-
// `None`. The default limit is non-zero, so an explicit `false` from the
82-
// Java surface has to zero the limit -- otherwise upstream would install a
77+
// `CacheManager::try_new` builds a default file-statistics cache whenever
78+
// `file_statistics_cache_limit > 0`, even when the cache slot is `None`.
79+
// The default limit is non-zero, so an explicit `false` from the Java
80+
// surface has to zero the limit -- otherwise upstream would install a
8381
// stats cache the caller just asked us not to.
8482
if let Some(enabled) = opts.file_statistics_cache_enabled {
85-
if enabled {
86-
config.file_statistics_cache = Some(Arc::new(DefaultFileStatisticsCache::default()));
87-
} else {
88-
config.file_statistics_cache = None;
83+
if !enabled {
8984
config.file_statistics_cache_limit = 0;
9085
}
9186
}

‎native/src/proto.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ pub extern "system" fn Java_org_apache_datafusion_SessionContext_createDataFrame
8080
let bytes: Vec<u8> = env.convert_byte_array(&plan_bytes)?;
8181

8282
let plan = runtime().block_on(async {
83-
let substrait_plan = datafusion_substrait::serializer::deserialize_bytes(bytes).await?;
83+
let substrait_plan = datafusion_substrait::serializer::deserialize_bytes(&bytes)?;
8484
datafusion_substrait::logical_plan::consumer::from_substrait_plan(
8585
&ctx.state(),
8686
&substrait_plan,

‎native/src/table_provider.rs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,12 @@ use datafusion::arrow::datatypes::SchemaRef;
3030
use datafusion::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
3131
use datafusion::arrow::record_batch::{RecordBatch, RecordBatchReader};
3232
use datafusion::catalog::Session;
33+
use datafusion::common::tree_node::TreeNodeRecursion;
3334
use datafusion::datasource::{TableProvider, TableType};
3435
use datafusion::error::{DataFusionError, Result};
3536
use datafusion::execution::context::TaskContext;
3637
use datafusion::logical_expr::Expr;
37-
use datafusion::physical_expr::EquivalenceProperties;
38+
use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr};
3839
use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType};
3940
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
4041
use datafusion::physical_plan::{
@@ -155,6 +156,15 @@ impl ExecutionPlan for JavaScanExec {
155156
vec![]
156157
}
157158

159+
/// A leaf scan: the Java side owns the row production, so there are no
160+
/// `PhysicalExpr`s on this node for the visitor to descend into.
161+
fn apply_expressions(
162+
&self,
163+
_f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
164+
) -> Result<TreeNodeRecursion> {
165+
Ok(TreeNodeRecursion::Continue)
166+
}
167+
158168
fn with_new_children(
159169
self: Arc<Self>,
160170
_children: Vec<Arc<dyn ExecutionPlan>>,

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ under the License.
4040
<maven.compiler.target>17</maven.compiler.target>
4141
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
4242
<junit.version>5.11.3</junit.version>
43-
<datafusion.version>54.1.0</datafusion.version>
43+
<datafusion.version>55.0.0</datafusion.version>
4444
<protobuf.version>3.25.5</protobuf.version>
4545
<arrow.version>19.0.0</arrow.version>
4646
<avro.version>1.12.0</avro.version>

0 commit comments

Comments
 (0)