Skip to content

Commit 7cf5eac

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/substrait-from-bytes
# Conflicts: # core/src/main/java/org/apache/datafusion/SessionContext.java # native/Cargo.toml
2 parents d2b16d3 + ff969cb commit 7cf5eac

58 files changed

Lines changed: 5727 additions & 94 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ name: Build
1919

2020
on:
2121
push:
22-
branches: [main]
22+
branches: [main, 'branch-*']
2323
pull_request:
24-
branches: [main]
24+
branches: [main, 'branch-*']
2525

2626
permissions:
2727
contents: read

‎Makefile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,20 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
.PHONY: all native jvm test clean tpch-data
18+
.PHONY: all native native-runtime-metrics jvm test clean tpch-data
1919

2020
all: native jvm
2121

2222
native:
2323
cd native && cargo build
2424

25+
# Build the native crate with the `runtime-metrics` Cargo feature enabled.
26+
# Requires `--cfg tokio_unstable` because tokio-metrics gates its API there.
27+
# Default `make native` does not pull this in; callers who need
28+
# SessionContext.runtimeStats() pick this target explicitly.
29+
native-runtime-metrics:
30+
cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
31+
2532
jvm:
2633
./mvnw package -DskipTests
2734

‎core/pom.xml‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,20 @@ under the License.
2626
<parent>
2727
<groupId>org.apache.datafusion</groupId>
2828
<artifactId>datafusion-java-parent</artifactId>
29-
<version>0.1.0-SNAPSHOT</version>
29+
<version>0.2.0-SNAPSHOT</version>
3030
</parent>
3131

3232
<artifactId>datafusion-java</artifactId>
3333
<packaging>jar</packaging>
3434

3535
<name>Apache DataFusion Java</name>
3636

37+
<properties>
38+
<!-- Which cargo build profile (debug/release) the bundled native
39+
library is copied from. Override with -Ddatafusion.native.profile=release. -->
40+
<datafusion.native.profile>debug</datafusion.native.profile>
41+
</properties>
42+
3743
<dependencies>
3844
<dependency>
3945
<groupId>org.junit.jupiter</groupId>
@@ -82,9 +88,34 @@ under the License.
8288
<groupId>org.apache.maven.plugins</groupId>
8389
<artifactId>maven-surefire-plugin</artifactId>
8490
<configuration>
85-
<argLine>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug --add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
91+
<argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
8692
</configuration>
8793
</plugin>
94+
<plugin>
95+
<groupId>org.apache.maven.plugins</groupId>
96+
<artifactId>maven-antrun-plugin</artifactId>
97+
<executions>
98+
<execution>
99+
<id>copy-native-lib</id>
100+
<phase>process-classes</phase>
101+
<goals><goal>run</goal></goals>
102+
<configuration>
103+
<target>
104+
<property name="datafusion.native.lib.source"
105+
value="${maven.multiModuleProjectDirectory}/native/target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
106+
<fail message="Native library not found at ${datafusion.native.lib.source}. Run 'cd native &amp;&amp; cargo build' (or 'make') before building the JAR.">
107+
<condition><not><available file="${datafusion.native.lib.source}"/></not></condition>
108+
</fail>
109+
<mkdir dir="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}"/>
110+
<copy file="${datafusion.native.lib.source}"
111+
tofile="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}/${datafusion.lib.filename}"
112+
preservelastmodified="true"
113+
verbose="true"/>
114+
</target>
115+
</configuration>
116+
</execution>
117+
</executions>
118+
</plugin>
88119
<plugin>
89120
<groupId>com.googlecode.maven-download-plugin</groupId>
90121
<artifactId>download-maven-plugin</artifactId>
@@ -154,4 +185,85 @@ under the License.
154185
</plugin>
155186
</plugins>
156187
</build>
188+
189+
<profiles>
190+
<!--
191+
One of these profiles activates per host so the antrun copy
192+
knows the OS/arch directory and library filename to use.
193+
Resource layout follows the bundling convention:
194+
195+
org/apache/datafusion/linux/amd64/libdatafusion_jni.so
196+
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
197+
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
198+
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
199+
org/apache/datafusion/windows/amd64/datafusion_jni.dll
200+
-->
201+
<profile>
202+
<id>native-linux-amd64</id>
203+
<activation>
204+
<os><family>unix</family><name>linux</name><arch>amd64</arch></os>
205+
</activation>
206+
<properties>
207+
<datafusion.lib.os>linux</datafusion.lib.os>
208+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
209+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
210+
</properties>
211+
</profile>
212+
<profile>
213+
<id>native-linux-aarch64</id>
214+
<activation>
215+
<os><family>unix</family><name>linux</name><arch>aarch64</arch></os>
216+
</activation>
217+
<properties>
218+
<datafusion.lib.os>linux</datafusion.lib.os>
219+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
220+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
221+
</properties>
222+
</profile>
223+
<profile>
224+
<id>native-mac-x86_64</id>
225+
<activation>
226+
<os><family>mac</family><arch>x86_64</arch></os>
227+
</activation>
228+
<properties>
229+
<datafusion.lib.os>darwin</datafusion.lib.os>
230+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
231+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
232+
</properties>
233+
</profile>
234+
<profile>
235+
<!-- Some JVMs report os.arch=amd64 even on macOS x86_64. -->
236+
<id>native-mac-amd64</id>
237+
<activation>
238+
<os><family>mac</family><arch>amd64</arch></os>
239+
</activation>
240+
<properties>
241+
<datafusion.lib.os>darwin</datafusion.lib.os>
242+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
243+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
244+
</properties>
245+
</profile>
246+
<profile>
247+
<id>native-mac-aarch64</id>
248+
<activation>
249+
<os><family>mac</family><arch>aarch64</arch></os>
250+
</activation>
251+
<properties>
252+
<datafusion.lib.os>darwin</datafusion.lib.os>
253+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
254+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
255+
</properties>
256+
</profile>
257+
<profile>
258+
<id>native-windows-amd64</id>
259+
<activation>
260+
<os><family>windows</family><arch>amd64</arch></os>
261+
</activation>
262+
<properties>
263+
<datafusion.lib.os>windows</datafusion.lib.os>
264+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
265+
<datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
266+
</properties>
267+
</profile>
268+
</profiles>
157269
</project>
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.time.Duration;
23+
24+
import org.apache.datafusion.protobuf.CacheManagerOptionsProto;
25+
import org.apache.datafusion.protobuf.ListFilesCacheOptionsProto;
26+
27+
/**
28+
* Configuration for DataFusion's built-in {@code CacheManager}. Pass an instance to {@link
29+
* SessionContextBuilder#cacheManager(CacheManagerOptions)} to turn on any of the three
30+
* upstream-provided caches at session construction time.
31+
*
32+
* <p>The three caches are independent; calling one setter does not affect the others. A setter that
33+
* is never called leaves the upstream default in place — so existing callers that don't touch this
34+
* builder see no behavioral change.
35+
*
36+
* <ul>
37+
* <li>{@link Builder#fileMetadataCache(long)} — caches file-embedded metadata (e.g. parquet
38+
* footers / page metadata). Backed by {@code DefaultFilesMetadataCache} with the supplied
39+
* byte cap.
40+
* <li>{@link Builder#listFilesCache(long, Duration)} — caches results of object-store {@code
41+
* list} operations. Backed by {@code DefaultListFilesCache} with the supplied byte cap and
42+
* optional TTL ({@code null} = infinite).
43+
* <li>{@link Builder#fileStatisticsCache(boolean)} — caches per-file row counts and column
44+
* statistics. Backed by {@code DefaultFileStatisticsCache}; no bytewise cap exists upstream.
45+
* </ul>
46+
*
47+
* <p>This v1 surface configures the <em>built-in</em> cache implementations. Plugging custom Java
48+
* cache implementations through a JNI upcall path is intentionally out of scope — every cache
49+
* lookup is a hot path during scans, and routing them through Java would defeat the cache.
50+
*/
51+
public final class CacheManagerOptions {
52+
53+
private final Long fileMetadataCacheMaxBytes;
54+
private final ListFilesCacheConfig listFilesCache;
55+
private final Boolean fileStatisticsCacheEnabled;
56+
57+
private CacheManagerOptions(Builder b) {
58+
this.fileMetadataCacheMaxBytes = b.fileMetadataCacheMaxBytes;
59+
this.listFilesCache = b.listFilesCache;
60+
this.fileStatisticsCacheEnabled = b.fileStatisticsCacheEnabled;
61+
}
62+
63+
/** Begin building a {@link CacheManagerOptions} instance. */
64+
public static Builder builder() {
65+
return new Builder();
66+
}
67+
68+
CacheManagerOptionsProto toProto() {
69+
CacheManagerOptionsProto.Builder b = CacheManagerOptionsProto.newBuilder();
70+
if (fileMetadataCacheMaxBytes != null) {
71+
b.setFileMetadataCacheMaxBytes(fileMetadataCacheMaxBytes);
72+
}
73+
if (listFilesCache != null) {
74+
ListFilesCacheOptionsProto.Builder lb = ListFilesCacheOptionsProto.newBuilder();
75+
if (listFilesCache.maxBytes != null) {
76+
lb.setMaxBytes(listFilesCache.maxBytes);
77+
}
78+
if (listFilesCache.ttlMillis != null) {
79+
lb.setTtlMillis(listFilesCache.ttlMillis);
80+
}
81+
b.setListFilesCache(lb.build());
82+
}
83+
if (fileStatisticsCacheEnabled != null) {
84+
b.setFileStatisticsCacheEnabled(fileStatisticsCacheEnabled);
85+
}
86+
return b.build();
87+
}
88+
89+
/** Internal carrier — `null`-vs-set is the source of truth for "user called this setter". */
90+
private static final class ListFilesCacheConfig {
91+
final Long maxBytes;
92+
final Long ttlMillis;
93+
94+
ListFilesCacheConfig(Long maxBytes, Long ttlMillis) {
95+
this.maxBytes = maxBytes;
96+
this.ttlMillis = ttlMillis;
97+
}
98+
}
99+
100+
/** Builder for {@link CacheManagerOptions}. */
101+
public static final class Builder {
102+
private Long fileMetadataCacheMaxBytes;
103+
private ListFilesCacheConfig listFilesCache;
104+
private Boolean fileStatisticsCacheEnabled;
105+
106+
private Builder() {}
107+
108+
/**
109+
* Enable the file-embedded metadata cache (parquet footers, page metadata) with the given byte
110+
* cap. The cap is the budget the upstream {@code DefaultFilesMetadataCache} uses to evict
111+
* entries; {@code 0} is legal and means "construct the cache but with a 0-byte budget"
112+
* (effectively disabled but observable in stats).
113+
*
114+
* @throws IllegalArgumentException if {@code maxBytes} is negative.
115+
*/
116+
public Builder fileMetadataCache(long maxBytes) {
117+
if (maxBytes < 0) {
118+
throw new IllegalArgumentException(
119+
"fileMetadataCache maxBytes must be non-negative, got " + maxBytes);
120+
}
121+
this.fileMetadataCacheMaxBytes = maxBytes;
122+
return this;
123+
}
124+
125+
/**
126+
* Enable the list-files cache with the given byte cap and TTL. Pass {@code null} for {@code
127+
* ttl} to use upstream's "no expiration" semantics (entries are evicted only by capacity
128+
* pressure).
129+
*
130+
* @throws IllegalArgumentException if {@code maxBytes} is negative or {@code ttl} is negative.
131+
*/
132+
public Builder listFilesCache(long maxBytes, Duration ttl) {
133+
if (maxBytes < 0) {
134+
throw new IllegalArgumentException(
135+
"listFilesCache maxBytes must be non-negative, got " + maxBytes);
136+
}
137+
Long ttlMillis = null;
138+
if (ttl != null) {
139+
if (ttl.isNegative()) {
140+
throw new IllegalArgumentException("listFilesCache ttl must be non-negative, got " + ttl);
141+
}
142+
ttlMillis = ttl.toMillis();
143+
}
144+
this.listFilesCache = new ListFilesCacheConfig(maxBytes, ttlMillis);
145+
return this;
146+
}
147+
148+
/**
149+
* Enable the file-statistics cache (per-file row counts / column statistics). When {@code
150+
* enabled} is {@code true}, the upstream {@code DefaultFileStatisticsCache} is installed. When
151+
* {@code false}, the slot is explicitly set to disabled in the wire format — the same end-state
152+
* as never calling this setter, but distinguishable for testing.
153+
*/
154+
public Builder fileStatisticsCache(boolean enabled) {
155+
this.fileStatisticsCacheEnabled = enabled;
156+
return this;
157+
}
158+
159+
public CacheManagerOptions build() {
160+
return new CacheManagerOptions(this);
161+
}
162+
}
163+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Invalid or unrecognised DataFusion configuration option. Surfaces upstream {@code
24+
* DataFusionError::Configuration}.
25+
*/
26+
public class ConfigurationException extends DataFusionException {
27+
28+
private static final long serialVersionUID = 1L;
29+
30+
public ConfigurationException(String message) {
31+
super(message);
32+
}
33+
34+
public ConfigurationException(String message, Throwable cause) {
35+
super(message, cause);
36+
}
37+
}

0 commit comments

Comments
 (0)