Skip to content

Commit 1d3011d

Browse files
committed
Merge remote-tracking branch 'upstream/main' into feat/dataframe-cancellation
2 parents 4bd2ad1 + ff969cb commit 1d3011d

69 files changed

Lines changed: 8747 additions & 121 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/build.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@ name: Build
1919

2020
on:
2121
push:
22-
branches: [main]
22+
branches: [main, 'branch-*']
2323
pull_request:
24-
branches: [main]
24+
branches: [main, 'branch-*']
2525

2626
permissions:
2727
contents: read

‎Makefile‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,20 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
.PHONY: all native jvm test clean tpch-data
18+
.PHONY: all native native-runtime-metrics jvm test clean tpch-data
1919

2020
all: native jvm
2121

2222
native:
2323
cd native && cargo build
2424

25+
# Build the native crate with the `runtime-metrics` Cargo feature enabled.
26+
# Requires `--cfg tokio_unstable` because tokio-metrics gates its API there.
27+
# Default `make native` does not pull this in; callers who need
28+
# SessionContext.runtimeStats() pick this target explicitly.
29+
native-runtime-metrics:
30+
cd native && RUSTFLAGS="--cfg tokio_unstable" cargo build --features runtime-metrics
31+
2532
jvm:
2633
./mvnw package -DskipTests
2734

‎core/pom.xml‎

Lines changed: 114 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,20 @@ under the License.
2626
<parent>
2727
<groupId>org.apache.datafusion</groupId>
2828
<artifactId>datafusion-java-parent</artifactId>
29-
<version>0.1.0-SNAPSHOT</version>
29+
<version>0.2.0-SNAPSHOT</version>
3030
</parent>
3131

3232
<artifactId>datafusion-java</artifactId>
3333
<packaging>jar</packaging>
3434

3535
<name>Apache DataFusion Java</name>
3636

37+
<properties>
38+
<!-- Which cargo build profile (debug/release) the bundled native
39+
library is copied from. Override with -Ddatafusion.native.profile=release. -->
40+
<datafusion.native.profile>debug</datafusion.native.profile>
41+
</properties>
42+
3743
<dependencies>
3844
<dependency>
3945
<groupId>org.junit.jupiter</groupId>
@@ -77,9 +83,34 @@ under the License.
7783
<groupId>org.apache.maven.plugins</groupId>
7884
<artifactId>maven-surefire-plugin</artifactId>
7985
<configuration>
80-
<argLine>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug --add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
86+
<argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
8187
</configuration>
8288
</plugin>
89+
<plugin>
90+
<groupId>org.apache.maven.plugins</groupId>
91+
<artifactId>maven-antrun-plugin</artifactId>
92+
<executions>
93+
<execution>
94+
<id>copy-native-lib</id>
95+
<phase>process-classes</phase>
96+
<goals><goal>run</goal></goals>
97+
<configuration>
98+
<target>
99+
<property name="datafusion.native.lib.source"
100+
value="${maven.multiModuleProjectDirectory}/native/target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
101+
<fail message="Native library not found at ${datafusion.native.lib.source}. Run 'cd native &amp;&amp; cargo build' (or 'make') before building the JAR.">
102+
<condition><not><available file="${datafusion.native.lib.source}"/></not></condition>
103+
</fail>
104+
<mkdir dir="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}"/>
105+
<copy file="${datafusion.native.lib.source}"
106+
tofile="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}/${datafusion.lib.filename}"
107+
preservelastmodified="true"
108+
verbose="true"/>
109+
</target>
110+
</configuration>
111+
</execution>
112+
</executions>
113+
</plugin>
83114
<plugin>
84115
<groupId>com.googlecode.maven-download-plugin</groupId>
85116
<artifactId>download-maven-plugin</artifactId>
@@ -149,4 +180,85 @@ under the License.
149180
</plugin>
150181
</plugins>
151182
</build>
183+
184+
<profiles>
185+
<!--
186+
One of these profiles activates per host so the antrun copy
187+
knows the OS/arch directory and library filename to use.
188+
Resource layout follows the bundling convention:
189+
190+
org/apache/datafusion/linux/amd64/libdatafusion_jni.so
191+
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
192+
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
193+
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
194+
org/apache/datafusion/windows/amd64/datafusion_jni.dll
195+
-->
196+
<profile>
197+
<id>native-linux-amd64</id>
198+
<activation>
199+
<os><family>unix</family><name>linux</name><arch>amd64</arch></os>
200+
</activation>
201+
<properties>
202+
<datafusion.lib.os>linux</datafusion.lib.os>
203+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
204+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
205+
</properties>
206+
</profile>
207+
<profile>
208+
<id>native-linux-aarch64</id>
209+
<activation>
210+
<os><family>unix</family><name>linux</name><arch>aarch64</arch></os>
211+
</activation>
212+
<properties>
213+
<datafusion.lib.os>linux</datafusion.lib.os>
214+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
215+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
216+
</properties>
217+
</profile>
218+
<profile>
219+
<id>native-mac-x86_64</id>
220+
<activation>
221+
<os><family>mac</family><arch>x86_64</arch></os>
222+
</activation>
223+
<properties>
224+
<datafusion.lib.os>darwin</datafusion.lib.os>
225+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
226+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
227+
</properties>
228+
</profile>
229+
<profile>
230+
<!-- Some JVMs report os.arch=amd64 even on macOS x86_64. -->
231+
<id>native-mac-amd64</id>
232+
<activation>
233+
<os><family>mac</family><arch>amd64</arch></os>
234+
</activation>
235+
<properties>
236+
<datafusion.lib.os>darwin</datafusion.lib.os>
237+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
238+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
239+
</properties>
240+
</profile>
241+
<profile>
242+
<id>native-mac-aarch64</id>
243+
<activation>
244+
<os><family>mac</family><arch>aarch64</arch></os>
245+
</activation>
246+
<properties>
247+
<datafusion.lib.os>darwin</datafusion.lib.os>
248+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
249+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
250+
</properties>
251+
</profile>
252+
<profile>
253+
<id>native-windows-amd64</id>
254+
<activation>
255+
<os><family>windows</family><arch>amd64</arch></os>
256+
</activation>
257+
<properties>
258+
<datafusion.lib.os>windows</datafusion.lib.os>
259+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
260+
<datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
261+
</properties>
262+
</profile>
263+
</profiles>
152264
</project>
Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.time.Duration;
23+
24+
import org.apache.datafusion.protobuf.CacheManagerOptionsProto;
25+
import org.apache.datafusion.protobuf.ListFilesCacheOptionsProto;
26+
27+
/**
28+
* Configuration for DataFusion's built-in {@code CacheManager}. Pass an instance to {@link
29+
* SessionContextBuilder#cacheManager(CacheManagerOptions)} to turn on any of the three
30+
* upstream-provided caches at session construction time.
31+
*
32+
* <p>The three caches are independent; calling one setter does not affect the others. A setter that
33+
* is never called leaves the upstream default in place — so existing callers that don't touch this
34+
* builder see no behavioral change.
35+
*
36+
* <ul>
37+
* <li>{@link Builder#fileMetadataCache(long)} — caches file-embedded metadata (e.g. parquet
38+
* footers / page metadata). Backed by {@code DefaultFilesMetadataCache} with the supplied
39+
* byte cap.
40+
* <li>{@link Builder#listFilesCache(long, Duration)} — caches results of object-store {@code
41+
* list} operations. Backed by {@code DefaultListFilesCache} with the supplied byte cap and
42+
* optional TTL ({@code null} = infinite).
43+
* <li>{@link Builder#fileStatisticsCache(boolean)} — caches per-file row counts and column
44+
* statistics. Backed by {@code DefaultFileStatisticsCache}; no bytewise cap exists upstream.
45+
* </ul>
46+
*
47+
* <p>This v1 surface configures the <em>built-in</em> cache implementations. Plugging custom Java
48+
* cache implementations through a JNI upcall path is intentionally out of scope — every cache
49+
* lookup is a hot path during scans, and routing them through Java would defeat the cache.
50+
*/
51+
public final class CacheManagerOptions {
52+
53+
private final Long fileMetadataCacheMaxBytes;
54+
private final ListFilesCacheConfig listFilesCache;
55+
private final Boolean fileStatisticsCacheEnabled;
56+
57+
private CacheManagerOptions(Builder b) {
58+
this.fileMetadataCacheMaxBytes = b.fileMetadataCacheMaxBytes;
59+
this.listFilesCache = b.listFilesCache;
60+
this.fileStatisticsCacheEnabled = b.fileStatisticsCacheEnabled;
61+
}
62+
63+
/** Begin building a {@link CacheManagerOptions} instance. */
64+
public static Builder builder() {
65+
return new Builder();
66+
}
67+
68+
CacheManagerOptionsProto toProto() {
69+
CacheManagerOptionsProto.Builder b = CacheManagerOptionsProto.newBuilder();
70+
if (fileMetadataCacheMaxBytes != null) {
71+
b.setFileMetadataCacheMaxBytes(fileMetadataCacheMaxBytes);
72+
}
73+
if (listFilesCache != null) {
74+
ListFilesCacheOptionsProto.Builder lb = ListFilesCacheOptionsProto.newBuilder();
75+
if (listFilesCache.maxBytes != null) {
76+
lb.setMaxBytes(listFilesCache.maxBytes);
77+
}
78+
if (listFilesCache.ttlMillis != null) {
79+
lb.setTtlMillis(listFilesCache.ttlMillis);
80+
}
81+
b.setListFilesCache(lb.build());
82+
}
83+
if (fileStatisticsCacheEnabled != null) {
84+
b.setFileStatisticsCacheEnabled(fileStatisticsCacheEnabled);
85+
}
86+
return b.build();
87+
}
88+
89+
/** Internal carrier — `null`-vs-set is the source of truth for "user called this setter". */
90+
private static final class ListFilesCacheConfig {
91+
final Long maxBytes;
92+
final Long ttlMillis;
93+
94+
ListFilesCacheConfig(Long maxBytes, Long ttlMillis) {
95+
this.maxBytes = maxBytes;
96+
this.ttlMillis = ttlMillis;
97+
}
98+
}
99+
100+
/** Builder for {@link CacheManagerOptions}. */
101+
public static final class Builder {
102+
private Long fileMetadataCacheMaxBytes;
103+
private ListFilesCacheConfig listFilesCache;
104+
private Boolean fileStatisticsCacheEnabled;
105+
106+
private Builder() {}
107+
108+
/**
109+
* Enable the file-embedded metadata cache (parquet footers, page metadata) with the given byte
110+
* cap. The cap is the budget the upstream {@code DefaultFilesMetadataCache} uses to evict
111+
* entries; {@code 0} is legal and means "construct the cache but with a 0-byte budget"
112+
* (effectively disabled but observable in stats).
113+
*
114+
* @throws IllegalArgumentException if {@code maxBytes} is negative.
115+
*/
116+
public Builder fileMetadataCache(long maxBytes) {
117+
if (maxBytes < 0) {
118+
throw new IllegalArgumentException(
119+
"fileMetadataCache maxBytes must be non-negative, got " + maxBytes);
120+
}
121+
this.fileMetadataCacheMaxBytes = maxBytes;
122+
return this;
123+
}
124+
125+
/**
126+
* Enable the list-files cache with the given byte cap and TTL. Pass {@code null} for {@code
127+
* ttl} to use upstream's "no expiration" semantics (entries are evicted only by capacity
128+
* pressure).
129+
*
130+
* @throws IllegalArgumentException if {@code maxBytes} is negative or {@code ttl} is negative.
131+
*/
132+
public Builder listFilesCache(long maxBytes, Duration ttl) {
133+
if (maxBytes < 0) {
134+
throw new IllegalArgumentException(
135+
"listFilesCache maxBytes must be non-negative, got " + maxBytes);
136+
}
137+
Long ttlMillis = null;
138+
if (ttl != null) {
139+
if (ttl.isNegative()) {
140+
throw new IllegalArgumentException("listFilesCache ttl must be non-negative, got " + ttl);
141+
}
142+
ttlMillis = ttl.toMillis();
143+
}
144+
this.listFilesCache = new ListFilesCacheConfig(maxBytes, ttlMillis);
145+
return this;
146+
}
147+
148+
/**
149+
* Enable the file-statistics cache (per-file row counts / column statistics). When {@code
150+
* enabled} is {@code true}, the upstream {@code DefaultFileStatisticsCache} is installed. When
151+
* {@code false}, the slot is explicitly set to disabled in the wire format — the same end-state
152+
* as never calling this setter, but distinguishable for testing.
153+
*/
154+
public Builder fileStatisticsCache(boolean enabled) {
155+
this.fileStatisticsCacheEnabled = enabled;
156+
return this;
157+
}
158+
159+
public CacheManagerOptions build() {
160+
return new CacheManagerOptions(this);
161+
}
162+
}
163+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
/**
23+
* Invalid or unrecognised DataFusion configuration option. Surfaces upstream {@code
24+
* DataFusionError::Configuration}.
25+
*/
26+
public class ConfigurationException extends DataFusionException {
27+
28+
private static final long serialVersionUID = 1L;
29+
30+
public ConfigurationException(String message) {
31+
super(message);
32+
}
33+
34+
public ConfigurationException(String message, Throwable cause) {
35+
super(message, cause);
36+
}
37+
}

0 commit comments

Comments
 (0)