Skip to content

Commit 5573fe2

Browse files
authored
feat(loader): bundle native library inside the JAR (#77)
1 parent c291b81 commit 5573fe2

5 files changed

Lines changed: 452 additions & 6 deletions

File tree

‎core/pom.xml‎

Lines changed: 113 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ under the License.
3434

3535
<name>Apache DataFusion Java</name>
3636

37+
<properties>
38+
<!-- Which cargo build profile (debug/release) the bundled native
39+
library is copied from. Override with -Ddatafusion.native.profile=release. -->
40+
<datafusion.native.profile>debug</datafusion.native.profile>
41+
</properties>
42+
3743
<dependencies>
3844
<dependency>
3945
<groupId>org.junit.jupiter</groupId>
@@ -77,9 +83,34 @@ under the License.
7783
<groupId>org.apache.maven.plugins</groupId>
7884
<artifactId>maven-surefire-plugin</artifactId>
7985
<configuration>
80-
<argLine>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug --add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
86+
<argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
8187
</configuration>
8288
</plugin>
89+
<plugin>
90+
<groupId>org.apache.maven.plugins</groupId>
91+
<artifactId>maven-antrun-plugin</artifactId>
92+
<executions>
93+
<execution>
94+
<id>copy-native-lib</id>
95+
<phase>process-classes</phase>
96+
<goals><goal>run</goal></goals>
97+
<configuration>
98+
<target>
99+
<property name="datafusion.native.lib.source"
100+
value="${maven.multiModuleProjectDirectory}/native/target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
101+
<fail message="Native library not found at ${datafusion.native.lib.source}. Run 'cd native &amp;&amp; cargo build' (or 'make') before building the JAR.">
102+
<condition><not><available file="${datafusion.native.lib.source}"/></not></condition>
103+
</fail>
104+
<mkdir dir="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}"/>
105+
<copy file="${datafusion.native.lib.source}"
106+
tofile="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}/${datafusion.lib.filename}"
107+
preservelastmodified="true"
108+
verbose="true"/>
109+
</target>
110+
</configuration>
111+
</execution>
112+
</executions>
113+
</plugin>
83114
<plugin>
84115
<groupId>com.googlecode.maven-download-plugin</groupId>
85116
<artifactId>download-maven-plugin</artifactId>
@@ -149,4 +180,85 @@ under the License.
149180
</plugin>
150181
</plugins>
151182
</build>
183+
184+
<profiles>
185+
<!--
186+
One of these profiles activates per host so the antrun copy
187+
knows the OS/arch directory and library filename to use.
188+
Resource layout follows the bundling convention:
189+
190+
org/apache/datafusion/linux/amd64/libdatafusion_jni.so
191+
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
192+
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
193+
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
194+
org/apache/datafusion/windows/amd64/datafusion_jni.dll
195+
-->
196+
<profile>
197+
<id>native-linux-amd64</id>
198+
<activation>
199+
<os><family>unix</family><name>linux</name><arch>amd64</arch></os>
200+
</activation>
201+
<properties>
202+
<datafusion.lib.os>linux</datafusion.lib.os>
203+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
204+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
205+
</properties>
206+
</profile>
207+
<profile>
208+
<id>native-linux-aarch64</id>
209+
<activation>
210+
<os><family>unix</family><name>linux</name><arch>aarch64</arch></os>
211+
</activation>
212+
<properties>
213+
<datafusion.lib.os>linux</datafusion.lib.os>
214+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
215+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
216+
</properties>
217+
</profile>
218+
<profile>
219+
<id>native-mac-x86_64</id>
220+
<activation>
221+
<os><family>mac</family><arch>x86_64</arch></os>
222+
</activation>
223+
<properties>
224+
<datafusion.lib.os>darwin</datafusion.lib.os>
225+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
226+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
227+
</properties>
228+
</profile>
229+
<profile>
230+
<!-- Some JVMs report os.arch=amd64 even on macOS x86_64. -->
231+
<id>native-mac-amd64</id>
232+
<activation>
233+
<os><family>mac</family><arch>amd64</arch></os>
234+
</activation>
235+
<properties>
236+
<datafusion.lib.os>darwin</datafusion.lib.os>
237+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
238+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
239+
</properties>
240+
</profile>
241+
<profile>
242+
<id>native-mac-aarch64</id>
243+
<activation>
244+
<os><family>mac</family><arch>aarch64</arch></os>
245+
</activation>
246+
<properties>
247+
<datafusion.lib.os>darwin</datafusion.lib.os>
248+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
249+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
250+
</properties>
251+
</profile>
252+
<profile>
253+
<id>native-windows-amd64</id>
254+
<activation>
255+
<os><family>windows</family><arch>amd64</arch></os>
256+
</activation>
257+
<properties>
258+
<datafusion.lib.os>windows</datafusion.lib.os>
259+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
260+
<datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
261+
</properties>
262+
</profile>
263+
</profiles>
152264
</project>

‎core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java‎

Lines changed: 84 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,96 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
27+
/**
28+
* Loads the {@code datafusion_jni} native library on demand.
29+
*
30+
* <p>The loader first tries {@link System#loadLibrary(String)} so that operators can override the
31+
* bundled library by placing a build on {@code java.library.path} (for example via {@code
32+
* -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that fails the loader extracts the
33+
* platform-specific library from the JAR resource tree and loads it via {@link
34+
* System#load(String)}.
35+
*
36+
* <p>The bundled libraries live at the conventional path {@code
37+
* org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}. Each JVM extracts into its own
38+
* private temporary directory created with {@link Files#createTempDirectory(String,
39+
* java.nio.file.attribute.FileAttribute[])}, which is mode 0700 on POSIX and user-private under the
40+
* default Windows {@code TMP}. Keeping the directory private prevents another user on the host from
41+
* planting a same-named library for the JVM to pick up.
42+
*/
2243
public final class NativeLibraryLoader {
23-
private static final String LIBRARY_NAME = "datafusion_jni";
24-
private static boolean loaded = false;
44+
45+
private static final String TMP_DIR_PREFIX = "datafusion-java-";
46+
47+
private static volatile boolean loaded;
2548

2649
private NativeLibraryLoader() {}
2750

2851
public static synchronized void loadLibrary() {
29-
if (!loaded) {
30-
System.loadLibrary(LIBRARY_NAME);
52+
if (loaded) {
53+
return;
54+
}
55+
if (tryLoadFromLibraryPath()) {
3156
loaded = true;
57+
return;
58+
}
59+
loadFromClasspath();
60+
loaded = true;
61+
}
62+
63+
private static boolean tryLoadFromLibraryPath() {
64+
try {
65+
System.loadLibrary(Platform.LIBRARY_NAME);
66+
return true;
67+
} catch (UnsatisfiedLinkError ignored) {
68+
return false;
3269
}
3370
}
71+
72+
private static void loadFromClasspath() {
73+
Platform platform = Platform.current();
74+
String resource = platform.resourcePath();
75+
try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
76+
if (check == null) {
77+
throw new UnsatisfiedLinkError(
78+
"No bundled datafusion_jni library for "
79+
+ platform
80+
+ " (expected classpath:"
81+
+ resource
82+
+ ")."
83+
+ " Build the native crate and add it to java.library.path,"
84+
+ " or depend on a JAR built for this platform.");
85+
}
86+
} catch (IOException e) {
87+
throw linkError("Failed to probe " + resource, e);
88+
}
89+
90+
try {
91+
Path extracted = extractToPrivateTempDir(resource, platform.libFileName());
92+
System.load(extracted.toAbsolutePath().toString());
93+
} catch (IOException e) {
94+
throw linkError("Failed to extract " + resource, e);
95+
}
96+
}
97+
98+
private static Path extractToPrivateTempDir(String resource, String fileName) throws IOException {
99+
Path tmpDir = Files.createTempDirectory(TMP_DIR_PREFIX);
100+
tmpDir.toFile().deleteOnExit();
101+
Path target = tmpDir.resolve(fileName);
102+
target.toFile().deleteOnExit();
103+
try (InputStream in = NativeLibraryLoader.class.getResourceAsStream(resource)) {
104+
Files.copy(in, target);
105+
}
106+
return target;
107+
}
108+
109+
private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
110+
UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
111+
err.initCause(cause);
112+
return err;
113+
}
34114
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import java.util.Locale;
23+
24+
/**
25+
* Identifies a target OS/arch pair and the path at which the bundled native library is published
26+
* inside the JAR.
27+
*
28+
* <p>Resource layout follows the convention from the {@code datafusion-java} packaging design:
29+
*
30+
* <pre>
31+
* org/apache/datafusion/linux/amd64/libdatafusion_jni.so
32+
* org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
33+
* org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
34+
* org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
35+
* org/apache/datafusion/windows/amd64/datafusion_jni.dll
36+
* </pre>
37+
*
38+
* <p>Package-private; consumed only by {@link NativeLibraryLoader}.
39+
*/
40+
final class Platform {
41+
42+
static final String LIBRARY_NAME = "datafusion_jni";
43+
static final String RESOURCE_PREFIX = "org/apache/datafusion";
44+
45+
enum Os {
46+
LINUX("linux", "lib", "so"),
47+
DARWIN("darwin", "lib", "dylib"),
48+
WINDOWS("windows", "", "dll");
49+
50+
final String dirName;
51+
final String libPrefix;
52+
final String libSuffix;
53+
54+
Os(String dirName, String libPrefix, String libSuffix) {
55+
this.dirName = dirName;
56+
this.libPrefix = libPrefix;
57+
this.libSuffix = libSuffix;
58+
}
59+
}
60+
61+
final Os os;
62+
final String arch;
63+
64+
private Platform(Os os, String arch) {
65+
this.os = os;
66+
this.arch = arch;
67+
}
68+
69+
static Platform current() {
70+
return of(System.getProperty("os.name"), System.getProperty("os.arch"));
71+
}
72+
73+
static Platform of(String osName, String osArch) {
74+
Os os = detectOs(osName);
75+
String arch = detectArch(os, osArch);
76+
return new Platform(os, arch);
77+
}
78+
79+
static Os detectOs(String osName) {
80+
if (osName == null) {
81+
throw new UnsupportedOperationException("os.name is not set");
82+
}
83+
String n = osName.toLowerCase(Locale.ROOT);
84+
if (n.startsWith("linux")) {
85+
return Os.LINUX;
86+
}
87+
if (n.startsWith("mac") || n.contains("darwin")) {
88+
return Os.DARWIN;
89+
}
90+
if (n.startsWith("windows")) {
91+
return Os.WINDOWS;
92+
}
93+
throw new UnsupportedOperationException("Unsupported OS for datafusion_jni: " + osName);
94+
}
95+
96+
/**
97+
* Returns the architecture segment used in the resource path for {@code os}.
98+
*
99+
* <p>Linux and Windows use {@code amd64} (Java's preferred name for x86_64), while macOS uses
100+
* {@code x86_64}; all use {@code aarch64} for ARM64.
101+
*/
102+
static String detectArch(Os os, String osArch) {
103+
if (osArch == null) {
104+
throw new UnsupportedOperationException("os.arch is not set");
105+
}
106+
String n = osArch.toLowerCase(Locale.ROOT);
107+
boolean isX64 = n.equals("amd64") || n.equals("x86_64") || n.equals("x64");
108+
boolean isArm64 = n.equals("aarch64") || n.equals("arm64");
109+
if (isX64) {
110+
return os == Os.DARWIN ? "x86_64" : "amd64";
111+
}
112+
if (isArm64) {
113+
return "aarch64";
114+
}
115+
throw new UnsupportedOperationException(
116+
"Unsupported CPU architecture for datafusion_jni: " + osArch);
117+
}
118+
119+
String libFileName() {
120+
return os.libPrefix + LIBRARY_NAME + "." + os.libSuffix;
121+
}
122+
123+
/**
124+
* Absolute classpath resource path (with leading slash) of the bundled native library for this
125+
* platform.
126+
*/
127+
String resourcePath() {
128+
return "/" + RESOURCE_PREFIX + "/" + os.dirName + "/" + arch + "/" + libFileName();
129+
}
130+
131+
@Override
132+
public String toString() {
133+
return os.dirName + "/" + arch;
134+
}
135+
}

0 commit comments

Comments
 (0)