Skip to content

Commit 283e79e

Browse files
committed
feat(loader): bundle native library inside the JAR
Rework NativeLibraryLoader so it can load datafusion_jni from a JAR-bundled native library instead of requiring java.library.path: 1. Try System.loadLibrary so operators can still override with a system-installed build via java.library.path / LD_LIBRARY_PATH. 2. On UnsatisfiedLinkError, detect the host OS/arch, look up the bundled resource at org/apache/datafusion/<os>/<arch>/lib<name>.<ext>, extract it to $TMPDIR/datafusion-java/<sha256>/ and load via System.load. Concurrent JVMs sharing the temp directory converge on the same SHA-256 hash directory; the extraction uses ATOMIC_MOVE so racing writers don't clobber each other. Wire core/pom.xml to copy the host's locally built native lib from native/target/<profile>/lib<name>.<ext> into target/classes at the matching resource path, so the produced JAR works out of the box on the build host. Per-platform Maven profiles set the OS/arch directory segments and library filename; -Ddatafusion.native.profile=release switches the copy source from debug to release. Drop -Djava.library.path from the surefire and exec-maven-plugin argLines so the test path now exercises the same resource-extraction code path users will hit. Refs #33 (work items 1 and 3); cross-build CI, Sonatype publishing, and release docs remain as follow-ups.
1 parent 7f49541 commit 283e79e

5 files changed

Lines changed: 486 additions & 6 deletions

File tree

‎core/pom.xml‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,12 @@ under the License.
3434

3535
<name>Apache DataFusion Java</name>
3636

37+
<properties>
38+
<!-- Which cargo build profile (debug/release) the bundled native
39+
library is copied from. Override with -Ddatafusion.native.profile=release. -->
40+
<datafusion.native.profile>debug</datafusion.native.profile>
41+
</properties>
42+
3743
<dependencies>
3844
<dependency>
3945
<groupId>org.junit.jupiter</groupId>
@@ -77,9 +83,34 @@ under the License.
7783
<groupId>org.apache.maven.plugins</groupId>
7884
<artifactId>maven-surefire-plugin</artifactId>
7985
<configuration>
80-
<argLine>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug --add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
86+
<argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
8187
</configuration>
8288
</plugin>
89+
<plugin>
90+
<groupId>org.apache.maven.plugins</groupId>
91+
<artifactId>maven-antrun-plugin</artifactId>
92+
<executions>
93+
<execution>
94+
<id>copy-native-lib</id>
95+
<phase>process-classes</phase>
96+
<goals><goal>run</goal></goals>
97+
<configuration>
98+
<target>
99+
<property name="datafusion.native.lib.source"
100+
value="${maven.multiModuleProjectDirectory}/native/target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
101+
<fail message="Native library not found at ${datafusion.native.lib.source}. Run 'cd native &amp;&amp; cargo build' (or 'make') before building the JAR.">
102+
<condition><not><available file="${datafusion.native.lib.source}"/></not></condition>
103+
</fail>
104+
<mkdir dir="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}"/>
105+
<copy file="${datafusion.native.lib.source}"
106+
tofile="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}/${datafusion.lib.filename}"
107+
preservelastmodified="true"
108+
verbose="true"/>
109+
</target>
110+
</configuration>
111+
</execution>
112+
</executions>
113+
</plugin>
83114
<plugin>
84115
<groupId>com.googlecode.maven-download-plugin</groupId>
85116
<artifactId>download-maven-plugin</artifactId>
@@ -149,4 +180,73 @@ under the License.
149180
</plugin>
150181
</plugins>
151182
</build>
183+
184+
<profiles>
185+
<!--
186+
One of these profiles activates per host so the antrun copy
187+
knows the OS/arch directory and library filename to use.
188+
Resource layout follows the bundling convention:
189+
190+
org/apache/datafusion/linux/amd64/libdatafusion_jni.so
191+
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
192+
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
193+
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
194+
-->
195+
<profile>
196+
<id>native-linux-amd64</id>
197+
<activation>
198+
<os><family>unix</family><name>linux</name><arch>amd64</arch></os>
199+
</activation>
200+
<properties>
201+
<datafusion.lib.os>linux</datafusion.lib.os>
202+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
203+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
204+
</properties>
205+
</profile>
206+
<profile>
207+
<id>native-linux-aarch64</id>
208+
<activation>
209+
<os><family>unix</family><name>linux</name><arch>aarch64</arch></os>
210+
</activation>
211+
<properties>
212+
<datafusion.lib.os>linux</datafusion.lib.os>
213+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
214+
<datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
215+
</properties>
216+
</profile>
217+
<profile>
218+
<id>native-mac-x86_64</id>
219+
<activation>
220+
<os><family>mac</family><arch>x86_64</arch></os>
221+
</activation>
222+
<properties>
223+
<datafusion.lib.os>darwin</datafusion.lib.os>
224+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
225+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
226+
</properties>
227+
</profile>
228+
<profile>
229+
<!-- Some JVMs report os.arch=amd64 even on macOS x86_64. -->
230+
<id>native-mac-amd64</id>
231+
<activation>
232+
<os><family>mac</family><arch>amd64</arch></os>
233+
</activation>
234+
<properties>
235+
<datafusion.lib.os>darwin</datafusion.lib.os>
236+
<datafusion.lib.arch>x86_64</datafusion.lib.arch>
237+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
238+
</properties>
239+
</profile>
240+
<profile>
241+
<id>native-mac-aarch64</id>
242+
<activation>
243+
<os><family>mac</family><arch>aarch64</arch></os>
244+
</activation>
245+
<properties>
246+
<datafusion.lib.os>darwin</datafusion.lib.os>
247+
<datafusion.lib.arch>aarch64</datafusion.lib.arch>
248+
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
249+
</properties>
250+
</profile>
251+
</profiles>
152252
</project>

‎core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java‎

Lines changed: 138 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,150 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.io.OutputStream;
25+
import java.nio.file.FileAlreadyExistsException;
26+
import java.nio.file.Files;
27+
import java.nio.file.Path;
28+
import java.nio.file.Paths;
29+
import java.nio.file.StandardCopyOption;
30+
import java.security.DigestInputStream;
31+
import java.security.MessageDigest;
32+
import java.security.NoSuchAlgorithmException;
33+
34+
/**
35+
* Loads the {@code datafusion_jni} native library on demand.
36+
*
37+
* <p>The loader first tries {@link System#loadLibrary(String)} so that
38+
* operators can override the bundled library by placing a build on
39+
* {@code java.library.path} (for example via
40+
* {@code -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that
41+
* fails the loader extracts the platform-specific library from the JAR
42+
* resource tree and loads it via {@link System#load(String)}.
43+
*
44+
* <p>The bundled libraries live at the conventional path
45+
* {@code org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}.
46+
* Extracted files are written under
47+
* {@code $TMPDIR/datafusion-java/&lt;sha256&gt;/} so that concurrent JVMs
48+
* sharing a temp directory converge on the same file rather than each
49+
* extracting their own copy.
50+
*/
2251
public final class NativeLibraryLoader {
23-
private static final String LIBRARY_NAME = "datafusion_jni";
24-
private static boolean loaded = false;
52+
53+
private static final String TMP_DIR_NAME = "datafusion-java";
54+
55+
private static volatile boolean loaded;
2556

2657
private NativeLibraryLoader() {}
2758

2859
public static synchronized void loadLibrary() {
29-
if (!loaded) {
30-
System.loadLibrary(LIBRARY_NAME);
60+
if (loaded) {
61+
return;
62+
}
63+
if (tryLoadFromLibraryPath()) {
3164
loaded = true;
65+
return;
66+
}
67+
loadFromClasspath();
68+
loaded = true;
69+
}
70+
71+
private static boolean tryLoadFromLibraryPath() {
72+
try {
73+
System.loadLibrary(Platform.LIBRARY_NAME);
74+
return true;
75+
} catch (UnsatisfiedLinkError ignored) {
76+
return false;
77+
}
78+
}
79+
80+
private static void loadFromClasspath() {
81+
Platform platform = Platform.current();
82+
String resource = platform.resourcePath();
83+
try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
84+
if (check == null) {
85+
throw new UnsatisfiedLinkError(
86+
"No bundled datafusion_jni library for " + platform
87+
+ " (expected classpath:" + resource + ")."
88+
+ " Build the native crate and add it to java.library.path,"
89+
+ " or depend on a JAR built for this platform.");
90+
}
91+
} catch (IOException e) {
92+
throw linkError("Failed to probe " + resource, e);
93+
}
94+
95+
try {
96+
Path extracted = extractToTempDir(resource, platform.libFileName());
97+
System.load(extracted.toAbsolutePath().toString());
98+
} catch (IOException e) {
99+
throw linkError("Failed to extract " + resource, e);
100+
}
101+
}
102+
103+
private static Path extractToTempDir(String resource, String fileName) throws IOException {
104+
Path tmpRoot = Files.createDirectories(
105+
Paths.get(System.getProperty("java.io.tmpdir"), TMP_DIR_NAME));
106+
Path staging = Files.createTempFile(tmpRoot, fileName + ".", ".part");
107+
108+
String hash;
109+
try (InputStream raw = NativeLibraryLoader.class.getResourceAsStream(resource);
110+
DigestInputStream in = new DigestInputStream(raw, sha256());
111+
OutputStream out = Files.newOutputStream(staging)) {
112+
in.transferTo(out);
113+
hash = toHex(in.getMessageDigest().digest());
114+
} catch (IOException e) {
115+
Files.deleteIfExists(staging);
116+
throw e;
117+
}
118+
119+
Path versionedDir = Files.createDirectories(tmpRoot.resolve(hash));
120+
Path target = versionedDir.resolve(fileName);
121+
122+
if (Files.exists(target) && Files.size(target) == Files.size(staging)) {
123+
Files.deleteIfExists(staging);
124+
return target;
125+
}
126+
127+
try {
128+
Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE);
129+
} catch (FileAlreadyExistsException e) {
130+
// Another JVM extracted the same content while we were writing.
131+
// Their copy is identical (same SHA-256), so discard ours.
132+
Files.deleteIfExists(staging);
133+
} catch (IOException e) {
134+
// Atomic move not supported on this filesystem. Fall back to a
135+
// replacement move; the hash directory guarantees content equality.
136+
try {
137+
Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
138+
} catch (IOException retry) {
139+
Files.deleteIfExists(staging);
140+
throw retry;
141+
}
32142
}
143+
return target;
144+
}
145+
146+
private static MessageDigest sha256() {
147+
try {
148+
return MessageDigest.getInstance("SHA-256");
149+
} catch (NoSuchAlgorithmException e) {
150+
throw new IllegalStateException("SHA-256 not available", e);
151+
}
152+
}
153+
154+
private static String toHex(byte[] bytes) {
155+
StringBuilder sb = new StringBuilder(bytes.length * 2);
156+
for (byte b : bytes) {
157+
sb.append(Character.forDigit((b >> 4) & 0xf, 16));
158+
sb.append(Character.forDigit(b & 0xf, 16));
159+
}
160+
return sb.toString();
161+
}
162+
163+
private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
164+
UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
165+
err.initCause(cause);
166+
return err;
33167
}
34168
}

0 commit comments

Comments
 (0)