Skip to content

Commit 9458b4e

Browse files
committed
fix(loader): isolate native lib extraction and add Windows build profile
Address PR #77 review feedback: - NativeLibraryLoader extracts to a per-JVM private temp directory via Files.createTempDirectory (mode 0700 on POSIX, user-private on Windows) instead of a shared $TMPDIR/datafusion-java/<sha256>/ tree. Removes the size-only existence check that would have let another user plant a same-sized library at the predictable hash path. - Add native-windows-amd64 Maven profile so datafusion.lib.os/arch/filename are defined on Windows; otherwise the process-classes copy step would abort the build even when the loader could fall back to System.loadLibrary. - Map Windows x86_64 -> amd64 in Platform.detectArch to match the new profile and Linux convention; extend PlatformTest accordingly.
1 parent 283e79e commit 9458b4e

4 files changed

Lines changed: 61 additions & 95 deletions

File tree

‎core/pom.xml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ under the License.
191191
org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
192192
org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
193193
org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
194+
org/apache/datafusion/windows/amd64/datafusion_jni.dll
194195
-->
195196
<profile>
196197
<id>native-linux-amd64</id>
@@ -248,5 +249,16 @@ under the License.
248249
<datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
249250
</properties>
250251
</profile>
252+
<profile>
253+
<id>native-windows-amd64</id>
254+
<activation>
255+
<os><family>windows</family><arch>amd64</arch></os>
256+
</activation>
257+
<properties>
258+
<datafusion.lib.os>windows</datafusion.lib.os>
259+
<datafusion.lib.arch>amd64</datafusion.lib.arch>
260+
<datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
261+
</properties>
262+
</profile>
251263
</profiles>
252264
</project>

‎core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java‎

Lines changed: 25 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -21,36 +21,28 @@
2121

2222
import java.io.IOException;
2323
import java.io.InputStream;
24-
import java.io.OutputStream;
25-
import java.nio.file.FileAlreadyExistsException;
2624
import java.nio.file.Files;
2725
import java.nio.file.Path;
28-
import java.nio.file.Paths;
29-
import java.nio.file.StandardCopyOption;
30-
import java.security.DigestInputStream;
31-
import java.security.MessageDigest;
32-
import java.security.NoSuchAlgorithmException;
3326

3427
/**
3528
* Loads the {@code datafusion_jni} native library on demand.
3629
*
37-
* <p>The loader first tries {@link System#loadLibrary(String)} so that
38-
* operators can override the bundled library by placing a build on
39-
* {@code java.library.path} (for example via
40-
* {@code -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that
41-
* fails the loader extracts the platform-specific library from the JAR
42-
* resource tree and loads it via {@link System#load(String)}.
30+
* <p>The loader first tries {@link System#loadLibrary(String)} so that operators can override the
31+
* bundled library by placing a build on {@code java.library.path} (for example via {@code
32+
* -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that fails the loader extracts the
33+
* platform-specific library from the JAR resource tree and loads it via {@link
34+
* System#load(String)}.
4335
*
44-
* <p>The bundled libraries live at the conventional path
45-
* {@code org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}.
46-
* Extracted files are written under
47-
* {@code $TMPDIR/datafusion-java/&lt;sha256&gt;/} so that concurrent JVMs
48-
* sharing a temp directory converge on the same file rather than each
49-
* extracting their own copy.
36+
* <p>The bundled libraries live at the conventional path {@code
37+
* org/apache/datafusion/&lt;os&gt;/&lt;arch&gt;/&lt;libfile&gt;}. Each JVM extracts into its own
38+
* private temporary directory created with {@link Files#createTempDirectory(String,
39+
* java.nio.file.attribute.FileAttribute[])}, which is mode 0700 on POSIX and user-private under the
40+
* default Windows {@code TMP}. Keeping the directory private prevents another user on the host from
41+
* planting a same-named library for the JVM to pick up.
5042
*/
5143
public final class NativeLibraryLoader {
5244

53-
private static final String TMP_DIR_NAME = "datafusion-java";
45+
private static final String TMP_DIR_PREFIX = "datafusion-java-";
5446

5547
private static volatile boolean loaded;
5648

@@ -83,8 +75,11 @@ private static void loadFromClasspath() {
8375
try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
8476
if (check == null) {
8577
throw new UnsatisfiedLinkError(
86-
"No bundled datafusion_jni library for " + platform
87-
+ " (expected classpath:" + resource + ")."
78+
"No bundled datafusion_jni library for "
79+
+ platform
80+
+ " (expected classpath:"
81+
+ resource
82+
+ ")."
8883
+ " Build the native crate and add it to java.library.path,"
8984
+ " or depend on a JAR built for this platform.");
9085
}
@@ -93,73 +88,24 @@ private static void loadFromClasspath() {
9388
}
9489

9590
try {
96-
Path extracted = extractToTempDir(resource, platform.libFileName());
91+
Path extracted = extractToPrivateTempDir(resource, platform.libFileName());
9792
System.load(extracted.toAbsolutePath().toString());
9893
} catch (IOException e) {
9994
throw linkError("Failed to extract " + resource, e);
10095
}
10196
}
10297

103-
private static Path extractToTempDir(String resource, String fileName) throws IOException {
104-
Path tmpRoot = Files.createDirectories(
105-
Paths.get(System.getProperty("java.io.tmpdir"), TMP_DIR_NAME));
106-
Path staging = Files.createTempFile(tmpRoot, fileName + ".", ".part");
107-
108-
String hash;
109-
try (InputStream raw = NativeLibraryLoader.class.getResourceAsStream(resource);
110-
DigestInputStream in = new DigestInputStream(raw, sha256());
111-
OutputStream out = Files.newOutputStream(staging)) {
112-
in.transferTo(out);
113-
hash = toHex(in.getMessageDigest().digest());
114-
} catch (IOException e) {
115-
Files.deleteIfExists(staging);
116-
throw e;
117-
}
118-
119-
Path versionedDir = Files.createDirectories(tmpRoot.resolve(hash));
120-
Path target = versionedDir.resolve(fileName);
121-
122-
if (Files.exists(target) && Files.size(target) == Files.size(staging)) {
123-
Files.deleteIfExists(staging);
124-
return target;
125-
}
126-
127-
try {
128-
Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE);
129-
} catch (FileAlreadyExistsException e) {
130-
// Another JVM extracted the same content while we were writing.
131-
// Their copy is identical (same SHA-256), so discard ours.
132-
Files.deleteIfExists(staging);
133-
} catch (IOException e) {
134-
// Atomic move not supported on this filesystem. Fall back to a
135-
// replacement move; the hash directory guarantees content equality.
136-
try {
137-
Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING);
138-
} catch (IOException retry) {
139-
Files.deleteIfExists(staging);
140-
throw retry;
141-
}
98+
private static Path extractToPrivateTempDir(String resource, String fileName) throws IOException {
99+
Path tmpDir = Files.createTempDirectory(TMP_DIR_PREFIX);
100+
tmpDir.toFile().deleteOnExit();
101+
Path target = tmpDir.resolve(fileName);
102+
target.toFile().deleteOnExit();
103+
try (InputStream in = NativeLibraryLoader.class.getResourceAsStream(resource)) {
104+
Files.copy(in, target);
142105
}
143106
return target;
144107
}
145108

146-
private static MessageDigest sha256() {
147-
try {
148-
return MessageDigest.getInstance("SHA-256");
149-
} catch (NoSuchAlgorithmException e) {
150-
throw new IllegalStateException("SHA-256 not available", e);
151-
}
152-
}
153-
154-
private static String toHex(byte[] bytes) {
155-
StringBuilder sb = new StringBuilder(bytes.length * 2);
156-
for (byte b : bytes) {
157-
sb.append(Character.forDigit((b >> 4) & 0xf, 16));
158-
sb.append(Character.forDigit(b & 0xf, 16));
159-
}
160-
return sb.toString();
161-
}
162-
163109
private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
164110
UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
165111
err.initCause(cause);

‎core/src/main/java/org/apache/datafusion/Platform.java‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,17 +22,17 @@
2222
import java.util.Locale;
2323

2424
/**
25-
* Identifies a target OS/arch pair and the path at which the bundled
26-
* native library is published inside the JAR.
25+
* Identifies a target OS/arch pair and the path at which the bundled native library is published
26+
* inside the JAR.
2727
*
28-
* <p>Resource layout follows the convention from the
29-
* {@code datafusion-java} packaging design:
28+
* <p>Resource layout follows the convention from the {@code datafusion-java} packaging design:
3029
*
3130
* <pre>
3231
* org/apache/datafusion/linux/amd64/libdatafusion_jni.so
3332
* org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
3433
* org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
3534
* org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
35+
* org/apache/datafusion/windows/amd64/datafusion_jni.dll
3636
* </pre>
3737
*
3838
* <p>Package-private; consumed only by {@link NativeLibraryLoader}.
@@ -90,15 +90,14 @@ static Os detectOs(String osName) {
9090
if (n.startsWith("windows")) {
9191
return Os.WINDOWS;
9292
}
93-
throw new UnsupportedOperationException(
94-
"Unsupported OS for datafusion_jni: " + osName);
93+
throw new UnsupportedOperationException("Unsupported OS for datafusion_jni: " + osName);
9594
}
9695

9796
/**
9897
* Returns the architecture segment used in the resource path for {@code os}.
9998
*
100-
* <p>Linux uses {@code amd64} (Java's preferred name for x86_64), while
101-
* macOS uses {@code x86_64}; both use {@code aarch64} for ARM64.
99+
* <p>Linux and Windows use {@code amd64} (Java's preferred name for x86_64), while macOS uses
100+
* {@code x86_64}; all use {@code aarch64} for ARM64.
102101
*/
103102
static String detectArch(Os os, String osArch) {
104103
if (osArch == null) {
@@ -108,7 +107,7 @@ static String detectArch(Os os, String osArch) {
108107
boolean isX64 = n.equals("amd64") || n.equals("x86_64") || n.equals("x64");
109108
boolean isArm64 = n.equals("aarch64") || n.equals("arm64");
110109
if (isX64) {
111-
return os == Os.LINUX ? "amd64" : "x86_64";
110+
return os == Os.DARWIN ? "x86_64" : "amd64";
112111
}
113112
if (isArm64) {
114113
return "aarch64";
@@ -122,8 +121,8 @@ String libFileName() {
122121
}
123122

124123
/**
125-
* Absolute classpath resource path (with leading slash) of the bundled
126-
* native library for this platform.
124+
* Absolute classpath resource path (with leading slash) of the bundled native library for this
125+
* platform.
127126
*/
128127
String resourcePath() {
129128
return "/" + RESOURCE_PREFIX + "/" + os.dirName + "/" + arch + "/" + libFileName();

‎core/src/test/java/org/apache/datafusion/PlatformTest.java‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,12 @@ void detectsWindowsFromOsName() {
4747

4848
@Test
4949
void rejectsUnknownOs() {
50-
assertThrows(UnsupportedOperationException.class,
51-
() -> Platform.detectOs("Solaris"));
50+
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs("Solaris"));
5251
}
5352

5453
@Test
5554
void rejectsNullOs() {
56-
assertThrows(UnsupportedOperationException.class,
57-
() -> Platform.detectOs(null));
55+
assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs(null));
5856
}
5957

6058
@Test
@@ -70,6 +68,13 @@ void usesX86_64OnDarwinForX86_64Aliases() {
7068
assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "x86_64"));
7169
}
7270

71+
@Test
72+
void usesAmd64OnWindowsForX86_64Aliases() {
73+
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "amd64"));
74+
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x86_64"));
75+
assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x64"));
76+
}
77+
7378
@Test
7479
void usesAarch64ForArm64Aliases() {
7580
assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "aarch64"));
@@ -80,7 +85,8 @@ void usesAarch64ForArm64Aliases() {
8085

8186
@Test
8287
void rejectsUnknownArch() {
83-
assertThrows(UnsupportedOperationException.class,
88+
assertThrows(
89+
UnsupportedOperationException.class,
8490
() -> Platform.detectArch(Platform.Os.LINUX, "ppc64le"));
8591
}
8692

@@ -107,5 +113,8 @@ void resourcePathMatchesSpec() {
107113
assertEquals(
108114
"/org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib",
109115
Platform.of("Mac OS X", "aarch64").resourcePath());
116+
assertEquals(
117+
"/org/apache/datafusion/windows/amd64/datafusion_jni.dll",
118+
Platform.of("Windows 11", "amd64").resourcePath());
110119
}
111120
}

0 commit comments

Comments
 (0)