fix: vendor libhdfs with the HDFS-16021 thread-ownership fix

NOTICE.txt

@@ -8,3 +8,9 @@ This product includes software developed at
 Apache Gluten (https://github.com/apache/incubator-gluten/)
 Specifically:
 - Optimizer rule to replace SortMergeJoin with ShuffleHashJoin
+
+This product includes software developed at
+Apache Hadoop (https://hadoop.apache.org/)
+Specifically:
+- The libhdfs C sources under native/hdfs-sys/libhdfs/, modified as described
+  in native/hdfs-sys/README.md

dev/ci/compute-changes.py

@@ -108,6 +108,9 @@
     ],
     "spark_3_4": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -134,6 +137,9 @@
     ],
     "spark_3_5": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -160,6 +166,9 @@
     ],
     "spark_4_0": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -186,6 +195,9 @@
     ],
     "spark_4_1": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -212,6 +224,9 @@
     ],
     "iceberg_1_8": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -236,6 +251,9 @@
     ],
     "iceberg_1_9": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -260,6 +278,9 @@
     ],
     "iceberg_1_10": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",
@@ -284,6 +305,9 @@
     ],
     "iceberg_1_11": [
         "native/**/src/**",
+        # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but
+        # lives outside any src/ directory, so it needs its own entry.
+        "native/hdfs-sys/**",
         "native/**/Cargo.toml",
         "native/Cargo.lock",
         "common/src/main/**",

native/Cargo.toml

@@ -17,7 +17,10 @@
 
 [workspace]
 default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"]
-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"]
+# `hdfs-sys` is a member so that `cargo fmt --all` and `cargo clippy --workspace` cover it, but
+# deliberately not a default member: it is only reached through `[patch.crates-io]` when the
+# `hdfs-opendal` feature is on, and building it needs a JDK and a C compiler.
+members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs-sys"]
 # The contrib crate at ../contrib/delta/native is intentionally NOT a workspace member
 # (workspace members must live hierarchically under the workspace root). It's pulled in
 # as a path dep by `core/Cargo.toml` when the `contrib-delta` feature is enabled.
@@ -82,3 +85,17 @@ codegen-units = 16   # Parallel codegen (faster compile, slightly larger binary)
 debug-assertions = true
 panic = "unwind"     # Allow panics to be caught and logged across FFI boundary
 # overflow-checks inherited as false from release
+
+# `hdfs-sys` 0.3.0, the only release, ships a libhdfs whose pthread TLS
+# destructor detaches threads it did not attach, dereferencing a freed JNIEnv and
+# crashing the JVM (apache/datafusion-comet#5023). `hdfs-sys` is reached through
+# `opendal`'s `services-hdfs` -> `hdrs`, so there is no version of the dependency
+# to move to and no upstream release to wait for. `hdfs-sys/` vendors the Hadoop
+# sources with the fix applied; see its README.md for the removal condition.
+#
+# Note that `[patch]` applies only to builds of this workspace. Comet's native
+# crates are not published, so nothing downstream is affected today, but a
+# published crate would silently resolve the unpatched `hdfs-sys`.
+[patch.crates-io]
+hdfs-sys = { path = "hdfs-sys" }
+

native/hdfs-sys/Cargo.toml

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+# Deliberately named after the crates.io crate: `native/Cargo.toml` substitutes
+# this for it via `[patch.crates-io]`, which requires a matching name and a
+# semver-compatible version. See README.md.
+name = "hdfs-sys"
+version = "0.3.0"
+edition = "2021"
+description = "Bindings to the Apache Hadoop libhdfs C API, carrying the HDFS-16021 fix"
+authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
+license = "Apache-2.0"
+publish = false
+readme = "README.md"
+
+[features]
+# The crates.io crate gates each vendored Hadoop version behind a feature, with
+# each enabling the one below it. Comet's dependency graph asks for `hdfs_3_3`
+# (from `core/Cargo.toml`) and the default (from `hdrs`), and only the 3.3
+# sources are vendored here, so the older names exist purely to keep those
+# requests resolvable.
+default = ["hdfs_2_6"]
+hdfs_2_2 = []
+hdfs_2_3 = ["hdfs_2_2"]
+hdfs_2_4 = ["hdfs_2_3"]
+hdfs_2_5 = ["hdfs_2_4"]
+hdfs_2_6 = ["hdfs_2_5"]
+hdfs_2_7 = ["hdfs_2_6"]
+hdfs_2_8 = ["hdfs_2_7"]
+hdfs_2_9 = ["hdfs_2_8"]
+hdfs_2_10 = ["hdfs_2_9"]
+hdfs_3_0 = ["hdfs_2_10"]
+hdfs_3_1 = ["hdfs_3_0"]
+hdfs_3_2 = ["hdfs_3_1"]
+hdfs_3_3 = ["hdfs_3_2"]
+# Skip the system-libhdfs search and always compile the vendored sources.
+vendored = []
+
+[build-dependencies]
+cc = "1"
+java-locator = { version = "0.1.9", features = ["locate-jdk-only"] }

native/hdfs-sys/README.md

@@ -0,0 +1,98 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# hdfs-sys
+
+Bindings to the Apache Hadoop `libhdfs` C API, carrying the
+[HDFS-16021](https://issues.apache.org/jira/browse/HDFS-16021) thread-ownership fix.
+
+This crate substitutes for the crates.io [`hdfs-sys`](https://github.com/Xuanwo/hdfs-sys) crate
+through a `[patch.crates-io]` entry in `native/Cargo.toml`. It is not published and is not
+intended for use outside Comet.
+
+## Why it exists
+
+`libhdfs` registers a pthread thread-local destructor, `hdfsThreadDestructor`, which detaches the
+current thread from the JVM. It does so for every thread that has a cached `JNIEnv`, including
+threads that libhdfs did not attach. Comet attaches each of its Tokio worker threads with
+`AttachCurrentThreadAsDaemon` and detaches them itself on thread stop, so by the time the pthread
+destructor runs the `JNIEnv` has already been freed. Dereferencing it jumps through a null function
+pointer and takes the JVM down with `SIGSEGV at pc=0x0`.
+
+That is [apache/datafusion-comet#5023](https://github.com/apache/datafusion-comet/issues/5023),
+which reproduces on both Linux and macOS in the `[scans]` CI bucket. `ParquetReadFromFakeHadoopFsSuite`
+is what routes a read through `libhdfs`, but the destructor fires whenever one of those threads
+later exits, so the crash lands in an unrelated suite sharing the same JVM.
+
+There is no released `hdfs-sys` with the fix, and no way to avoid the dependency: it arrives through
+`opendal`'s `services-hdfs` feature by way of `hdrs`. The last crates.io release is 0.3.0 from July
+2023, and a fix merged upstream in January 2026 is still unreleased, so waiting is not a strategy.
+
+## Provenance
+
+| Component             | Source                                                                                                                                   |
+| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
+| `libhdfs/hdfs_3_3/**` | Apache Hadoop, via the `hdfs-sys` 0.3.0 crate's vendored copy of `hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs` |
+| `libhdfs/config.h`    | Written for Comet. Hadoop generates this with CMake; the upstream crate ships it empty                                                   |
+| `src/lib.rs`          | Written for Comet, transcribed from the vendored `libhdfs/hdfs_3_3/include/hdfs/hdfs.h`                                                  |
+| `build.rs`            | Written for Comet, following the file list in Hadoop's `hadoop-hdfs-native-client/src/CMakeLists.txt`                                    |
+
+The C sources are Apache Hadoop's own, carry their original ASF license headers, and are
+unmodified apart from the changes listed below. No code authored by the `hdfs-sys` maintainer is
+copied here: the Rust binding layer and the build script were written for Comet against Hadoop's
+public header, which is why only the API surface `hdrs` uses is declared.
+
+Relative to the upstream crate this copy also drops everything Comet does not build: the twelve
+vendored Hadoop versions older than 3.3, the Windows platform layer, and the bundled `libdirent`
+(MIT). Comet ships no Windows native artifacts.
+
+## Modifications
+
+Three files differ from Hadoop's originals. Each carries a notice at the top of the file, as
+required by section 4(b) of the Apache License.
+
+- `libhdfs/hdfs_3_3/os/thread_local_storage.h` — adds an `attachedByLibhdfs` flag to
+  `struct ThreadLocalState`.
+- `libhdfs/hdfs_3_3/jni_helper.c` — `getGlobalJNIEnv` reports whether it attached the current
+  thread, and calls `GetEnv` before `AttachCurrentThread` so that an attachment made by the JVM or
+  by the embedding application is reused rather than claimed.
+- `libhdfs/hdfs_3_3/os/posix/thread_local_storage.c` — `hdfsThreadDestructor` detaches only when
+  `attachedByLibhdfs` is set, and `threadLocalStorageCreate` initialises the two fields it
+  previously left holding `malloc` garbage.
+
+The first and third come from the patch attached to HDFS-16021. The `GetEnv` check in the second
+does not, and is the part that matters for Comet: `AttachCurrentThread` succeeds on an
+already-attached thread and returns the same `JNIEnv`, so without it libhdfs would still record
+itself as the owner of an attachment Comet made.
+
+The same changes are proposed upstream as
+[Xuanwo/hdfs-sys#47](https://github.com/Xuanwo/hdfs-sys/pull/47).
+
+## Removal condition
+
+Delete this directory, drop the `[patch.crates-io]` entry from `native/Cargo.toml`, and remove the
+NOTICE.txt stanza once a crates.io release of `hdfs-sys` contains the fix. Hadoop's own copy is not
+sufficient on its own: HDFS-16021 is still open, and trunk still has the unguarded destructor.
+
+## A system libhdfs will not carry the fix
+
+`build.rs` keeps the upstream resolution order, so setting `HDFS_LIB_DIR` or `HADOOP_HOME` links a
+prebuilt `libhdfs` instead of compiling these sources, and that library has whatever behaviour its
+own build gave it. The `vendored` feature, which Comet enables on macOS through `hdrs`, skips the
+search. On Linux neither variable is set in CI, so the vendored sources are built there too.