From 3152da621232a6497518c2b110e9342ff723f291 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 12 Sep 2026 15:43:45 -0600 Subject: [PATCH] fix: vendor libhdfs with the HDFS-16021 thread-ownership fix libhdfs registers a pthread TLS destructor that detaches the current thread from the JVM whenever it finds a cached JNIEnv, including on threads it did not attach. Comet attaches its own Tokio workers with AttachCurrentThreadAsDaemon and detaches them on thread stop, so by the time the destructor runs the JNIEnv is already freed and dereferencing it faults at pc=0. That is the sporadic [scans] SIGSEGV in #5023. No released hdfs-sys carries the fix and the crate has not shipped a release since 0.3.0 in July 2023, so vendor the Hadoop sources Comet actually builds -- hdfs_3_3, POSIX only -- with the fix applied, and substitute them through [patch.crates-io]. Closes #5023. --- NOTICE.txt | 6 + dev/ci/compute-changes.py | 24 + native/Cargo.lock | 2 - native/Cargo.toml | 19 +- native/hdfs-sys/Cargo.toml | 56 + native/hdfs-sys/README.md | 98 + native/hdfs-sys/build.rs | 142 + native/hdfs-sys/libhdfs/config.h | 26 + native/hdfs-sys/libhdfs/hdfs_3_3/exception.c | 272 ++ native/hdfs-sys/libhdfs/hdfs_3_3/exception.h | 165 + native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c | 3831 +++++++++++++++++ .../libhdfs/hdfs_3_3/include/hdfs/hdfs.h | 1105 +++++ native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c | 136 + native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h | 112 + native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c | 994 +++++ native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h | 221 + native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h | 55 + .../libhdfs/hdfs_3_3/os/posix/mutexes.c | 50 + .../libhdfs/hdfs_3_3/os/posix/platform.h | 34 + .../libhdfs/hdfs_3_3/os/posix/thread.c | 52 + .../hdfs_3_3/os/posix/thread_local_storage.c | 207 + native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h | 54 + .../hdfs_3_3/os/thread_local_storage.h | 110 + native/hdfs-sys/src/lib.rs | 191 + 24 files changed, 7959 insertions(+), 3 deletions(-) create mode 100644 native/hdfs-sys/Cargo.toml create mode 100644 native/hdfs-sys/README.md create mode 100644 native/hdfs-sys/build.rs create mode 100644 native/hdfs-sys/libhdfs/config.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/exception.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/exception.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h create mode 100644 native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h create mode 100644 native/hdfs-sys/src/lib.rs diff --git a/NOTICE.txt b/NOTICE.txt index b572b1fa29d..5a994053030 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -8,3 +8,9 @@ This product includes software developed at Apache Gluten (https://github.com/apache/incubator-gluten/) Specifically: - Optimizer rule to replace SortMergeJoin with ShuffleHashJoin + +This product includes software developed at +Apache Hadoop (https://hadoop.apache.org/) +Specifically: +- The libhdfs C sources under native/hdfs-sys/libhdfs/, modified as described + in native/hdfs-sys/README.md diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 9716ced2342..1eea11973d3 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -108,6 +108,9 @@ ], "spark_3_4": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -134,6 +137,9 @@ ], "spark_3_5": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -160,6 +166,9 @@ ], "spark_4_0": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -186,6 +195,9 @@ ], "spark_4_1": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -212,6 +224,9 @@ ], "iceberg_1_8": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -236,6 +251,9 @@ ], "iceberg_1_9": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -260,6 +278,9 @@ ], "iceberg_1_10": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", @@ -284,6 +305,9 @@ ], "iceberg_1_11": [ "native/**/src/**", + # The vendored Hadoop C in native/hdfs-sys/ is compiled into libcomet but + # lives outside any src/ directory, so it needs its own entry. + "native/hdfs-sys/**", "native/**/Cargo.toml", "native/Cargo.lock", "common/src/main/**", diff --git a/native/Cargo.lock b/native/Cargo.lock index df5fd4ac14a..545f105b1f2 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -3314,8 +3314,6 @@ dependencies = [ [[package]] name = "hdfs-sys" version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33e2d5cefba2d51a26b44d2a493f963a32725a0f6593c91be4a610ad449c49cb" dependencies = [ "cc", "java-locator", diff --git a/native/Cargo.toml b/native/Cargo.toml index 1805a185e9d..b3fbe60a0c6 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -17,7 +17,10 @@ [workspace] default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] -members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle"] +# `hdfs-sys` is a member so that `cargo fmt --all` and `cargo clippy --workspace` cover it, but +# deliberately not a default member: it is only reached through `[patch.crates-io]` when the +# `hdfs-opendal` feature is on, and building it needs a JDK and a C compiler. +members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "hdfs-sys"] # The contrib crate at ../contrib/delta/native is intentionally NOT a workspace member # (workspace members must live hierarchically under the workspace root). It's pulled in # as a path dep by `core/Cargo.toml` when the `contrib-delta` feature is enabled. @@ -82,3 +85,17 @@ codegen-units = 16 # Parallel codegen (faster compile, slightly larger binary) debug-assertions = true panic = "unwind" # Allow panics to be caught and logged across FFI boundary # overflow-checks inherited as false from release + +# `hdfs-sys` 0.3.0, the only release, ships a libhdfs whose pthread TLS +# destructor detaches threads it did not attach, dereferencing a freed JNIEnv and +# crashing the JVM (apache/datafusion-comet#5023). `hdfs-sys` is reached through +# `opendal`'s `services-hdfs` -> `hdrs`, so there is no version of the dependency +# to move to and no upstream release to wait for. `hdfs-sys/` vendors the Hadoop +# sources with the fix applied; see its README.md for the removal condition. +# +# Note that `[patch]` applies only to builds of this workspace. Comet's native +# crates are not published, so nothing downstream is affected today, but a +# published crate would silently resolve the unpatched `hdfs-sys`. +[patch.crates-io] +hdfs-sys = { path = "hdfs-sys" } + diff --git a/native/hdfs-sys/Cargo.toml b/native/hdfs-sys/Cargo.toml new file mode 100644 index 00000000000..0f53dc7c97c --- /dev/null +++ b/native/hdfs-sys/Cargo.toml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +# Deliberately named after the crates.io crate: `native/Cargo.toml` substitutes +# this for it via `[patch.crates-io]`, which requires a matching name and a +# semver-compatible version. See README.md. +name = "hdfs-sys" +version = "0.3.0" +edition = "2021" +description = "Bindings to the Apache Hadoop libhdfs C API, carrying the HDFS-16021 fix" +authors = ["Apache DataFusion "] +license = "Apache-2.0" +publish = false +readme = "README.md" + +[features] +# The crates.io crate gates each vendored Hadoop version behind a feature, with +# each enabling the one below it. Comet's dependency graph asks for `hdfs_3_3` +# (from `core/Cargo.toml`) and the default (from `hdrs`), and only the 3.3 +# sources are vendored here, so the older names exist purely to keep those +# requests resolvable. +default = ["hdfs_2_6"] +hdfs_2_2 = [] +hdfs_2_3 = ["hdfs_2_2"] +hdfs_2_4 = ["hdfs_2_3"] +hdfs_2_5 = ["hdfs_2_4"] +hdfs_2_6 = ["hdfs_2_5"] +hdfs_2_7 = ["hdfs_2_6"] +hdfs_2_8 = ["hdfs_2_7"] +hdfs_2_9 = ["hdfs_2_8"] +hdfs_2_10 = ["hdfs_2_9"] +hdfs_3_0 = ["hdfs_2_10"] +hdfs_3_1 = ["hdfs_3_0"] +hdfs_3_2 = ["hdfs_3_1"] +hdfs_3_3 = ["hdfs_3_2"] +# Skip the system-libhdfs search and always compile the vendored sources. +vendored = [] + +[build-dependencies] +cc = "1" +java-locator = { version = "0.1.9", features = ["locate-jdk-only"] } diff --git a/native/hdfs-sys/README.md b/native/hdfs-sys/README.md new file mode 100644 index 00000000000..884f44f0e80 --- /dev/null +++ b/native/hdfs-sys/README.md @@ -0,0 +1,98 @@ + + +# hdfs-sys + +Bindings to the Apache Hadoop `libhdfs` C API, carrying the +[HDFS-16021](https://issues.apache.org/jira/browse/HDFS-16021) thread-ownership fix. + +This crate substitutes for the crates.io [`hdfs-sys`](https://github.com/Xuanwo/hdfs-sys) crate +through a `[patch.crates-io]` entry in `native/Cargo.toml`. It is not published and is not +intended for use outside Comet. + +## Why it exists + +`libhdfs` registers a pthread thread-local destructor, `hdfsThreadDestructor`, which detaches the +current thread from the JVM. It does so for every thread that has a cached `JNIEnv`, including +threads that libhdfs did not attach. Comet attaches each of its Tokio worker threads with +`AttachCurrentThreadAsDaemon` and detaches them itself on thread stop, so by the time the pthread +destructor runs the `JNIEnv` has already been freed. Dereferencing it jumps through a null function +pointer and takes the JVM down with `SIGSEGV at pc=0x0`. + +That is [apache/datafusion-comet#5023](https://github.com/apache/datafusion-comet/issues/5023), +which reproduces on both Linux and macOS in the `[scans]` CI bucket. `ParquetReadFromFakeHadoopFsSuite` +is what routes a read through `libhdfs`, but the destructor fires whenever one of those threads +later exits, so the crash lands in an unrelated suite sharing the same JVM. + +There is no released `hdfs-sys` with the fix, and no way to avoid the dependency: it arrives through +`opendal`'s `services-hdfs` feature by way of `hdrs`. The last crates.io release is 0.3.0 from July +2023, and a fix merged upstream in January 2026 is still unreleased, so waiting is not a strategy. + +## Provenance + +| Component | Source | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `libhdfs/hdfs_3_3/**` | Apache Hadoop, via the `hdfs-sys` 0.3.0 crate's vendored copy of `hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs` | +| `libhdfs/config.h` | Written for Comet. Hadoop generates this with CMake; the upstream crate ships it empty | +| `src/lib.rs` | Written for Comet, transcribed from the vendored `libhdfs/hdfs_3_3/include/hdfs/hdfs.h` | +| `build.rs` | Written for Comet, following the file list in Hadoop's `hadoop-hdfs-native-client/src/CMakeLists.txt` | + +The C sources are Apache Hadoop's own, carry their original ASF license headers, and are +unmodified apart from the changes listed below. No code authored by the `hdfs-sys` maintainer is +copied here: the Rust binding layer and the build script were written for Comet against Hadoop's +public header, which is why only the API surface `hdrs` uses is declared. + +Relative to the upstream crate this copy also drops everything Comet does not build: the twelve +vendored Hadoop versions older than 3.3, the Windows platform layer, and the bundled `libdirent` +(MIT). Comet ships no Windows native artifacts. + +## Modifications + +Three files differ from Hadoop's originals. Each carries a notice at the top of the file, as +required by section 4(b) of the Apache License. + +- `libhdfs/hdfs_3_3/os/thread_local_storage.h` — adds an `attachedByLibhdfs` flag to + `struct ThreadLocalState`. +- `libhdfs/hdfs_3_3/jni_helper.c` — `getGlobalJNIEnv` reports whether it attached the current + thread, and calls `GetEnv` before `AttachCurrentThread` so that an attachment made by the JVM or + by the embedding application is reused rather than claimed. +- `libhdfs/hdfs_3_3/os/posix/thread_local_storage.c` — `hdfsThreadDestructor` detaches only when + `attachedByLibhdfs` is set, and `threadLocalStorageCreate` initialises the two fields it + previously left holding `malloc` garbage. + +The first and third come from the patch attached to HDFS-16021. The `GetEnv` check in the second +does not, and is the part that matters for Comet: `AttachCurrentThread` succeeds on an +already-attached thread and returns the same `JNIEnv`, so without it libhdfs would still record +itself as the owner of an attachment Comet made. + +The same changes are proposed upstream as +[Xuanwo/hdfs-sys#47](https://github.com/Xuanwo/hdfs-sys/pull/47). + +## Removal condition + +Delete this directory, drop the `[patch.crates-io]` entry from `native/Cargo.toml`, and remove the +NOTICE.txt stanza once a crates.io release of `hdfs-sys` contains the fix. Hadoop's own copy is not +sufficient on its own: HDFS-16021 is still open, and trunk still has the unguarded destructor. + +## A system libhdfs will not carry the fix + +`build.rs` keeps the upstream resolution order, so setting `HDFS_LIB_DIR` or `HADOOP_HOME` links a +prebuilt `libhdfs` instead of compiling these sources, and that library has whatever behaviour its +own build gave it. The `vendored` feature, which Comet enables on macOS through `hdrs`, skips the +search. On Linux neither variable is set in CI, so the vendored sources are built there too. diff --git a/native/hdfs-sys/build.rs b/native/hdfs-sys/build.rs new file mode 100644 index 00000000000..1aa1d7ca0d8 --- /dev/null +++ b/native/hdfs-sys/build.rs @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Links `libjvm` and either finds a system `libhdfs` or compiles the vendored +//! Apache Hadoop sources under `libhdfs/hdfs_3_3/`. +//! +//! The resolution order matches the crates.io `hdfs-sys` crate this stands in +//! for, so setting `HDFS_LIB_DIR` or `HADOOP_HOME` still selects a system +//! `libhdfs`. Note that such a library will not carry the HDFS-16021 fix; see +//! `README.md`. + +use std::env; +use std::path::Path; + +type Result = std::result::Result>; + +/// The only vendored libhdfs version. Upstream ships every Hadoop release from +/// 2.2 onwards behind a feature; Comet builds `hdfs_3_3` and nothing else. +const SRC: &str = "libhdfs/hdfs_3_3"; + +fn main() -> Result<()> { + // Nothing to link against when docs.rs builds documentation. + if env::var_os("DOCS_RS").is_some() { + return Ok(()); + } + + link_jvm()?; + + if !find_system_libhdfs()? { + build_vendored_libhdfs()?; + } + + Ok(()) +} + +/// Points the linker at the `libjvm` belonging to the JDK that `java-locator` +/// resolves, which is `JAVA_HOME` when it is set. +fn link_jvm() -> Result<()> { + let jvm_path = java_locator::locate_jvm_dyn_library()?; + println!("cargo:rustc-link-lib=jvm"); + println!("cargo:rustc-link-search=native={jvm_path}"); + Ok(()) +} + +/// Returns `true` when a prebuilt `libhdfs` was found and linked. +/// +/// Checks `HDFS_LIB_DIR` first, then `HADOOP_HOME`. `HDFS_STATIC` selects static +/// linking. The `vendored` feature skips the search outright. +fn find_system_libhdfs() -> Result { + println!("cargo:rerun-if-env-changed=HDFS_LIB_DIR"); + println!("cargo:rerun-if-env-changed=HDFS_STATIC"); + println!("cargo:rerun-if-env-changed=HADOOP_HOME"); + + if cfg!(feature = "vendored") { + return Ok(false); + } + + let lib_dir = if let Ok(lib_dir) = env::var("HDFS_LIB_DIR") { + lib_dir + } else if let Ok(hadoop_home) = env::var("HADOOP_HOME") { + format!("{hadoop_home}/lib/native") + } else { + return Ok(false); + }; + + let mode = if env::var_os("HDFS_STATIC").is_some() { + "static" + } else { + "dylib" + }; + println!("cargo:rustc-link-search=native={lib_dir}"); + println!("cargo:rustc-link-lib={mode}=hdfs"); + + Ok(true) +} + +/// Compiles the vendored Hadoop C into a static `libhdfs.a`. +/// +/// The file list follows `hadoop-hdfs-native-client/src/CMakeLists.txt` for the +/// 3.3 tree: the three top-level translation units, `jclasses.c` (added in 3.3) +/// and the POSIX platform layer. `htable.c` was removed in 3.3, and the Windows +/// platform layer is not vendored because Comet ships no Windows native builds. +fn build_vendored_libhdfs() -> Result<()> { + let java_home = java_locator::locate_java_home()?; + + println!("cargo:rustc-link-lib=static=hdfs"); + + let mut builder = cc::Build::new(); + // The vendored sources are Hadoop's, not ours, and are not warning-clean. + builder.warnings(false); + builder.flag_if_supported("-w"); + builder.flag_if_supported("-fvisibility=hidden"); + // Restore the pre-GCC-10 tentative-definition behaviour the sources assume. + builder.flag_if_supported("-fcommon"); + + builder.include(format!("{java_home}/include")); + if cfg!(target_os = "linux") { + builder.include(format!("{java_home}/include/linux")); + } + if cfg!(target_os = "macos") { + builder.include(format!("{java_home}/include/darwin")); + } + + builder + .include("libhdfs") + .include(SRC) + .include(format!("{SRC}/include")) + .include(format!("{SRC}/os")) + .include(format!("{SRC}/os/posix")); + + for file in [ + "exception.c", + "jni_helper.c", + "hdfs.c", + "jclasses.c", + "os/posix/mutexes.c", + "os/posix/thread.c", + "os/posix/thread_local_storage.c", + ] { + let path = format!("{SRC}/{file}"); + assert!(Path::new(&path).exists(), "missing vendored source {path}"); + println!("cargo:rerun-if-changed={path}"); + builder.file(path); + } + + builder.compile("hdfs"); + Ok(()) +} diff --git a/native/hdfs-sys/libhdfs/config.h b/native/hdfs-sys/libhdfs/config.h new file mode 100644 index 00000000000..5950534a16a --- /dev/null +++ b/native/hdfs-sys/libhdfs/config.h @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * In Apache Hadoop this header is generated by CMake and defines the feature + * probes libhdfs compiles against, chiefly HAVE_BETTER_TLS. jni_helper.c + * includes it unconditionally, so it has to exist. Leaving every probe undefined + * selects the portable pthread_getspecific path in os/thread_local_storage.h, + * which is what this crate wants and what the upstream hdfs-sys crate also + * shipped (as a zero-byte file). + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c new file mode 100644 index 00000000000..fec9a103b4e --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.c @@ -0,0 +1,272 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include + +#define EXCEPTION_INFO_LEN (sizeof(gExceptionInfo)/sizeof(gExceptionInfo[0])) + +struct ExceptionInfo { + const char * const name; + int noPrintFlag; + int excErrno; +}; + +static const struct ExceptionInfo gExceptionInfo[] = { + { + "java.io.FileNotFoundException", + NOPRINT_EXC_FILE_NOT_FOUND, + ENOENT, + }, + { + "org.apache.hadoop.security.AccessControlException", + NOPRINT_EXC_ACCESS_CONTROL, + EACCES, + }, + { + "org.apache.hadoop.fs.UnresolvedLinkException", + NOPRINT_EXC_UNRESOLVED_LINK, + ENOLINK, + }, + { + "org.apache.hadoop.fs.ParentNotDirectoryException", + NOPRINT_EXC_PARENT_NOT_DIRECTORY, + ENOTDIR, + }, + { + "java.lang.IllegalArgumentException", + NOPRINT_EXC_ILLEGAL_ARGUMENT, + EINVAL, + }, + { + "java.lang.OutOfMemoryError", + 0, + ENOMEM, + }, + { + "org.apache.hadoop.hdfs.server.namenode.SafeModeException", + 0, + EROFS, + }, + { + "org.apache.hadoop.fs.FileAlreadyExistsException", + 0, + EEXIST, + }, + { + "org.apache.hadoop.hdfs.protocol.QuotaExceededException", + 0, + EDQUOT, + }, + { + "java.lang.UnsupportedOperationException", + 0, + ENOTSUP, + }, + { + "org.apache.hadoop.hdfs.server.namenode.LeaseExpiredException", + 0, + ESTALE, + }, +}; + +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint) +{ + int i; + + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (strstr(gExceptionInfo[i].name, excName)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + *shouldPrint = !(gExceptionInfo[i].noPrintFlag & noPrintFlags); + *excErrno = gExceptionInfo[i].excErrno; + } else { + *shouldPrint = 1; + *excErrno = EINTERNAL; + } +} + +/** + * getExceptionUtilString: A helper function that calls 'methodName' in + * ExceptionUtils. The function 'methodName' should have a return type of a + * java String. + * + * @param env The JNI environment. + * @param exc The exception to get information for. + * @param methodName The method of ExceptionUtils to call that has a String + * return type. + * + * @return A C-type string containing the string returned by + * ExceptionUtils.'methodName', or NULL on failure. + */ +static char* getExceptionUtilString(JNIEnv *env, jthrowable exc, char *methodName) +{ + jthrowable jthr; + jvalue jVal; + jstring jStr = NULL; + char *excString = NULL; + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_EXCEPTION_UTILS, + methodName, "(Ljava/lang/Throwable;)Ljava/lang/String;", exc); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + jStr = jVal.l; + jthr = newCStr(env, jStr, &excString); + if (jthr) { + destroyLocalReference(env, jthr); + return NULL; + } + destroyLocalReference(env, jStr); + return excString; +} + +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap) +{ + int i, noPrint, excErrno; + char *className = NULL; + jthrowable jthr; + const char *stackTrace; + const char *rootCause; + + jthr = classNameOfObject(exc, env, &className); + if (jthr) { + fprintf(stderr, "PrintExceptionAndFree: error determining class name " + "of exception.\n"); + className = strdup("(unknown)"); + destroyLocalReference(env, jthr); + } + for (i = 0; i < EXCEPTION_INFO_LEN; i++) { + if (!strcmp(gExceptionInfo[i].name, className)) { + break; + } + } + if (i < EXCEPTION_INFO_LEN) { + noPrint = (gExceptionInfo[i].noPrintFlag & noPrintFlags); + excErrno = gExceptionInfo[i].excErrno; + } else { + noPrint = 0; + excErrno = EINTERNAL; + } + + // We don't want to use ExceptionDescribe here, because that requires a + // pending exception. Instead, use ExceptionUtils. + rootCause = getExceptionUtilString(env, exc, "getRootCauseMessage"); + stackTrace = getExceptionUtilString(env, exc, "getStackTrace"); + // Save the exception details in the thread-local state. + setTLSExceptionStrings(rootCause, stackTrace); + + if (!noPrint) { + vfprintf(stderr, fmt, ap); + fprintf(stderr, " error:\n"); + + if (!rootCause) { + fprintf(stderr, "(unable to get root cause for %s)\n", className); + } else { + fprintf(stderr, "%s", rootCause); + } + if (!stackTrace) { + fprintf(stderr, "(unable to get stack trace for %s)\n", className); + } else { + fprintf(stderr, "%s", stackTrace); + } + } + + destroyLocalReference(env, exc); + free(className); + return excErrno; +} + +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + return ret; +} + +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) +{ + va_list ap; + int ret; + jthrowable exc; + + exc = (*env)->ExceptionOccurred(env); + if (!exc) { + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, " error: (no exception)"); + ret = 0; + } else { + (*env)->ExceptionClear(env); + va_start(ap, fmt); + ret = printExceptionAndFreeV(env, exc, noPrintFlags, fmt, ap); + va_end(ap); + } + return ret; +} + +jthrowable getPendingExceptionAndClear(JNIEnv *env) +{ + jthrowable jthr = (*env)->ExceptionOccurred(env); + if (!jthr) + return NULL; + (*env)->ExceptionClear(env); + return jthr; +} + +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) +{ + char buf[512]; + jobject out, exc; + jstring jstr; + va_list ap; + + va_start(ap, fmt); + vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + jstr = (*env)->NewStringUTF(env, buf); + if (!jstr) { + // We got an out of memory exception rather than a RuntimeException. + // Too bad... + return getPendingExceptionAndClear(env); + } + exc = constructNewObjectOfClass(env, &out, "RuntimeException", + "(java/lang/String;)V", jstr); + (*env)->DeleteLocalRef(env, jstr); + // Again, we'll either get an out of memory exception or the + // RuntimeException we wanted. + return (exc) ? exc : out; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h new file mode 100644 index 00000000000..cdf93a16065 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/exception.h @@ -0,0 +1,165 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_EXCEPTION_H +#define LIBHDFS_EXCEPTION_H + +/** + * Exception handling routines for libhdfs. + * + * The convention we follow here is to clear pending exceptions as soon as they + * are raised. Never assume that the caller of your function will clean up + * after you-- do it yourself. Unhandled exceptions can lead to memory leaks + * and other undefined behavior. + * + * If you encounter an exception, return a local reference to it. The caller is + * responsible for freeing the local reference, by calling a function like + * printExceptionAndFree. (You can also free exceptions directly by calling + * DeleteLocalRef. However, that would not produce an error message, so it's + * usually not what you want.) + * + * The root cause and stack trace exception strings retrieved from the last + * exception that happened on a thread are stored in the corresponding + * thread local state and are accessed by hdfsGetLastExceptionRootCause and + * hdfsGetLastExceptionStackTrace respectively. + */ + +#include "platform.h" + +#include +#include + +#include +#include +#include +#include + +/** + * Exception noprint flags + * + * Theses flags determine which exceptions should NOT be printed to stderr by + * the exception printing routines. For example, if you expect to see + * FileNotFound, you might use NOPRINT_EXC_FILE_NOT_FOUND, to avoid filling the + * logs with messages about routine events. + * + * On the other hand, if you don't expect any failures, you might pass + * PRINT_EXC_ALL. + * + * You can OR these flags together to avoid printing multiple classes of + * exceptions. + */ +#define PRINT_EXC_ALL 0x00 +#define NOPRINT_EXC_FILE_NOT_FOUND 0x01 +#define NOPRINT_EXC_ACCESS_CONTROL 0x02 +#define NOPRINT_EXC_UNRESOLVED_LINK 0x04 +#define NOPRINT_EXC_PARENT_NOT_DIRECTORY 0x08 +#define NOPRINT_EXC_ILLEGAL_ARGUMENT 0x10 + +/** + * Get information about an exception. + * + * @param excName The Exception name. + * This is a Java class name in JNI format. + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param excErrno (out param) The POSIX error number associated with the + * exception. + * @param shouldPrint (out param) Nonzero if we should print this exception, + * based on the noPrintFlags and its name. + */ +void getExceptionInfo(const char *excName, int noPrintFlags, + int *excErrno, int *shouldPrint); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ap Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFreeV(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, va_list ap); + +/** + * Store the information about an exception in the thread-local state and print + * it and free the jthrowable object. + * + * @param env The JNI environment + * @param exc The exception to print and free + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printExceptionAndFree(JNIEnv *env, jthrowable exc, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(4, 5); + +/** + * Store the information about the pending exception in the thread-local state + * and print it and free the jthrowable object. + * + * @param env The JNI environment + * @param noPrintFlags Flags which determine which exceptions we should NOT + * print. + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return The POSIX error number associated with the exception + * object. + */ +int printPendingExceptionAndFree(JNIEnv *env, int noPrintFlags, + const char *fmt, ...) TYPE_CHECKED_PRINTF_FORMAT(3, 4); + +/** + * Get a local reference to the pending exception and clear it. + * + * Once it is cleared, the exception will no longer be pending. The caller will + * have to decide what to do with the exception object. + * + * @param env The JNI environment + * + * @return The exception, or NULL if there was no exception + */ +jthrowable getPendingExceptionAndClear(JNIEnv *env); + +/** + * Create a new runtime error. + * + * This creates (but does not throw) a new RuntimeError. + * + * @param env The JNI environment + * @param fmt Printf-style format list + * @param ... Printf-style varargs + * + * @return A local reference to a RuntimeError + */ +jthrowable newRuntimeError(JNIEnv *env, const char *fmt, ...) + TYPE_CHECKED_PRINTF_FORMAT(2, 3); + +#undef TYPE_CHECKED_PRINTF_FORMAT +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c b/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c new file mode 100644 index 00000000000..60f2826c741 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/hdfs.c @@ -0,0 +1,3831 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "hdfs/hdfs.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" + +#include +#include +#include +#include + +#define JAVA_VOID "V" + +/* Macros for constructing method signatures */ +#define JPARAM(X) "L" X ";" +#define JARRPARAM(X) "[L" X ";" +#define JMETHOD1(X, R) "(" X ")" R +#define JMETHOD2(X, Y, R) "(" X Y ")" R +#define JMETHOD3(X, Y, Z, R) "(" X Y Z")" R + +#define KERBEROS_TICKET_CACHE_PATH "hadoop.security.kerberos.ticket.cache.path" + +// Bit fields for hdfsFile_internal flags +#define HDFS_FILE_SUPPORTS_DIRECT_READ (1<<0) +#define HDFS_FILE_SUPPORTS_DIRECT_PREAD (1<<1) + +/** + * Reads bytes using the read(ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length); + +/** + * Reads bytes using the read(long, ByteBuffer) API. By using Java + * DirectByteBuffers we can avoid copying the bytes onto the Java heap. + * Instead the data will be directly copied from kernel space to the C heap. + */ +tSize preadDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +int preadFullyDirect(hdfsFS fs, hdfsFile file, tOffset position, void* buffer, + tSize length); + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo); + +/** + * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . + */ +enum hdfsStreamType +{ + HDFS_STREAM_UNINITIALIZED = 0, + HDFS_STREAM_INPUT = 1, + HDFS_STREAM_OUTPUT = 2, +}; + +/** + * The 'file-handle' to a file in hdfs. + */ +struct hdfsFile_internal { + void* file; + enum hdfsStreamType type; + int flags; +}; + +#define HDFS_EXTENDED_FILE_INFO_ENCRYPTED 0x1 + +/** + * Extended file information. + */ +struct hdfsExtendedFileInfo { + int flags; +}; + +int hdfsFileIsOpenForRead(hdfsFile file) +{ + return (file->type == HDFS_STREAM_INPUT); +} + +int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) +{ + jthrowable jthr; + jobject hedgedReadMetrics = NULL; + jvalue jVal; + struct hdfsHedgedReadMetrics *m = NULL; + int ret; + jobject jFS = (jobject)fs; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "getHedgedReadMetrics", + "()Lorg/apache/hadoop/hdfs/DFSHedgedReadMetrics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadMetrics: getHedgedReadMetrics failed"); + goto done; + } + hedgedReadMetrics = jVal.l; + + m = malloc(sizeof(struct hdfsHedgedReadMetrics)); + if (!m) { + ret = ENOMEM; + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOps", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOps failed"); + goto done; + } + m->hedgedReadOps = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadWins", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadWins failed"); + goto done; + } + m->hedgedReadOpsWin = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, hedgedReadMetrics, + JC_DFS_HEDGED_READ_METRICS, "getHedgedReadOpsInCurThread", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHedgedReadStatistics: getHedgedReadOpsInCurThread failed"); + goto done; + } + m->hedgedReadOpsInCurThread = jVal.j; + + *metrics = m; + m = NULL; + ret = 0; + +done: + destroyLocalReference(env, hedgedReadMetrics); + free(m); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) +{ + free(metrics); +} + +int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats) +{ + jthrowable jthr; + jobject readStats = NULL; + jvalue jVal; + struct hdfsReadStatistics *s = NULL; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "getReadStatistics", + "()Lorg/apache/hadoop/hdfs/ReadStatistics;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getReadStatistics failed"); + goto done; + } + readStats = jVal.l; + s = malloc(sizeof(struct hdfsReadStatistics)); + if (!s) { + ret = ENOMEM; + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalBytesRead failed"); + goto done; + } + s->totalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalLocalBytesRead", "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalLocalBytesRead failed"); + goto done; + } + s->totalLocalBytesRead = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalShortCircuitBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalShortCircuitBytesRead failed"); + goto done; + } + s->totalShortCircuitBytesRead = jVal.j; + jthr = invokeMethod(env, &jVal, INSTANCE, readStats, + JC_READ_STATISTICS, "getTotalZeroCopyBytesRead", + "()J"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileGetReadStatistics: getTotalZeroCopyBytesRead failed"); + goto done; + } + s->totalZeroCopyBytesRead = jVal.j; + *stats = s; + s = NULL; + ret = 0; + +done: + destroyLocalReference(env, readStats); + free(s); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats) +{ + return stats->totalBytesRead - stats->totalLocalBytesRead; +} + +int hdfsFileClearReadStatistics(hdfsFile file) +{ + jthrowable jthr; + int ret; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return EINTERNAL; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = EINVAL; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_HDFS_DATA_INPUT_STREAM, "clearReadStatistics", + "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFileClearReadStatistics: clearReadStatistics failed"); + goto done; + } + ret = 0; +done: + if (ret) { + errno = ret; + return ret; + } + return 0; +} + +void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats) +{ + free(stats); +} + +int hdfsFileIsOpenForWrite(hdfsFile file) +{ + return (file->type == HDFS_STREAM_OUTPUT); +} + +int hdfsFileUsesDirectRead(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) != 0; +} + +void hdfsFileDisableDirectRead(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_READ; +} + +int hdfsFileUsesDirectPread(hdfsFile file) +{ + return (file->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) != 0; +} + +void hdfsFileDisableDirectPread(hdfsFile file) +{ + file->flags &= ~HDFS_FILE_SUPPORTS_DIRECT_PREAD; +} + + +int hdfsDisableDomainSocketSecurity(void) +{ + jthrowable jthr; + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = invokeMethod(env, NULL, STATIC, NULL, JC_DOMAIN_SOCKET, + "disableBindPathValidation", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "DomainSocket#disableBindPathValidation"); + return -1; + } + return 0; +} + +/** + * hdfsJniEnv: A wrapper struct to be used as 'value' + * while saving thread -> JNIEnv* mappings + */ +typedef struct +{ + JNIEnv* env; +} hdfsJniEnv; + +/** + * Helper function to create a org.apache.hadoop.fs.Path object. + * @param env: The JNIEnv pointer. + * @param path: The file-path for which to construct org.apache.hadoop.fs.Path + * object. + * @return Returns a jobject on success and NULL on error. + */ +static jthrowable constructNewObjectOfPath(JNIEnv *env, const char *path, + jobject *out) +{ + jthrowable jthr; + jstring jPathString; + jobject jPath; + + //Construct a java.lang.String object + jthr = newJavaStr(env, path, &jPathString); + if (jthr) + return jthr; + //Construct the org.apache.hadoop.fs.Path object + jthr = constructNewObjectOfCachedClass(env, &jPath, JC_PATH, + "(Ljava/lang/String;)V", jPathString); + destroyLocalReference(env, jPathString); + if (jthr) + return jthr; + *out = jPath; + return NULL; +} + +static jthrowable hadoopConfGetStr(JNIEnv *env, jobject jConfiguration, + const char *key, char **val) +{ + jthrowable jthr; + jvalue jVal; + jstring jkey = NULL, jRet = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "get", JMETHOD1(JPARAM(JAVA_STRING), + JPARAM(JAVA_STRING)), jkey); + if (jthr) + goto done; + jRet = jVal.l; + jthr = newCStr(env, jRet, val); +done: + destroyLocalReference(env, jkey); + destroyLocalReference(env, jRet); + return jthr; +} + +int hdfsConfGetStr(const char *key, char **val) +{ + JNIEnv *env; + int ret; + jthrowable jthr; + jobject jConfiguration = NULL; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetStr(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetStr(%s): hadoopConfGetStr", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +void hdfsConfStrFree(char *val) +{ + free(val); +} + +static jthrowable hadoopConfGetInt(JNIEnv *env, jobject jConfiguration, + const char *key, int32_t *val) +{ + jthrowable jthr = NULL; + jvalue jVal; + jstring jkey = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + return jthr; + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + JMETHOD2(JPARAM(JAVA_STRING), "I", "I"), jkey, (jint)(*val)); + destroyLocalReference(env, jkey); + if (jthr) + return jthr; + *val = jVal.i; + return NULL; +} + +int hdfsConfGetInt(const char *key, int32_t *val) +{ + JNIEnv *env; + int ret; + jobject jConfiguration = NULL; + jthrowable jthr; + + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): new Configuration", key); + goto done; + } + jthr = hadoopConfGetInt(env, jConfiguration, key, val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsConfGetInt(%s): hadoopConfGetInt", key); + goto done; + } + ret = 0; +done: + destroyLocalReference(env, jConfiguration); + if (ret) + errno = ret; + return ret; +} + +struct hdfsBuilderConfOpt { + struct hdfsBuilderConfOpt *next; + const char *key; + const char *val; +}; + +struct hdfsBuilder { + int forceNewInstance; + const char *nn; + tPort port; + const char *kerbTicketCachePath; + const char *userName; + struct hdfsBuilderConfOpt *opts; +}; + +struct hdfsBuilder *hdfsNewBuilder(void) +{ + struct hdfsBuilder *bld = calloc(1, sizeof(struct hdfsBuilder)); + if (!bld) { + errno = ENOMEM; + return NULL; + } + return bld; +} + +int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val) +{ + struct hdfsBuilderConfOpt *opt, *next; + + opt = calloc(1, sizeof(struct hdfsBuilderConfOpt)); + if (!opt) + return -ENOMEM; + next = bld->opts; + bld->opts = opt; + opt->next = next; + opt->key = key; + opt->val = val; + return 0; +} + +void hdfsFreeBuilder(struct hdfsBuilder *bld) +{ + struct hdfsBuilderConfOpt *cur, *next; + + cur = bld->opts; + for (cur = bld->opts; cur; ) { + next = cur->next; + free(cur); + cur = next; + } + free(bld); +} + +void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld) +{ + bld->forceNewInstance = 1; +} + +void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn) +{ + bld->nn = nn; +} + +void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port) +{ + bld->port = port; +} + +void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName) +{ + bld->userName = userName; +} + +void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath) +{ + bld->kerbTicketCachePath = kerbTicketCachePath; +} + +hdfsFS hdfsConnect(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectNewInstance(const char *host, tPort port) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + return hdfsBuilderConnect(bld); +} + +hdfsFS hdfsConnectAsUser(const char *host, tPort port, const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + +/** Always return a new FileSystem handle */ +hdfsFS hdfsConnectAsUserNewInstance(const char *host, tPort port, + const char *user) +{ + struct hdfsBuilder *bld = hdfsNewBuilder(); + if (!bld) + return NULL; + hdfsBuilderSetNameNode(bld, host); + hdfsBuilderSetNameNodePort(bld, port); + hdfsBuilderSetForceNewInstance(bld); + hdfsBuilderSetUserName(bld, user); + return hdfsBuilderConnect(bld); +} + + +/** + * Calculate the effective URI to use, given a builder configuration. + * + * If there is not already a URI scheme, we prepend 'hdfs://'. + * + * If there is not already a port specified, and a port was given to the + * builder, we suffix that port. If there is a port specified but also one in + * the URI, that is an error. + * + * @param bld The hdfs builder object + * @param uri (out param) dynamically allocated string representing the + * effective URI + * + * @return 0 on success; error code otherwise + */ +static int calcEffectiveURI(struct hdfsBuilder *bld, char ** uri) +{ + const char *scheme; + char suffix[64]; + const char *lastColon; + char *u; + size_t uriLen; + + if (!bld->nn) + return EINVAL; + scheme = (strstr(bld->nn, "://")) ? "" : "hdfs://"; + if (bld->port == 0) { + suffix[0] = '\0'; + } else { + lastColon = strrchr(bld->nn, ':'); + if (lastColon && (strspn(lastColon + 1, "0123456789") == + strlen(lastColon + 1))) { + fprintf(stderr, "port %d was given, but URI '%s' already " + "contains a port!\n", bld->port, bld->nn); + return EINVAL; + } + snprintf(suffix, sizeof(suffix), ":%d", bld->port); + } + + uriLen = strlen(scheme) + strlen(bld->nn) + strlen(suffix); + u = malloc((uriLen + 1) * (sizeof(char))); + if (!u) { + fprintf(stderr, "calcEffectiveURI: out of memory"); + return ENOMEM; + } + snprintf(u, uriLen + 1, "%s%s%s", scheme, bld->nn, suffix); + *uri = u; + return 0; +} + +static const char *maybeNull(const char *str) +{ + return str ? str : "(NULL)"; +} + +static const char *hdfsBuilderToStr(const struct hdfsBuilder *bld, + char *buf, size_t bufLen) +{ + snprintf(buf, bufLen, "forceNewInstance=%d, nn=%s, port=%d, " + "kerbTicketCachePath=%s, userName=%s", + bld->forceNewInstance, maybeNull(bld->nn), bld->port, + maybeNull(bld->kerbTicketCachePath), maybeNull(bld->userName)); + return buf; +} + +hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld) +{ + JNIEnv *env = 0; + jobject jConfiguration = NULL, jFS = NULL, jURI = NULL, jCachePath = NULL; + jstring jURIString = NULL, jUserString = NULL; + jvalue jVal; + jthrowable jthr = NULL; + char *cURI = 0, buf[512]; + int ret; + jobject jRet = NULL; + struct hdfsBuilderConfOpt *opt; + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + ret = EINTERNAL; + goto done; + } + + // jConfiguration = new Configuration(); + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + // set configuration values + for (opt = bld->opts; opt; opt = opt->next) { + jthr = hadoopConfSetStr(env, jConfiguration, opt->key, opt->val); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s): error setting conf '%s' to '%s'", + hdfsBuilderToStr(bld, buf, sizeof(buf)), opt->key, opt->val); + goto done; + } + } + + //Check what type of FileSystem the caller wants... + if (bld->nn == NULL) { + // Get a local filesystem. + if (bld->forceNewInstance) { + // fs = FileSytem#newInstanceLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstanceLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + // fs = FileSytem#getLocal(conf); + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getLocal", + JMETHOD1(JPARAM(HADOOP_CONF), JPARAM(HADOOP_LOCALFS)), + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } else { + if (!strcmp(bld->nn, "default")) { + // jURI = FileSystem.getDefaultUri(conf) + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "getDefaultUri", + "(Lorg/apache/hadoop/conf/Configuration;)Ljava/net/URI;", + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } else { + // fs = FileSystem#get(URI, conf, ugi); + ret = calcEffectiveURI(bld, &cURI); + if (ret) + goto done; + jthr = newJavaStr(env, cURI, &jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_URI, "create", + "(Ljava/lang/String;)Ljava/net/URI;", jURIString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jURI = jVal.l; + } + + if (bld->kerbTicketCachePath) { + jthr = hadoopConfSetStr(env, jConfiguration, + KERBEROS_TICKET_CACHE_PATH, bld->kerbTicketCachePath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + } + jthr = newJavaStr(env, bld->userName, &jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + if (bld->forceNewInstance) { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "newInstance", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } else { + jthr = invokeMethod(env, &jVal, STATIC, NULL, + JC_FILE_SYSTEM, "get", + JMETHOD3(JPARAM(JAVA_NET_URI), JPARAM(HADOOP_CONF), + JPARAM(JAVA_STRING), JPARAM(HADOOP_FS)), jURI, + jConfiguration, jUserString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + jFS = jVal.l; + } + } + jRet = (*env)->NewGlobalRef(env, jFS); + if (!jRet) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsBuilderConnect(%s)", + hdfsBuilderToStr(bld, buf, sizeof(buf))); + goto done; + } + ret = 0; + +done: + // Release unnecessary local references + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jFS); + destroyLocalReference(env, jURI); + destroyLocalReference(env, jCachePath); + destroyLocalReference(env, jURIString); + destroyLocalReference(env, jUserString); + free(cURI); + hdfsFreeBuilder(bld); + + if (ret) { + errno = ret; + return NULL; + } + return (hdfsFS)jRet; +} + +int hdfsDisconnect(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.close() + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + int ret; + jobject jFS; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + jFS = (jobject)fs; + + //Sanity check + if (fs == NULL) { + errno = EBADF; + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "close", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDisconnect: FileSystem#close"); + } else { + ret = 0; + } + (*env)->DeleteGlobalRef(env, jFS); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +/** + * Get the default block size of a FileSystem object. + * + * @param env The Java env + * @param jFS The FileSystem object + * @param jPath The path to find the default blocksize at + * @param out (out param) the default block size + * + * @return NULL on success; or the exception + */ +static jthrowable getDefaultBlockSize(JNIEnv *env, jobject jFS, + jobject jPath, jlong *out) +{ + jthrowable jthr; + jvalue jVal; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", JMETHOD1(JPARAM(HADOOP_PATH), + "J"), jPath); + if (jthr) + return jthr; + *out = jVal.j; + return NULL; +} + +hdfsFile hdfsOpenFile(hdfsFS fs, const char *path, int flags, + int bufferSize, short replication, tSize blockSize) +{ + struct hdfsStreamBuilder *bld = hdfsStreamBuilderAlloc(fs, path, flags); + if (bufferSize != 0) { + hdfsStreamBuilderSetBufferSize(bld, bufferSize); + } + if (replication != 0) { + hdfsStreamBuilderSetReplication(bld, replication); + } + if (blockSize != 0) { + hdfsStreamBuilderSetDefaultBlockSize(bld, blockSize); + } + return hdfsStreamBuilderBuild(bld); +} + +struct hdfsStreamBuilder { + hdfsFS fs; + int flags; + int32_t bufferSize; + int16_t replication; + int64_t defaultBlockSize; + char path[1]; +}; + +struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags) +{ + size_t path_len = strlen(path); + struct hdfsStreamBuilder *bld; + + // Check for overflow in path_len + if (path_len > SIZE_MAX - sizeof(struct hdfsStreamBuilder)) { + errno = EOVERFLOW; + return NULL; + } + // sizeof(hdfsStreamBuilder->path) includes one byte for the string + // terminator + bld = malloc(sizeof(struct hdfsStreamBuilder) + path_len); + if (!bld) { + errno = ENOMEM; + return NULL; + } + bld->fs = fs; + bld->flags = flags; + bld->bufferSize = 0; + bld->replication = 0; + bld->defaultBlockSize = 0; + memcpy(bld->path, path, path_len); + bld->path[path_len] = '\0'; + return bld; +} + +void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld) +{ + free(bld); +} + +int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->bufferSize = bufferSize; + return 0; +} + +int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->replication = replication; + return 0; +} + +int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize) +{ + if ((bld->flags & O_ACCMODE) != O_WRONLY) { + errno = EINVAL; + return -1; + } + bld->defaultBlockSize = defaultBlockSize; + return 0; +} + +/** + * Delegates to FsDataInputStream#hasCapability(String). Used to check if a + * given input stream supports certain methods, such as + * ByteBufferReadable#read(ByteBuffer). + * + * @param jFile the FsDataInputStream to call hasCapability on + * @param capability the name of the capability to query; for a full list of + * possible values see StreamCapabilities + * + * @return true if the given jFile has the given capability, false otherwise + * + * @see org.apache.hadoop.fs.StreamCapabilities + */ +static int hdfsHasStreamCapability(jobject jFile, + const char *capability) { + int ret = 0; + jthrowable jthr = NULL; + jvalue jVal; + jstring jCapabilityString = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return 0; + } + + jthr = newJavaStr(env, capability, &jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): newJavaStr", capability); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFile, + JC_FS_DATA_INPUT_STREAM, "hasCapability", "(Ljava/lang/String;)Z", + jCapabilityString); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHasStreamCapability(%s): FSDataInputStream#hasCapability", + capability); + goto done; + } + +done: + destroyLocalReference(env, jthr); + destroyLocalReference(env, jCapabilityString); + if (ret) { + errno = ret; + return 0; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +static hdfsFile hdfsOpenFileImpl(hdfsFS fs, const char *path, int flags, + int32_t bufferSize, int16_t replication, int64_t blockSize) +{ + /* + JAVA EQUIVALENT: + File f = new File(path); + FSData{Input|Output}Stream f{is|os} = fs.create(f); + return f{is|os}; + */ + int accmode = flags & O_ACCMODE; + jstring jStrBufferSize = NULL, jStrReplication = NULL; + jobject jConfiguration = NULL, jPath = NULL, jFile = NULL; + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + hdfsFile file = NULL; + int ret; + jint jBufferSize = bufferSize; + jshort jReplication = replication; + + /* The hadoop java api/signature */ + const char *method = NULL; + const char *signature = NULL; + + /* Get the JNIEnv* corresponding to current thread */ + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + + if (accmode == O_RDONLY || accmode == O_WRONLY) { + /* yay */ + } else if (accmode == O_RDWR) { + fprintf(stderr, "ERROR: cannot open an hdfs file in O_RDWR mode\n"); + errno = ENOTSUP; + return NULL; + } else { + fprintf(stderr, "ERROR: cannot open an hdfs file in mode 0x%x\n", + accmode); + errno = EINVAL; + return NULL; + } + + if ((flags & O_CREAT) && (flags & O_EXCL)) { + fprintf(stderr, + "WARN: hdfs does not truly support O_CREATE && O_EXCL\n"); + } + + if (accmode == O_RDONLY) { + method = "open"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "I", JPARAM(HADOOP_FSDISTRM)); + } else if (flags & O_APPEND) { + method = "append"; + signature = JMETHOD1(JPARAM(HADOOP_PATH), JPARAM(HADOOP_FSDOSTRM)); + } else { + method = "create"; + signature = JMETHOD2(JPARAM(HADOOP_PATH), "ZISJ", JPARAM(HADOOP_FSDOSTRM)); + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): constructNewObjectOfPath", path); + goto done; + } + + /* Get the Configuration object from the FileSystem object */ + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getConf", JMETHOD1("", JPARAM(HADOOP_CONF))); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#getConf", path); + goto done; + } + jConfiguration = jVal.l; + + jStrBufferSize = (*env)->NewStringUTF(env, "io.file.buffer.size"); + if (!jStrBufferSize) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + jStrReplication = (*env)->NewStringUTF(env, "dfs.replication"); + if (!jStrReplication) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, "OOM"); + goto done; + } + + if (!bufferSize) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrBufferSize, 4096); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsOpenFile(%s): Configuration#getInt(io.file.buffer.size)", + path); + goto done; + } + jBufferSize = jVal.i; + } + + if ((accmode == O_WRONLY) && (flags & O_APPEND) == 0) { + if (!replication) { + jthr = invokeMethod(env, &jVal, INSTANCE, jConfiguration, + JC_CONFIGURATION, "getInt", + "(Ljava/lang/String;I)I", jStrReplication, 1); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): Configuration#getInt(dfs.replication)", + path); + goto done; + } + jReplication = (jshort)jVal.i; + } + } + + /* Create and return either the FSDataInputStream or + FSDataOutputStream references jobject jStream */ + + // READ? + if (accmode == O_RDONLY) { + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jBufferSize); + } else if ((accmode == O_WRONLY) && (flags & O_APPEND)) { + // WRITE/APPEND? + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath); + } else { + // WRITE/CREATE + jboolean jOverWrite = 1; + jlong jBlockSize = blockSize; + + if (jBlockSize == 0) { + jthr = getDefaultBlockSize(env, jFS, jPath, &jBlockSize); + if (jthr) { + ret = EIO; + goto done; + } + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + method, signature, jPath, jOverWrite, jBufferSize, + jReplication, jBlockSize); + } + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsOpenFile(%s): FileSystem#%s(%s)", path, method, signature); + goto done; + } + jFile = jVal.l; + + file = calloc(1, sizeof(struct hdfsFile_internal)); + if (!file) { + fprintf(stderr, "hdfsOpenFile(%s): OOM create hdfsFile\n", path); + ret = ENOMEM; + goto done; + } + file->file = (*env)->NewGlobalRef(env, jFile); + if (!file->file) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsOpenFile(%s): NewGlobalRef", path); + goto done; + } + file->type = (((flags & O_WRONLY) == 0) ? HDFS_STREAM_INPUT : + HDFS_STREAM_OUTPUT); + file->flags = 0; + + if ((flags & O_WRONLY) == 0) { + // Check the StreamCapabilities of jFile to see if we can do direct + // reads + if (hdfsHasStreamCapability(jFile, "in:readbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_READ; + } + + // Check the StreamCapabilities of jFile to see if we can do direct + // preads + if (hdfsHasStreamCapability(jFile, "in:preadbytebuffer")) { + file->flags |= HDFS_FILE_SUPPORTS_DIRECT_PREAD; + } + } + ret = 0; + +done: + destroyLocalReference(env, jStrBufferSize); + destroyLocalReference(env, jStrReplication); + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFile); + if (ret) { + if (file) { + if (file->file) { + (*env)->DeleteGlobalRef(env, file->file); + } + free(file); + } + errno = ret; + return NULL; + } + return file; +} + +hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld) +{ + hdfsFile file = hdfsOpenFileImpl(bld->fs, bld->path, bld->flags, + bld->bufferSize, bld->replication, bld->defaultBlockSize); + int prevErrno = errno; + hdfsStreamBuilderFree(bld); + errno = prevErrno; + return file; +} + +int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength) +{ + jobject jFS = (jobject)fs; + jthrowable jthr; + jvalue jVal; + jobject jPath = NULL; + + JNIEnv *env = getJNIEnv(); + + if (!env) { + errno = EINTERNAL; + return -1; + } + + /* Create an object of org.apache.hadoop.fs.Path */ + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): constructNewObjectOfPath", path); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "truncate", JMETHOD2(JPARAM(HADOOP_PATH), "J", "Z"), + jPath, newlength); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTruncateFile(%s): FileSystem#truncate", path); + return -1; + } + if (jVal.z == JNI_TRUE) { + return 1; + } + return 0; +} + +int hdfsUnbufferFile(hdfsFile file) +{ + int ret; + jthrowable jthr; + JNIEnv *env = getJNIEnv(); + + if (!env) { + ret = EINTERNAL; + goto done; + } + if (file->type != HDFS_STREAM_INPUT) { + ret = ENOTSUP; + goto done; + } + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "unbuffer", "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + HADOOP_FSDISTRM "#unbuffer failed:"); + goto done; + } + ret = 0; + +done: + errno = ret; + return ret; +} + +int hdfsCloseFile(hdfsFS fs, hdfsFile file) +{ + int ret; + // JAVA EQUIVALENT: + // file.close + + //The interface whose 'close' method to be called + CachedJavaClass cachedJavaClass; + const char *interfaceShortName; + + //Caught exception + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!file || file->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (file->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + + jthr = invokeMethod(env, NULL, INSTANCE, file->file, + cachedJavaClass, "close", "()V"); + if (jthr) { + interfaceShortName = (file->type == HDFS_STREAM_INPUT) ? + "FSDataInputStream" : "FSDataOutputStream"; + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "%s#close", interfaceShortName); + } else { + ret = 0; + } + + //De-allocate memory + (*env)->DeleteGlobalRef(env, file->file); + free(file); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsExists(hdfsFS fs, const char *path) +{ + JNIEnv *env = getJNIEnv(); + jobject jPath; + jvalue jVal; + jobject jFS = (jobject)fs; + jthrowable jthr; + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (path == NULL) { + errno = EINVAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: constructNewObjectOfPath"); + return -1; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "exists", JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsExists: invokeMethod(%s)", + JMETHOD1(JPARAM(HADOOP_PATH), "Z")); + return -1; + } + if (jVal.z) { + return 0; + } else { + errno = ENOENT; + return -1; + } +} + +// Checks input file for readiness for reading. +static int readPrepare(JNIEnv* env, hdfsFS fs, hdfsFile f, + jobject* jInputStream) +{ + *jInputStream = (jobject)(f ? f->file : NULL); + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + return 0; +} + +/** + * If the underlying stream supports the ByteBufferReadable interface then + * this method will transparently use read(ByteBuffer). This can help + * improve performance as it avoids unnecessarily copying data on to the Java + * heap. Instead the data will be directly copied from kernel space to the C + * heap. + */ +tSize hdfsRead(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + jobject jInputStream; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_READ) { + return readDirect(fs, f, buffer, length); + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(bR); + + //Get the JNIEnv* corresponding to current thread + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Parameters + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", "([B)I", jbRarray); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRead: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + // We only copy the portion of the jbRarray that was actually filled by + // the call to FsDataInputStream#read; #read is not guaranteed to fill the + // entire buffer, instead it returns the number of bytes read into the + // buffer; we use the return value as the input in GetByteArrayRegion to + // ensure don't copy more bytes than necessary + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsRead: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize readDirect(hdfsFS fs, hdfsFile f, void* buffer, tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(buf); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (readPrepare(env, fs, f, &jInputStream) == -1) { + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "read", + "(Ljava/nio/ByteBuffer;)I", bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "readDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * If the underlying stream supports the ByteBufferPositionedReadable + * interface then this method will transparently use read(long, ByteBuffer). + * This can help improve performance as it avoids unnecessarily copying data + * on to the Java heap. Instead the data will be directly copied from kernel + * space to the C heap. + */ +tSize hdfsPread(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) +{ + JNIEnv* env; + jbyteArray jbRarray; + jvalue jVal; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(J[BII)I", position, + jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + if (jVal.i < 0) { + // EOF + destroyLocalReference(env, jbRarray); + return 0; + } else if (jVal.i == 0) { + destroyLocalReference(env, jbRarray); + errno = EINTR; + return -1; + } + (*env)->GetByteArrayRegion(env, jbRarray, 0, jVal.i, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return jVal.i; +} + +tSize preadDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jvalue jVal; + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "read", "(JLjava/nio/ByteBuffer;)I", + position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + // Reached EOF, return 0 + if (jVal.i < 0) { + return 0; + } + // 0 bytes read, return error + if (jVal.i == 0) { + errno = EINTR; + return -1; + } + return jVal.i; +} + +/** + * Like hdfsPread, if the underlying stream supports the + * ByteBufferPositionedReadable interface then this method will transparently + * use readFully(long, ByteBuffer). + */ +int hdfsPreadFully(hdfsFS fs, hdfsFile f, tOffset position, + void* buffer, tSize length) { + JNIEnv* env; + jbyteArray jbRarray; + jthrowable jthr; + + if (length == 0) { + return 0; + } else if (length < 0) { + errno = EINVAL; + return -1; + } + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + if (f->flags & HDFS_FILE_SUPPORTS_DIRECT_PREAD) { + return preadFullyDirect(fs, f, position, buffer, length); + } + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + // JAVA EQUIVALENT: + // byte [] bR = new byte[length]; + // fis.read(pos, bR, 0, length); + jbRarray = (*env)->NewByteArray(env, length); + if (!jbRarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: NewByteArray"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", "(J[BII)V", + position, jbRarray, 0, length); + if (jthr) { + destroyLocalReference(env, jbRarray); + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsPread: FSDataInputStream#read"); + return -1; + } + + (*env)->GetByteArrayRegion(env, jbRarray, 0, length, buffer); + destroyLocalReference(env, jbRarray); + if ((*env)->ExceptionCheck(env)) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsPread: GetByteArrayRegion"); + return -1; + } + return 0; +} + +int preadFullyDirect(hdfsFS fs, hdfsFile f, tOffset position, void* buffer, + tSize length) +{ + // JAVA EQUIVALENT: + // ByteBuffer buf = ByteBuffer.allocateDirect(length) // wraps C buffer + // fis.read(position, buf); + + jthrowable jthr; + jobject bb; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Error checking... make sure that this file is 'readable' + if (f->type != HDFS_STREAM_INPUT) { + fprintf(stderr, "Cannot read from a non-InputStream object!\n"); + errno = EINVAL; + return -1; + } + + //Read the requisite bytes + bb = (*env)->NewDirectByteBuffer(env, buffer, length); + if (bb == NULL) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "readDirect: NewDirectByteBuffer"); + return -1; + } + + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_INPUT_STREAM, "readFully", + "(JLjava/nio/ByteBuffer;)V", position, bb); + destroyLocalReference(env, bb); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "preadDirect: FSDataInputStream#read"); + return -1; + } + return 0; +} + +tSize hdfsWrite(hdfsFS fs, hdfsFile f, const void* buffer, tSize length) +{ + // JAVA EQUIVALENT + // byte b[] = str.getBytes(); + // fso.write(b); + + jobject jOutputStream; + jbyteArray jbWarray; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + + if (length < 0) { + errno = EINVAL; + return -1; + } + + //Error checking... make sure that this file is 'writable' + if (f->type != HDFS_STREAM_OUTPUT) { + fprintf(stderr, "Cannot write into a non-OutputStream object!\n"); + errno = EINVAL; + return -1; + } + + if (length < 0) { + errno = EINVAL; + return -1; + } + if (length == 0) { + return 0; + } + //Write the requisite bytes into the file + jbWarray = (*env)->NewByteArray(env, length); + if (!jbWarray) { + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite: NewByteArray"); + return -1; + } + (*env)->SetByteArrayRegion(env, jbWarray, 0, length, buffer); + if ((*env)->ExceptionCheck(env)) { + destroyLocalReference(env, jbWarray); + errno = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsWrite(length = %d): SetByteArrayRegion", length); + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "write", "([B)V", + jbWarray); + destroyLocalReference(env, jbWarray); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsWrite: FSDataOutputStream#write"); + return -1; + } + // Unlike most Java streams, FSDataOutputStream never does partial writes. + // If we succeeded, all the data was written. + return length; +} + +int hdfsSeek(hdfsFS fs, hdfsFile f, tOffset desiredPos) +{ + // JAVA EQUIVALENT + // fis.seek(pos); + + jobject jInputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + jInputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "seek", "(J)V", desiredPos); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSeek(desiredPos=%" PRId64 ")" + ": FSDataInputStream#seek", desiredPos); + return -1; + } + return 0; +} + +tOffset hdfsTell(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // pos = f.getPos(); + + jobject jStream; + CachedJavaClass cachedJavaClass; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type == HDFS_STREAM_UNINITIALIZED) { + errno = EBADF; + return -1; + } + + //Parameters + jStream = f->file; + if (f->type == HDFS_STREAM_INPUT) { + cachedJavaClass = JC_FS_DATA_INPUT_STREAM; + } else { + cachedJavaClass = JC_FS_DATA_OUTPUT_STREAM; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStream, + cachedJavaClass, "getPos", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsTell: %s#getPos", + ((f->type == HDFS_STREAM_INPUT) ? "FSDataInputStream" : + "FSDataOutputStream")); + return -1; + } + return jVal.j; +} + +int hdfsFlush(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fos.flush(); + + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + jthr = invokeMethod(env, NULL, INSTANCE, f->file, + JC_FS_DATA_OUTPUT_STREAM, "flush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsFlush: FSDataInputStream#flush"); + return -1; + } + return 0; +} + +int hdfsHFlush(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hflush", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHFlush: FSDataOutputStream#hflush"); + return -1; + } + return 0; +} + +int hdfsHSync(hdfsFS fs, hdfsFile f) +{ + jobject jOutputStream; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_OUTPUT) { + errno = EBADF; + return -1; + } + + jOutputStream = f->file; + jthr = invokeMethod(env, NULL, INSTANCE, jOutputStream, + JC_FS_DATA_OUTPUT_STREAM, "hsync", "()V"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsHSync: FSDataOutputStream#hsync"); + return -1; + } + return 0; +} + +int hdfsAvailable(hdfsFS fs, hdfsFile f) +{ + // JAVA EQUIVALENT + // fis.available(); + + jobject jInputStream; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Sanity check + if (!f || f->type != HDFS_STREAM_INPUT) { + errno = EBADF; + return -1; + } + + //Parameters + jInputStream = f->file; + jthr = invokeMethod(env, &jVal, INSTANCE, jInputStream, + JC_FS_DATA_INPUT_STREAM, "available", "()I"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsAvailable: FSDataInputStream#available"); + return -1; + } + return jVal.i; +} + +static int hdfsCopyImpl(hdfsFS srcFS, const char *src, hdfsFS dstFS, + const char *dst, jboolean deleteSource) +{ + //JAVA EQUIVALENT + // FileUtil#copy(srcFS, srcPath, dstFS, dstPath, + // deleteSource = false, conf) + + //Parameters + jobject jSrcFS = (jobject)srcFS; + jobject jDstFS = (jobject)dstFS; + jobject jConfiguration = NULL, jSrcPath = NULL, jDstPath = NULL; + jthrowable jthr; + jvalue jVal; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, src, &jSrcPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s): constructNewObjectOfPath", src); + goto done; + } + jthr = constructNewObjectOfPath(env, dst, &jDstPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(dst=%s): constructNewObjectOfPath", dst); + goto done; + } + + //Create the org.apache.hadoop.conf.Configuration object + jthr = constructNewObjectOfCachedClass(env, &jConfiguration, + JC_CONFIGURATION, "()V"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl: Configuration constructor"); + goto done; + } + + //FileUtil#copy + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_FILE_UTIL, + "copy", + "(Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "Lorg/apache/hadoop/fs/FileSystem;Lorg/apache/hadoop/fs/Path;" + "ZLorg/apache/hadoop/conf/Configuration;)Z", + jSrcFS, jSrcPath, jDstFS, jDstPath, deleteSource, + jConfiguration); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCopyImpl(src=%s, dst=%s, deleteSource=%d): " + "FileUtil#copy", src, dst, deleteSource); + goto done; + } + if (!jVal.z) { + ret = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jConfiguration); + destroyLocalReference(env, jSrcPath); + destroyLocalReference(env, jDstPath); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsCopy(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 0); +} + +int hdfsMove(hdfsFS srcFS, const char *src, hdfsFS dstFS, const char *dst) +{ + return hdfsCopyImpl(srcFS, src, dstFS, dst, 1); +} + +int hdfsDelete(hdfsFS fs, const char *path, int recursive) +{ + // JAVA EQUIVALENT: + // Path p = new Path(path); + // bool retval = fs.delete(p, recursive); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + jboolean jRecursive; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s): constructNewObjectOfPath", path); + return -1; + } + jRecursive = recursive ? JNI_TRUE : JNI_FALSE; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "delete", "(Lorg/apache/hadoop/fs/Path;Z)Z", jPath, + jRecursive); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsDelete(path=%s, recursive=%d): " + "FileSystem#delete", path, recursive); + return -1; + } + if (!jVal.z) { + errno = EIO; + return -1; + } + return 0; +} + + + +int hdfsRename(hdfsFS fs, const char *oldPath, const char *newPath) +{ + // JAVA EQUIVALENT: + // Path old = new Path(oldPath); + // Path new = new Path(newPath); + // fs.rename(old, new); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jOldPath = NULL, jNewPath = NULL; + int ret = -1; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + jthr = constructNewObjectOfPath(env, oldPath, &jOldPath ); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", oldPath); + goto done; + } + jthr = constructNewObjectOfPath(env, newPath, &jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename: constructNewObjectOfPath(%s)", newPath); + goto done; + } + + // Rename the file + // TODO: use rename2 here? (See HDFS-3592) + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "rename", JMETHOD2(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_PATH), "Z"), jOldPath, jNewPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsRename(oldPath=%s, newPath=%s): FileSystem#rename", + oldPath, newPath); + goto done; + } + if (!jVal.z) { + errno = EIO; + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jOldPath); + destroyLocalReference(env, jNewPath); + return ret; +} + + + +char* hdfsGetWorkingDirectory(hdfsFS fs, char* buffer, size_t bufferSize) +{ + // JAVA EQUIVALENT: + // Path p = fs.getWorkingDirectory(); + // return p.toString() + + jobject jPath = NULL; + jstring jPathString = NULL; + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + int ret; + const char *jPathChars = NULL; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //FileSystem#getWorkingDirectory() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getWorkingDirectory", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: FileSystem#getWorkingDirectory"); + goto done; + } + jPath = jVal.l; + if (!jPath) { + fprintf(stderr, "hdfsGetWorkingDirectory: " + "FileSystem#getWorkingDirectory returned NULL"); + ret = -EIO; + goto done; + } + + //Path#toString() + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: Path#toString"); + goto done; + } + jPathString = jVal.l; + jPathChars = (*env)->GetStringUTFChars(env, jPathString, NULL); + if (!jPathChars) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetWorkingDirectory: GetStringUTFChars"); + goto done; + } + + //Copy to user-provided buffer + ret = snprintf(buffer, bufferSize, "%s", jPathChars); + if (ret >= bufferSize) { + ret = ENAMETOOLONG; + goto done; + } + ret = 0; + +done: + if (jPathChars) { + (*env)->ReleaseStringUTFChars(env, jPathString, jPathChars); + } + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathString); + + if (ret) { + errno = ret; + return NULL; + } + return buffer; +} + + + +int hdfsSetWorkingDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.setWorkingDirectory(Path(path)); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetWorkingDirectory(%s): constructNewObjectOfPath", + path); + return -1; + } + + //FileSystem#setWorkingDirectory() + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setWorkingDirectory", "(Lorg/apache/hadoop/fs/Path;)V", + jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, NOPRINT_EXC_ILLEGAL_ARGUMENT, + "hdfsSetWorkingDirectory(%s): FileSystem#setWorkingDirectory", + path); + return -1; + } + return 0; +} + + + +int hdfsCreateDirectory(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.mkdirs(new Path(path)); + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsCreateDirectory(%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jVal.z = 0; + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "mkdirs", "(Lorg/apache/hadoop/fs/Path;)Z", jPath); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK | NOPRINT_EXC_PARENT_NOT_DIRECTORY, + "hdfsCreateDirectory(%s): FileSystem#mkdirs", path); + return -1; + } + if (!jVal.z) { + // It's unclear under exactly which conditions FileSystem#mkdirs + // is supposed to return false (as opposed to throwing an exception.) + // It seems like the current code never actually returns false. + // So we're going to translate this to EIO, since there seems to be + // nothing more specific we can do with it. + errno = EIO; + return -1; + } + return 0; +} + + +int hdfsSetReplication(hdfsFS fs, const char *path, int16_t replication) +{ + // JAVA EQUIVALENT: + // fs.setReplication(new Path(path), replication); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath; + jvalue jVal; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s): constructNewObjectOfPath", path); + return -1; + } + + //Create the directory + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "setReplication", "(Lorg/apache/hadoop/fs/Path;S)Z", + jPath, replication); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsSetReplication(path=%s, replication=%d): " + "FileSystem#setReplication", path, replication); + return -1; + } + if (!jVal.z) { + // setReplication returns false "if file does not exist or is a + // directory." So the nearest translation to that is ENOENT. + errno = ENOENT; + return -1; + } + + return 0; +} + +int hdfsChown(hdfsFS fs, const char *path, const char *owner, const char *group) +{ + // JAVA EQUIVALENT: + // fs.setOwner(path, owner, group) + + jobject jFS = (jobject)fs; + jobject jPath = NULL; + jstring jOwner = NULL, jGroup = NULL; + jthrowable jthr; + int ret; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + if (owner == NULL && group == NULL) { + return 0; + } + + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = newJavaStr(env, owner, &jOwner); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, owner); + goto done; + } + jthr = newJavaStr(env, group, &jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChown(path=%s): newJavaStr(%s)", path, group); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setOwner", JMETHOD3(JPARAM(HADOOP_PATH), + JPARAM(JAVA_STRING), JPARAM(JAVA_STRING), JAVA_VOID), + jPath, jOwner, jGroup); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChown(path=%s, owner=%s, group=%s): " + "FileSystem#setOwner", path, owner, group); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jOwner); + destroyLocalReference(env, jGroup); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsChmod(hdfsFS fs, const char *path, short mode) +{ + int ret; + // JAVA EQUIVALENT: + // fs.setPermission(path, FsPermission) + + jthrowable jthr; + jobject jPath = NULL, jPermObj = NULL; + jobject jFS = (jobject)fs; + jshort jmode = mode; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + // construct jPerm = FsPermission.createImmutable(short mode); + jthr = constructNewObjectOfCachedClass(env, &jPermObj, JC_FS_PERMISSION, + "(S)V",jmode); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "constructNewObjectOfCachedClass(%s)", HADOOP_FSPERM); + goto done; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsChmod(%s): constructNewObjectOfPath", path); + goto done; + } + + //Create the directory + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setPermission", JMETHOD2(JPARAM(HADOOP_PATH), + JPARAM(HADOOP_FSPERM), JAVA_VOID), jPath, jPermObj); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsChmod(%s): FileSystem#setPermission", path); + goto done; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPermObj); + + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +int hdfsUtime(hdfsFS fs, const char *path, tTime mtime, tTime atime) +{ + // JAVA EQUIVALENT: + // fs.setTimes(src, mtime, atime) + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + static const tTime NO_CHANGE = -1; + jlong jmtime, jatime; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsUtime(path=%s): constructNewObjectOfPath", path); + return -1; + } + + jmtime = (mtime == NO_CHANGE) ? -1 : (mtime * (jlong)1000); + jatime = (atime == NO_CHANGE) ? -1 : (atime * (jlong)1000); + + jthr = invokeMethod(env, NULL, INSTANCE, jFS, JC_FILE_SYSTEM, + "setTimes", JMETHOD3(JPARAM(HADOOP_PATH), "J", "J", + JAVA_VOID), jPath, jmtime, jatime); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsUtime(path=%s): FileSystem#setTimes", path); + return -1; + } + return 0; +} + +/** + * Zero-copy options. + * + * We cache the EnumSet of ReadOptions which has to be passed into every + * readZero call, to avoid reconstructing it each time. This cache is cleared + * whenever an element changes. + */ +struct hadoopRzOptions +{ + JNIEnv *env; + int skipChecksums; + jobject byteBufferPool; + jobject cachedEnumSet; +}; + +struct hadoopRzOptions *hadoopRzOptionsAlloc(void) +{ + struct hadoopRzOptions *opts; + JNIEnv *env; + + env = getJNIEnv(); + if (!env) { + // Check to make sure the JNI environment is set up properly. + errno = EINTERNAL; + return NULL; + } + opts = calloc(1, sizeof(struct hadoopRzOptions)); + if (!opts) { + errno = ENOMEM; + return NULL; + } + return opts; +} + +static void hadoopRzOptionsClearCached(JNIEnv *env, + struct hadoopRzOptions *opts) +{ + if (!opts->cachedEnumSet) { + return; + } + (*env)->DeleteGlobalRef(env, opts->cachedEnumSet); + opts->cachedEnumSet = NULL; +} + +int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + hadoopRzOptionsClearCached(env, opts); + opts->skipChecksums = !!skip; + return 0; +} + +int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className) +{ + JNIEnv *env; + jthrowable jthr; + jobject byteBufferPool = NULL; + jobject globalByteBufferPool = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return -1; + } + + if (className) { + // Note: we don't have to call hadoopRzOptionsClearCached in this + // function, since the ByteBufferPool is passed separately from the + // EnumSet of ReadOptions. + + jthr = constructNewObjectOfClass(env, &byteBufferPool, className, "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", className); + ret = EINVAL; + goto done; + } + // Only set opts->byteBufferPool if creating a global reference is + // successful + globalByteBufferPool = (*env)->NewGlobalRef(env, byteBufferPool); + if (!globalByteBufferPool) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopRzOptionsSetByteBufferPool(className=%s): ", + className); + ret = EINVAL; + goto done; + } + // Delete any previous ByteBufferPool we had before setting a new one. + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + } + opts->byteBufferPool = globalByteBufferPool; + } else if (opts->byteBufferPool) { + // If the specified className is NULL, delete any previous + // ByteBufferPool we had. + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + ret = 0; +done: + destroyLocalReference(env, byteBufferPool); + if (ret) { + errno = ret; + return -1; + } + return 0; +} + +void hadoopRzOptionsFree(struct hadoopRzOptions *opts) +{ + JNIEnv *env; + env = getJNIEnv(); + if (!env) { + return; + } + hadoopRzOptionsClearCached(env, opts); + if (opts->byteBufferPool) { + (*env)->DeleteGlobalRef(env, opts->byteBufferPool); + opts->byteBufferPool = NULL; + } + free(opts); +} + +struct hadoopRzBuffer +{ + jobject byteBuffer; + uint8_t *ptr; + int32_t length; + int direct; +}; + +static jthrowable hadoopRzOptionsGetEnumSet(JNIEnv *env, + struct hadoopRzOptions *opts, jobject *enumSet) +{ + jthrowable jthr = NULL; + jobject enumInst = NULL, enumSetObj = NULL; + jvalue jVal; + + if (opts->cachedEnumSet) { + // If we cached the value, return it now. + *enumSet = opts->cachedEnumSet; + goto done; + } + if (opts->skipChecksums) { + jthr = fetchEnumInstance(env, HADOOP_RO, + "SKIP_CHECKSUMS", &enumInst); + if (jthr) { + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "of", "(Ljava/lang/Enum;)Ljava/util/EnumSet;", enumInst); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } else { + jclass clazz = (*env)->FindClass(env, HADOOP_RO); + if (!clazz) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + jthr = invokeMethod(env, &jVal, STATIC, NULL, JC_ENUM_SET, + "noneOf", "(Ljava/lang/Class;)Ljava/util/EnumSet;", clazz); + if (jthr) { + goto done; + } + enumSetObj = jVal.l; + } + // create global ref + opts->cachedEnumSet = (*env)->NewGlobalRef(env, enumSetObj); + if (!opts->cachedEnumSet) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *enumSet = opts->cachedEnumSet; + jthr = NULL; +done: + (*env)->DeleteLocalRef(env, enumInst); + (*env)->DeleteLocalRef(env, enumSetObj); + return jthr; +} + +static int hadoopReadZeroExtractBuffer(JNIEnv *env, + const struct hadoopRzOptions *opts, struct hadoopRzBuffer *buffer) +{ + int ret; + jthrowable jthr; + jvalue jVal; + uint8_t *directStart; + void *mallocBuf = NULL; + jint position; + jarray array = NULL; + + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "remaining", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#remaining failed: "); + goto done; + } + buffer->length = jVal.i; + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "position", "()I"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#position failed: "); + goto done; + } + position = jVal.i; + directStart = (*env)->GetDirectBufferAddress(env, buffer->byteBuffer); + if (directStart) { + // Handle direct buffers. + buffer->ptr = directStart + position; + buffer->direct = 1; + ret = 0; + goto done; + } + // Handle indirect buffers. + // The JNI docs don't say that GetDirectBufferAddress throws any exceptions + // when it fails. However, they also don't clearly say that it doesn't. It + // seems safest to clear any pending exceptions here, to prevent problems on + // various JVMs. + (*env)->ExceptionClear(env); + if (!opts->byteBufferPool) { + fputs("hadoopReadZeroExtractBuffer: we read through the " + "zero-copy path, but failed to get the address of the buffer via " + "GetDirectBufferAddress. Please make sure your JVM supports " + "GetDirectBufferAddress.\n", stderr); + ret = ENOTSUP; + goto done; + } + // Get the backing array object of this buffer. + jthr = invokeMethod(env, &jVal, INSTANCE, buffer->byteBuffer, + JC_BYTE_BUFFER, "array", "()[B"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: ByteBuffer#array failed: "); + goto done; + } + array = jVal.l; + if (!array) { + fputs("hadoopReadZeroExtractBuffer: ByteBuffer#array returned NULL.", + stderr); + ret = EIO; + goto done; + } + mallocBuf = malloc(buffer->length); + if (!mallocBuf) { + fprintf(stderr, "hadoopReadZeroExtractBuffer: failed to allocate %d bytes of memory\n", + buffer->length); + ret = ENOMEM; + goto done; + } + (*env)->GetByteArrayRegion(env, array, position, buffer->length, mallocBuf); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZeroExtractBuffer: GetByteArrayRegion failed: "); + goto done; + } + buffer->ptr = mallocBuf; + buffer->direct = 0; + ret = 0; + +done: + free(mallocBuf); + (*env)->DeleteLocalRef(env, array); + return ret; +} + +static int translateZCRException(JNIEnv *env, jthrowable exc) +{ + int ret; + char *className = NULL; + jthrowable jthr = classNameOfObject(exc, env, &className); + + if (jthr) { + fputs("hadoopReadZero: failed to get class name of " + "exception from read().\n", stderr); + destroyLocalReference(env, exc); + destroyLocalReference(env, jthr); + ret = EIO; + goto done; + } + if (!strcmp(className, "java.lang.UnsupportedOperationException")) { + ret = EPROTONOSUPPORT; + destroyLocalReference(env, exc); + goto done; + } + ret = printExceptionAndFree(env, exc, PRINT_EXC_ALL, + "hadoopZeroCopyRead: ZeroCopyCursor#read failed"); +done: + free(className); + return ret; +} + +struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength) +{ + JNIEnv *env; + jthrowable jthr = NULL; + jvalue jVal; + jobject enumSet = NULL, byteBuffer = NULL; + struct hadoopRzBuffer* buffer = NULL; + int ret; + + env = getJNIEnv(); + if (!env) { + errno = EINTERNAL; + return NULL; + } + if (file->type != HDFS_STREAM_INPUT) { + fputs("Cannot read from a non-InputStream object!\n", stderr); + ret = EINVAL; + goto done; + } + buffer = calloc(1, sizeof(struct hadoopRzBuffer)); + if (!buffer) { + ret = ENOMEM; + goto done; + } + jthr = hadoopRzOptionsGetEnumSet(env, opts, &enumSet); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopReadZero: hadoopRzOptionsGetEnumSet failed: "); + goto done; + } + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "read", + "(Lorg/apache/hadoop/io/ByteBufferPool;ILjava/util/EnumSet;)" + "Ljava/nio/ByteBuffer;", opts->byteBufferPool, maxLength, enumSet); + if (jthr) { + ret = translateZCRException(env, jthr); + goto done; + } + byteBuffer = jVal.l; + if (!byteBuffer) { + buffer->byteBuffer = NULL; + buffer->length = 0; + buffer->ptr = NULL; + } else { + buffer->byteBuffer = (*env)->NewGlobalRef(env, byteBuffer); + if (!buffer->byteBuffer) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hadoopReadZero: failed to create global ref to ByteBuffer"); + goto done; + } + ret = hadoopReadZeroExtractBuffer(env, opts, buffer); + if (ret) { + goto done; + } + } + ret = 0; +done: + (*env)->DeleteLocalRef(env, byteBuffer); + if (ret) { + if (buffer) { + if (buffer->byteBuffer) { + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + free(buffer); + } + errno = ret; + return NULL; + } else { + errno = 0; + } + return buffer; +} + +int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer) +{ + return buffer->length; +} + +const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer) +{ + return buffer->ptr; +} + +void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer) +{ + jvalue jVal; + jthrowable jthr; + JNIEnv* env; + + env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return; + } + if (buffer->byteBuffer) { + jthr = invokeMethod(env, &jVal, INSTANCE, file->file, + JC_FS_DATA_INPUT_STREAM, "releaseBuffer", + "(Ljava/nio/ByteBuffer;)V", buffer->byteBuffer); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hadoopRzBufferFree: releaseBuffer failed: "); + // even on error, we have to delete the reference. + } + (*env)->DeleteGlobalRef(env, buffer->byteBuffer); + } + if (!buffer->direct) { + free(buffer->ptr); + } + memset(buffer, 0, sizeof(*buffer)); + free(buffer); +} + +char*** +hdfsGetHosts(hdfsFS fs, const char *path, tOffset start, tOffset length) +{ + // JAVA EQUIVALENT: + // fs.getFileBlockLoctions(new Path(path), start, length); + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + jobject jFileStatus = NULL; + jvalue jFSVal, jVal; + jobjectArray jBlockLocations = NULL, jFileBlockHosts = NULL; + jstring jHost = NULL; + char*** blockHosts = NULL; + int i, j, ret; + jsize jNumFileBlocks = 0; + jobject jFileBlock; + jsize jNumBlockHosts; + const char *hostName; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s): constructNewObjectOfPath", path); + goto done; + } + jthr = invokeMethod(env, &jFSVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", "(Lorg/apache/hadoop/fs/Path;)" + "Lorg/apache/hadoop/fs/FileStatus;", jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, NOPRINT_EXC_FILE_NOT_FOUND, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileStatus", path, start, length); + destroyLocalReference(env, jPath); + goto done; + } + jFileStatus = jFSVal.l; + + //org.apache.hadoop.fs.FileSystem#getFileBlockLocations + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileBlockLocations", + "(Lorg/apache/hadoop/fs/FileStatus;JJ)" + "[Lorg/apache/hadoop/fs/BlockLocation;", jFileStatus, start, + length); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "FileSystem#getFileBlockLocations", path, start, length); + goto done; + } + jBlockLocations = jVal.l; + + //Figure out no of entries in jBlockLocations + //Allocate memory and add NULL at the end + jNumFileBlocks = (*env)->GetArrayLength(env, jBlockLocations); + + blockHosts = calloc(jNumFileBlocks + 1, sizeof(char**)); + if (blockHosts == NULL) { + ret = ENOMEM; + goto done; + } + if (jNumFileBlocks == 0) { + ret = 0; + goto done; + } + + //Now parse each block to get hostnames + for (i = 0; i < jNumFileBlocks; ++i) { + jFileBlock = + (*env)->GetObjectArrayElement(env, jBlockLocations, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jFileBlock) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "GetObjectArrayElement(%d)", path, start, length, i); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFileBlock, + JC_BLOCK_LOCATION, "getHosts", + "()[Ljava/lang/String;"); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts", path, start, length); + goto done; + } + jFileBlockHosts = jVal.l; + if (!jFileBlockHosts) { + fprintf(stderr, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"):" + "BlockLocation#getHosts returned NULL", path, start, length); + ret = EINTERNAL; + goto done; + } + //Figure out no of hosts in jFileBlockHosts, and allocate the memory + jNumBlockHosts = (*env)->GetArrayLength(env, jFileBlockHosts); + blockHosts[i] = calloc(jNumBlockHosts + 1, sizeof(char*)); + if (!blockHosts[i]) { + ret = ENOMEM; + goto done; + } + + //Now parse each hostname + for (j = 0; j < jNumBlockHosts; ++j) { + jHost = (*env)->GetObjectArrayElement(env, jFileBlockHosts, j); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !jHost) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64"): " + "NewByteArray", path, start, length); + goto done; + } + hostName = + (const char*)((*env)->GetStringUTFChars(env, jHost, NULL)); + if (!hostName) { + ret = printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "hdfsGetHosts(path=%s, start=%"PRId64", length=%"PRId64", " + "j=%d out of %d): GetStringUTFChars", + path, start, length, j, jNumBlockHosts); + goto done; + } + blockHosts[i][j] = strdup(hostName); + (*env)->ReleaseStringUTFChars(env, jHost, hostName); + if (!blockHosts[i][j]) { + ret = ENOMEM; + goto done; + } + destroyLocalReference(env, jHost); + jHost = NULL; + } + + destroyLocalReference(env, jFileBlockHosts); + jFileBlockHosts = NULL; + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jFileStatus); + destroyLocalReference(env, jBlockLocations); + destroyLocalReference(env, jFileBlockHosts); + destroyLocalReference(env, jHost); + if (ret) { + errno = ret; + if (blockHosts) { + hdfsFreeHosts(blockHosts); + } + return NULL; + } + + return blockHosts; +} + + +void hdfsFreeHosts(char ***blockHosts) +{ + int i, j; + for (i=0; blockHosts[i]; i++) { + for (j=0; blockHosts[i][j]; j++) { + free(blockHosts[i][j]); + } + free(blockHosts[i]); + } + free(blockHosts); +} + + +tOffset hdfsGetDefaultBlockSize(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getDefaultBlockSize() + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getDefaultBlockSize", "()J"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize: FileSystem#getDefaultBlockSize"); + return -1; + } + return jVal.j; +} + + +tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // fs.getDefaultBlockSize(path); + + jthrowable jthr; + jobject jFS = (jobject)fs; + jobject jPath; + tOffset blockSize; + JNIEnv* env = getJNIEnv(); + + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): constructNewObjectOfPath", + path); + return -1; + } + jthr = getDefaultBlockSize(env, jFS, jPath, (jlong *)&blockSize); + (*env)->DeleteLocalRef(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetDefaultBlockSize(path=%s): " + "FileSystem#getDefaultBlockSize", path); + return -1; + } + return blockSize; +} + + +tOffset hdfsGetCapacity(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getCapacity(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, + JC_FS_STATUS, "getCapacity", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetCapacity: FsStatus#getCapacity"); + return -1; + } + return jVal.j; +} + + + +tOffset hdfsGetUsed(hdfsFS fs) +{ + // JAVA EQUIVALENT: + // FsStatus fss = fs.getStatus(); + // return Fss.getUsed(); + + jobject jFS = (jobject)fs; + jvalue jVal; + jthrowable jthr; + jobject fss; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return -1; + } + + //FileSystem#getStatus + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getStatus", "()Lorg/apache/hadoop/fs/FsStatus;"); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FileSystem#getStatus"); + return -1; + } + fss = (jobject)jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, fss, JC_FS_STATUS, + "getUsed", "()J"); + destroyLocalReference(env, fss); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetUsed: FsStatus#getUsed"); + return -1; + } + return jVal.j; +} + +/** + * We cannot add new fields to the hdfsFileInfo structure because it would break + * binary compatibility. The reason is because we return an array + * of hdfsFileInfo structures from hdfsListDirectory. So changing the size of + * those structures would break all programs that relied on finding the second + * element in the array at + sizeof(struct hdfsFileInfo). + * + * So instead, we add the new fields to the hdfsExtendedFileInfo structure. + * This structure is contained in the mOwner string found inside the + * hdfsFileInfo. Specifically, the format of mOwner is: + * + * [owner-string] [null byte] [padding] [hdfsExtendedFileInfo structure] + * + * The padding is added so that the hdfsExtendedFileInfo structure starts on an + * 8-byte boundary. + * + * @param str The string to locate the extended info in. + * @return The offset of the hdfsExtendedFileInfo structure. + */ +static size_t getExtendedFileInfoOffset(const char *str) +{ + int num_64_bit_words = ((strlen(str) + 1) + 7) / 8; + return num_64_bit_words * 8; +} + +static struct hdfsExtendedFileInfo *getExtendedFileInfo(hdfsFileInfo *fileInfo) +{ + char *owner = fileInfo->mOwner; + return (struct hdfsExtendedFileInfo *)(owner + + getExtendedFileInfoOffset(owner)); +} + +static jthrowable +getFileInfoFromStat(JNIEnv *env, jobject jStat, hdfsFileInfo *fileInfo) +{ + jvalue jVal; + jthrowable jthr; + jobject jPath = NULL; + jstring jPathName = NULL; + jstring jUserName = NULL; + jstring jGroupName = NULL; + jobject jPermission = NULL; + const char *cPathName; + const char *cUserName; + const char *cGroupName; + struct hdfsExtendedFileInfo *extInfo; + size_t extOffset; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "isDir", + "()Z"); + if (jthr) + goto done; + fileInfo->mKind = jVal.z ? kObjectKindDirectory : kObjectKindFile; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getReplication", "()S"); + if (jthr) + goto done; + fileInfo->mReplication = jVal.s; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getBlockSize", "()J"); + if (jthr) + goto done; + fileInfo->mBlockSize = jVal.j; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getModificationTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastMod = jVal.j / 1000; + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getAccessTime", "()J"); + if (jthr) + goto done; + fileInfo->mLastAccess = (tTime) (jVal.j / 1000); + + if (fileInfo->mKind == kObjectKindFile) { + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getLen", "()J"); + if (jthr) + goto done; + fileInfo->mSize = jVal.j; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPath", "()Lorg/apache/hadoop/fs/Path;"); + if (jthr) + goto done; + jPath = jVal.l; + if (jPath == NULL) { + jthr = newRuntimeError(env, "org.apache.hadoop.fs.FileStatus#" + "getPath returned NULL!"); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jPath, JC_PATH, "toString", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jPathName = jVal.l; + cPathName = + (const char*) ((*env)->GetStringUTFChars(env, jPathName, NULL)); + if (!cPathName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mName = strdup(cPathName); + (*env)->ReleaseStringUTFChars(env, jPathName, cPathName); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, "getOwner", + "()Ljava/lang/String;"); + if (jthr) + goto done; + jUserName = jVal.l; + cUserName = + (const char*) ((*env)->GetStringUTFChars(env, jUserName, NULL)); + if (!cUserName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + extOffset = getExtendedFileInfoOffset(cUserName); + fileInfo->mOwner = malloc(extOffset + sizeof(struct hdfsExtendedFileInfo)); + if (!fileInfo->mOwner) { + jthr = newRuntimeError(env, "getFileInfo: OOM allocating mOwner"); + goto done; + } + strcpy(fileInfo->mOwner, cUserName); + (*env)->ReleaseStringUTFChars(env, jUserName, cUserName); + extInfo = getExtendedFileInfo(fileInfo); + memset(extInfo, 0, sizeof(*extInfo)); + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "isEncrypted", "()Z"); + if (jthr) { + goto done; + } + if (jVal.z == JNI_TRUE) { + extInfo->flags |= HDFS_EXTENDED_FILE_INFO_ENCRYPTED; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getGroup", "()Ljava/lang/String;"); + if (jthr) + goto done; + jGroupName = jVal.l; + cGroupName = (const char*) ((*env)->GetStringUTFChars(env, jGroupName, NULL)); + if (!cGroupName) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + fileInfo->mGroup = strdup(cGroupName); + (*env)->ReleaseStringUTFChars(env, jGroupName, cGroupName); + + jthr = invokeMethod(env, &jVal, INSTANCE, jStat, JC_FILE_STATUS, + "getPermission", + "()Lorg/apache/hadoop/fs/permission/FsPermission;"); + if (jthr) + goto done; + if (jVal.l == NULL) { + jthr = newRuntimeError(env, "%s#getPermission returned NULL!", + HADOOP_FILESTAT); + goto done; + } + jPermission = jVal.l; + jthr = invokeMethod(env, &jVal, INSTANCE, jPermission, + JC_FS_PERMISSION, "toShort", "()S"); + if (jthr) + goto done; + fileInfo->mPermissions = jVal.s; + jthr = NULL; + +done: + if (jthr) + hdfsFreeFileInfoEntry(fileInfo); + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathName); + destroyLocalReference(env, jUserName); + destroyLocalReference(env, jGroupName); + destroyLocalReference(env, jPermission); + return jthr; +} + +static jthrowable +getFileInfo(JNIEnv *env, jobject jFS, jobject jPath, hdfsFileInfo **fileInfo) +{ + // JAVA EQUIVALENT: + // fs.isDirectory(f) + // fs.getModificationTime() + // fs.getAccessTime() + // fs.getLength(f) + // f.getPath() + // f.getOwner() + // f.getGroup() + // f.getPermission().toShort() + jobject jStat; + jvalue jVal; + jthrowable jthr; + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, "exists", + JMETHOD1(JPARAM(HADOOP_PATH), "Z"), jPath); + if (jthr) + return jthr; + if (jVal.z == 0) { + *fileInfo = NULL; + return NULL; + } + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, JC_FILE_SYSTEM, + "getFileStatus", JMETHOD1(JPARAM(HADOOP_PATH), JPARAM + (HADOOP_FILESTAT)), jPath); + if (jthr) + return jthr; + jStat = jVal.l; + *fileInfo = calloc(1, sizeof(hdfsFileInfo)); + if (!*fileInfo) { + destroyLocalReference(env, jStat); + return newRuntimeError(env, "getFileInfo: OOM allocating hdfsFileInfo"); + } + jthr = getFileInfoFromStat(env, jStat, *fileInfo); + destroyLocalReference(env, jStat); + return jthr; +} + + + +hdfsFileInfo* hdfsListDirectory(hdfsFS fs, const char *path, int *numEntries) +{ + // JAVA EQUIVALENT: + // Path p(path); + // Path []pathList = fs.listPaths(p) + // foreach path in pathList + // getFileInfo(path) + + jobject jFS = (jobject)fs; + jthrowable jthr; + jobject jPath = NULL; + hdfsFileInfo *pathList = NULL; + jobjectArray jPathList = NULL; + jvalue jVal; + jsize jPathListSize = 0; + int ret; + jsize i; + jobject tmpStat; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): constructNewObjectOfPath", path); + goto done; + } + + jthr = invokeMethod(env, &jVal, INSTANCE, jFS, + JC_DISTRIBUTED_FILE_SYSTEM, "listStatus", + JMETHOD1(JPARAM(HADOOP_PATH), JARRPARAM(HADOOP_FILESTAT)), jPath); + if (jthr) { + ret = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsListDirectory(%s): FileSystem#listStatus", path); + goto done; + } + jPathList = jVal.l; + + //Figure out the number of entries in that directory + jPathListSize = (*env)->GetArrayLength(env, jPathList); + if (jPathListSize == 0) { + ret = 0; + goto done; + } + + //Allocate memory + pathList = calloc(jPathListSize, sizeof(hdfsFileInfo)); + if (pathList == NULL) { + ret = ENOMEM; + goto done; + } + + //Save path information in pathList + for (i=0; i < jPathListSize; ++i) { + tmpStat = (*env)->GetObjectArrayElement(env, jPathList, i); + jthr = (*env)->ExceptionOccurred(env); + if (jthr || !tmpStat) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): GetObjectArrayElement(%d out of %d)", + path, i, jPathListSize); + goto done; + } + jthr = getFileInfoFromStat(env, tmpStat, &pathList[i]); + destroyLocalReference(env, tmpStat); + if (jthr) { + ret = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsListDirectory(%s): getFileInfoFromStat(%d out of %d)", + path, i, jPathListSize); + goto done; + } + } + ret = 0; + +done: + destroyLocalReference(env, jPath); + destroyLocalReference(env, jPathList); + + if (ret) { + hdfsFreeFileInfo(pathList, jPathListSize); + errno = ret; + return NULL; + } + *numEntries = jPathListSize; + errno = 0; + return pathList; +} + + + +hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char *path) +{ + // JAVA EQUIVALENT: + // File f(path); + // fs.isDirectory(f) + // fs.lastModified() ?? + // fs.getLength(f) + // f.getPath() + + jobject jFS = (jobject)fs; + jobject jPath; + jthrowable jthr; + hdfsFileInfo *fileInfo; + + //Get the JNIEnv* corresponding to current thread + JNIEnv* env = getJNIEnv(); + if (env == NULL) { + errno = EINTERNAL; + return NULL; + } + + //Create an object of org.apache.hadoop.fs.Path + jthr = constructNewObjectOfPath(env, path, &jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "hdfsGetPathInfo(%s): constructNewObjectOfPath", path); + return NULL; + } + jthr = getFileInfo(env, jFS, jPath, &fileInfo); + destroyLocalReference(env, jPath); + if (jthr) { + errno = printExceptionAndFree(env, jthr, + NOPRINT_EXC_ACCESS_CONTROL | NOPRINT_EXC_FILE_NOT_FOUND | + NOPRINT_EXC_UNRESOLVED_LINK, + "hdfsGetPathInfo(%s): getFileInfo", path); + return NULL; + } + if (!fileInfo) { + errno = ENOENT; + return NULL; + } + return fileInfo; +} + +static void hdfsFreeFileInfoEntry(hdfsFileInfo *hdfsFileInfo) +{ + free(hdfsFileInfo->mName); + free(hdfsFileInfo->mOwner); + free(hdfsFileInfo->mGroup); + memset(hdfsFileInfo, 0, sizeof(*hdfsFileInfo)); +} + +void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries) +{ + //Free the mName, mOwner, and mGroup + int i; + for (i=0; i < numEntries; ++i) { + hdfsFreeFileInfoEntry(hdfsFileInfo + i); + } + + //Free entire block + free(hdfsFileInfo); +} + +int hdfsFileIsEncrypted(hdfsFileInfo *fileInfo) +{ + struct hdfsExtendedFileInfo *extInfo; + + extInfo = getExtendedFileInfo(fileInfo); + return !!(extInfo->flags & HDFS_EXTENDED_FILE_INFO_ENCRYPTED); +} + +char* hdfsGetLastExceptionRootCause() +{ + return getLastTLSExceptionRootCause(); +} + +char* hdfsGetLastExceptionStackTrace() +{ + return getLastTLSExceptionStackTrace(); +} + +/** + * vim: ts=4: sw=4: et: + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h b/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h new file mode 100644 index 00000000000..e58a6232d20 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/include/hdfs/hdfs.h @@ -0,0 +1,1105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_HDFS_H +#define LIBHDFS_HDFS_H + +#include /* for EINTERNAL, etc. */ +#include /* for O_RDONLY, O_WRONLY */ +#include /* for uint64_t, etc. */ +#include /* for time_t */ + +/* + * Support export of DLL symbols during libhdfs build, and import of DLL symbols + * during client application build. A client application may optionally define + * symbol LIBHDFS_DLL_IMPORT in its build. This is not strictly required, but + * the compiler can produce more efficient code with it. + */ +#ifdef WIN32 + #ifdef LIBHDFS_DLL_EXPORT + #define LIBHDFS_EXTERNAL __declspec(dllexport) + #elif LIBHDFS_DLL_IMPORT + #define LIBHDFS_EXTERNAL __declspec(dllimport) + #else + #define LIBHDFS_EXTERNAL + #endif +#else + #ifdef LIBHDFS_DLL_EXPORT + #define LIBHDFS_EXTERNAL __attribute__((visibility("default"))) + #elif LIBHDFS_DLL_IMPORT + #define LIBHDFS_EXTERNAL __attribute__((visibility("default"))) + #else + #define LIBHDFS_EXTERNAL + #endif +#endif + +#ifndef O_RDONLY +#define O_RDONLY 1 +#endif + +#ifndef O_WRONLY +#define O_WRONLY 2 +#endif + +#ifndef EINTERNAL +#define EINTERNAL 255 +#endif + +#define ELASTIC_BYTE_BUFFER_POOL_CLASS \ + "org/apache/hadoop/io/ElasticByteBufferPool" + +/** All APIs set errno to meaningful values */ + +#ifdef __cplusplus +extern "C" { +#endif + /** + * Some utility decls used in libhdfs. + */ + struct hdfsBuilder; + typedef int32_t tSize; /// size of data for read/write io ops + typedef time_t tTime; /// time type in seconds + typedef int64_t tOffset;/// offset within the file + typedef uint16_t tPort; /// port + typedef enum tObjectKind { + kObjectKindFile = 'F', + kObjectKindDirectory = 'D', + } tObjectKind; + struct hdfsStreamBuilder; + + + /** + * The C reflection of org.apache.org.hadoop.FileSystem . + */ + struct hdfs_internal; + typedef struct hdfs_internal* hdfsFS; + + struct hdfsFile_internal; + typedef struct hdfsFile_internal* hdfsFile; + + struct hadoopRzOptions; + + struct hadoopRzBuffer; + + /** + * Determine if a file is open for read. + * + * @param file The HDFS file + * @return 1 if the file is open for read; 0 otherwise + */ + LIBHDFS_EXTERNAL + int hdfsFileIsOpenForRead(hdfsFile file); + + /** + * Determine if a file is open for write. + * + * @param file The HDFS file + * @return 1 if the file is open for write; 0 otherwise + */ + LIBHDFS_EXTERNAL + int hdfsFileIsOpenForWrite(hdfsFile file); + + struct hdfsReadStatistics { + uint64_t totalBytesRead; + uint64_t totalLocalBytesRead; + uint64_t totalShortCircuitBytesRead; + uint64_t totalZeroCopyBytesRead; + }; + + /** + * Get read statistics about a file. This is only applicable to files + * opened for reading. + * + * @param file The HDFS file + * @param stats (out parameter) on a successful return, the read + * statistics. Unchanged otherwise. You must free the + * returned statistics with hdfsFileFreeReadStatistics. + * @return 0 if the statistics were successfully returned, + * -1 otherwise. On a failure, please check errno against + * ENOTSUP. webhdfs, LocalFilesystem, and so forth may + * not support read statistics. + */ + LIBHDFS_EXTERNAL + int hdfsFileGetReadStatistics(hdfsFile file, + struct hdfsReadStatistics **stats); + + /** + * @param stats HDFS read statistics for a file. + * + * @return the number of remote bytes read. + */ + LIBHDFS_EXTERNAL + int64_t hdfsReadStatisticsGetRemoteBytesRead( + const struct hdfsReadStatistics *stats); + + /** + * Clear the read statistics for a file. + * + * @param file The file to clear the read statistics of. + * + * @return 0 on success; the error code otherwise. + * EINVAL: the file is not open for reading. + * ENOTSUP: the file does not support clearing the read + * statistics. + * Errno will also be set to this code on failure. + */ + LIBHDFS_EXTERNAL + int hdfsFileClearReadStatistics(hdfsFile file); + + /** + * Free some HDFS read statistics. + * + * @param stats The HDFS read statistics to free. + */ + LIBHDFS_EXTERNAL + void hdfsFileFreeReadStatistics(struct hdfsReadStatistics *stats); + + struct hdfsHedgedReadMetrics { + uint64_t hedgedReadOps; + uint64_t hedgedReadOpsWin; + uint64_t hedgedReadOpsInCurThread; + }; + + /** + * Get cluster wide hedged read metrics. + * + * @param fs The configured filesystem handle + * @param metrics (out parameter) on a successful return, the hedged read + * metrics. Unchanged otherwise. You must free the returned + * statistics with hdfsFreeHedgedReadMetrics. + * @return 0 if the metrics were successfully returned, -1 otherwise. + * On a failure, please check errno against + * ENOTSUP. webhdfs, LocalFilesystem, and so forth may + * not support hedged read metrics. + */ + LIBHDFS_EXTERNAL + int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics); + + /** + * Free HDFS Hedged read metrics. + * + * @param metrics The HDFS Hedged read metrics to free + */ + LIBHDFS_EXTERNAL + void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics); + + /** + * hdfsConnectAsUser - Connect to a hdfs file system as a specific user + * Connect to the hdfs. + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port) + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectAsUser(const char* nn, tPort port, const char *user); + + /** + * hdfsConnect - Connect to a hdfs file system. + * Connect to the hdfs. + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnect(const char* nn, tPort port); + + /** + * hdfsConnect - Connect to an hdfs file system. + * + * Forces a new instance to be created + * + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @param user The user name to use when connecting + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectAsUserNewInstance(const char* nn, tPort port, const char *user ); + + /** + * hdfsConnect - Connect to an hdfs file system. + * + * Forces a new instance to be created + * + * @param nn The NameNode. See hdfsBuilderSetNameNode for details. + * @param port The port on which the server is listening. + * @return Returns a handle to the filesystem or NULL on error. + * @deprecated Use hdfsBuilderConnect instead. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsConnectNewInstance(const char* nn, tPort port); + + /** + * Connect to HDFS using the parameters defined by the builder. + * + * The HDFS builder will be freed, whether or not the connection was + * successful. + * + * Every successful call to hdfsBuilderConnect should be matched with a call + * to hdfsDisconnect, when the hdfsFS is no longer needed. + * + * @param bld The HDFS builder + * @return Returns a handle to the filesystem, or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFS hdfsBuilderConnect(struct hdfsBuilder *bld); + + /** + * Create an HDFS builder. + * + * @return The HDFS builder, or NULL on error. + */ + LIBHDFS_EXTERNAL + struct hdfsBuilder *hdfsNewBuilder(void); + + /** + * Force the builder to always create a new instance of the FileSystem, + * rather than possibly finding one in the cache. + * + * @param bld The HDFS builder + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetForceNewInstance(struct hdfsBuilder *bld); + + /** + * Set the HDFS NameNode to connect to. + * + * @param bld The HDFS builder + * @param nn The NameNode to use. + * + * If the string given is 'default', the default NameNode + * configuration will be used (from the XML configuration files) + * + * If NULL is given, a LocalFileSystem will be created. + * + * If the string starts with a protocol type such as file:// or + * hdfs://, this protocol type will be used. If not, the + * hdfs:// protocol type will be used. + * + * You may specify a NameNode port in the usual way by + * passing a string of the format hdfs://:. + * Alternately, you may set the port with + * hdfsBuilderSetNameNodePort. However, you must not pass the + * port in two different ways. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetNameNode(struct hdfsBuilder *bld, const char *nn); + + /** + * Set the port of the HDFS NameNode to connect to. + * + * @param bld The HDFS builder + * @param port The port. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetNameNodePort(struct hdfsBuilder *bld, tPort port); + + /** + * Set the username to use when connecting to the HDFS cluster. + * + * @param bld The HDFS builder + * @param userName The user name. The string will be shallow-copied. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetUserName(struct hdfsBuilder *bld, const char *userName); + + /** + * Set the path to the Kerberos ticket cache to use when connecting to + * the HDFS cluster. + * + * @param bld The HDFS builder + * @param kerbTicketCachePath The Kerberos ticket cache path. The string + * will be shallow-copied. + */ + LIBHDFS_EXTERNAL + void hdfsBuilderSetKerbTicketCachePath(struct hdfsBuilder *bld, + const char *kerbTicketCachePath); + + /** + * Free an HDFS builder. + * + * It is normally not necessary to call this function since + * hdfsBuilderConnect frees the builder. + * + * @param bld The HDFS builder + */ + LIBHDFS_EXTERNAL + void hdfsFreeBuilder(struct hdfsBuilder *bld); + + /** + * Set a configuration string for an HdfsBuilder. + * + * @param key The key to set. + * @param val The value, or NULL to set no value. + * This will be shallow-copied. You are responsible for + * ensuring that it remains valid until the builder is + * freed. + * + * @return 0 on success; nonzero error code otherwise. + */ + LIBHDFS_EXTERNAL + int hdfsBuilderConfSetStr(struct hdfsBuilder *bld, const char *key, + const char *val); + + /** + * Get a configuration string. + * + * @param key The key to find + * @param val (out param) The value. This will be set to NULL if the + * key isn't found. You must free this string with + * hdfsConfStrFree. + * + * @return 0 on success; nonzero error code otherwise. + * Failure to find the key is not an error. + */ + LIBHDFS_EXTERNAL + int hdfsConfGetStr(const char *key, char **val); + + /** + * Get a configuration integer. + * + * @param key The key to find + * @param val (out param) The value. This will NOT be changed if the + * key isn't found. + * + * @return 0 on success; nonzero error code otherwise. + * Failure to find the key is not an error. + */ + LIBHDFS_EXTERNAL + int hdfsConfGetInt(const char *key, int32_t *val); + + /** + * Free a configuration string found with hdfsConfGetStr. + * + * @param val A configuration string obtained from hdfsConfGetStr + */ + LIBHDFS_EXTERNAL + void hdfsConfStrFree(char *val); + + /** + * hdfsDisconnect - Disconnect from the hdfs file system. + * Disconnect from hdfs. + * @param fs The configured filesystem handle. + * @return Returns 0 on success, -1 on error. + * Even if there is an error, the resources associated with the + * hdfsFS will be freed. + */ + LIBHDFS_EXTERNAL + int hdfsDisconnect(hdfsFS fs); + + /** + * hdfsOpenFile - Open a hdfs file in given mode. + * @deprecated Use the hdfsStreamBuilder functions instead. + * This function does not support setting block sizes bigger than 2 GB. + * + * @param fs The configured filesystem handle. + * @param path The full path to the file. + * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), + * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP. + * @param bufferSize Size of buffer for read/write - pass 0 if you want + * to use the default configured values. + * @param replication Block replication - pass 0 if you want to use + * the default configured values. + * @param blocksize Size of block - pass 0 if you want to use the + * default configured values. Note that if you want a block size bigger + * than 2 GB, you must use the hdfsStreamBuilder API rather than this + * deprecated function. + * @return Returns the handle to the open file or NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, + int bufferSize, short replication, tSize blocksize); + + /** + * hdfsStreamBuilderAlloc - Allocate an HDFS stream builder. + * + * @param fs The configured filesystem handle. + * @param path The full path to the file. Will be deep-copied. + * @param flags The open flags, as in hdfsOpenFile. + * @return Returns the hdfsStreamBuilder, or NULL on error. + */ + LIBHDFS_EXTERNAL + struct hdfsStreamBuilder *hdfsStreamBuilderAlloc(hdfsFS fs, + const char *path, int flags); + + /** + * hdfsStreamBuilderFree - Free an HDFS file builder. + * + * It is normally not necessary to call this function since + * hdfsStreamBuilderBuild frees the builder. + * + * @param bld The hdfsStreamBuilder to free. + */ + LIBHDFS_EXTERNAL + void hdfsStreamBuilderFree(struct hdfsStreamBuilder *bld); + + /** + * hdfsStreamBuilderSetBufferSize - Set the stream buffer size. + * + * @param bld The hdfs stream builder. + * @param bufferSize The buffer size to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetBufferSize(struct hdfsStreamBuilder *bld, + int32_t bufferSize); + + /** + * hdfsStreamBuilderSetReplication - Set the replication for the stream. + * This is only relevant for output streams, which will create new blocks. + * + * @param bld The hdfs stream builder. + * @param replication The replication to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + * If you call this on an input stream builder, you will get + * EINVAL, because this configuration is not relevant to input + * streams. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetReplication(struct hdfsStreamBuilder *bld, + int16_t replication); + + /** + * hdfsStreamBuilderSetDefaultBlockSize - Set the default block size for + * the stream. This is only relevant for output streams, which will create + * new blocks. + * + * @param bld The hdfs stream builder. + * @param defaultBlockSize The default block size to set. + * + * @return 0 on success, or -1 on error. Errno will be set on error. + * If you call this on an input stream builder, you will get + * EINVAL, because this configuration is not relevant to input + * streams. + */ + LIBHDFS_EXTERNAL + int hdfsStreamBuilderSetDefaultBlockSize(struct hdfsStreamBuilder *bld, + int64_t defaultBlockSize); + + /** + * hdfsStreamBuilderBuild - Build the stream by calling open or create. + * + * @param bld The hdfs stream builder. This pointer will be freed, whether + * or not the open succeeds. + * + * @return the stream pointer on success, or NULL on error. Errno will be + * set on error. + */ + LIBHDFS_EXTERNAL + hdfsFile hdfsStreamBuilderBuild(struct hdfsStreamBuilder *bld); + + /** + * hdfsTruncateFile - Truncate a hdfs file to given lenght. + * @param fs The configured filesystem handle. + * @param path The full path to the file. + * @param newlength The size the file is to be truncated to + * @return 1 if the file has been truncated to the desired newlength + * and is immediately available to be reused for write operations + * such as append. + * 0 if a background process of adjusting the length of the last + * block has been started, and clients should wait for it to + * complete before proceeding with further file updates. + * -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsTruncateFile(hdfsFS fs, const char* path, tOffset newlength); + + /** + * hdfsUnbufferFile - Reduce the buffering done on a file. + * + * @param file The file to unbuffer. + * @return 0 on success + * ENOTSUP if the file does not support unbuffering + * Errno will also be set to this value. + */ + LIBHDFS_EXTERNAL + int hdfsUnbufferFile(hdfsFile file); + + /** + * hdfsCloseFile - Close an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns 0 on success, -1 on error. + * On error, errno will be set appropriately. + * If the hdfs file was valid, the memory associated with it will + * be freed at the end of this call, even if there was an I/O + * error. + */ + LIBHDFS_EXTERNAL + int hdfsCloseFile(hdfsFS fs, hdfsFile file); + + + /** + * hdfsExists - Checks if a given path exsits on the filesystem + * @param fs The configured filesystem handle. + * @param path The path to look for + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsExists(hdfsFS fs, const char *path); + + + /** + * hdfsSeek - Seek to given offset in file. + * This works only for files opened in read-only mode. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param desiredPos Offset into the file to seek into. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); + + + /** + * hdfsTell - Get the current offset in the file, in bytes. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Current offset, -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsTell(hdfsFS fs, hdfsFile file); + + + /** + * hdfsRead - Read data from an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return On success, a positive number indicating how many bytes + * were read. + * On end-of-file, 0. + * On error, -1. Errno will be set to the error code. + * Just like the POSIX read function, hdfsRead will return -1 + * and set errno to EINTR if data is temporarily unavailable, + * but we are not yet at the end of the file. + */ + LIBHDFS_EXTERNAL + tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length); + + /** + * hdfsPread - Positional read of data from an open file. Reads up to the + * number of specified bytes in length. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param position Position from which to read + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return See hdfsRead + */ + LIBHDFS_EXTERNAL + tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, + void* buffer, tSize length); + + /** + * hdfsPreadFully - Positional read of data from an open file. Reads the + * number of specified bytes in length, or until the end of the data is + * reached. Unlike hdfsRead and hdfsPread, this method does not return + * the number of bytes read because either (1) the entire length of the + * buffer is filled, or (2) the end of the file is reached. If the eof is + * reached, an exception is thrown and errno is set to EINTR. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param position Position from which to read + * @param buffer The buffer to copy read bytes into. + * @param length The length of the buffer. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsPreadFully(hdfsFS fs, hdfsFile file, tOffset position, + void* buffer, tSize length); + + + /** + * hdfsWrite - Write data into an open file. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @param buffer The data. + * @param length The no. of bytes to write. + * @return Returns the number of bytes written, -1 on error. + */ + LIBHDFS_EXTERNAL + tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, + tSize length); + + + /** + * hdfsWrite - Flush the data. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsFlush(hdfsFS fs, hdfsFile file); + + + /** + * hdfsHFlush - Flush out the data in client's user buffer. After the + * return of this call, new readers will see the data. + * @param fs configured filesystem handle + * @param file file handle + * @return 0 on success, -1 on error and sets errno + */ + LIBHDFS_EXTERNAL + int hdfsHFlush(hdfsFS fs, hdfsFile file); + + + /** + * hdfsHSync - Similar to posix fsync, Flush out the data in client's + * user buffer. all the way to the disk device (but the disk may have + * it in its cache). + * @param fs configured filesystem handle + * @param file file handle + * @return 0 on success, -1 on error and sets errno + */ + LIBHDFS_EXTERNAL + int hdfsHSync(hdfsFS fs, hdfsFile file); + + + /** + * hdfsAvailable - Number of bytes that can be read from this + * input stream without blocking. + * @param fs The configured filesystem handle. + * @param file The file handle. + * @return Returns available bytes; -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsAvailable(hdfsFS fs, hdfsFile file); + + + /** + * hdfsCopy - Copy file from one filesystem to another. + * @param srcFS The handle to source filesystem. + * @param src The path of source file. + * @param dstFS The handle to destination filesystem. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); + + + /** + * hdfsMove - Move file from one filesystem to another. + * @param srcFS The handle to source filesystem. + * @param src The path of source file. + * @param dstFS The handle to destination filesystem. + * @param dst The path of destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); + + + /** + * hdfsDelete - Delete file. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @param recursive if path is a directory and set to + * non-zero, the directory is deleted else throws an exception. In + * case of a file the recursive argument is irrelevant. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsDelete(hdfsFS fs, const char* path, int recursive); + + /** + * hdfsRename - Rename file. + * @param fs The configured filesystem handle. + * @param oldPath The path of the source file. + * @param newPath The path of the destination file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath); + + + /** + * hdfsGetWorkingDirectory - Get the current working directory for + * the given filesystem. + * @param fs The configured filesystem handle. + * @param buffer The user-buffer to copy path of cwd into. + * @param bufferSize The length of user-buffer. + * @return Returns buffer, NULL on error. + */ + LIBHDFS_EXTERNAL + char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize); + + + /** + * hdfsSetWorkingDirectory - Set the working directory. All relative + * paths will be resolved relative to it. + * @param fs The configured filesystem handle. + * @param path The path of the new 'cwd'. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSetWorkingDirectory(hdfsFS fs, const char* path); + + + /** + * hdfsCreateDirectory - Make the given file and all non-existent + * parents into directories. + * @param fs The configured filesystem handle. + * @param path The path of the directory. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsCreateDirectory(hdfsFS fs, const char* path); + + + /** + * hdfsSetReplication - Set the replication of the specified + * file to the supplied value + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @return Returns 0 on success, -1 on error. + */ + LIBHDFS_EXTERNAL + int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication); + + + /** + * hdfsFileInfo - Information about a file/directory. + */ + typedef struct { + tObjectKind mKind; /* file or directory */ + char *mName; /* the name of the file */ + tTime mLastMod; /* the last modification time for the file in seconds */ + tOffset mSize; /* the size of the file in bytes */ + short mReplication; /* the count of replicas */ + tOffset mBlockSize; /* the block size for the file */ + char *mOwner; /* the owner of the file */ + char *mGroup; /* the group associated with the file */ + short mPermissions; /* the permissions associated with the file */ + tTime mLastAccess; /* the last access time for the file in seconds */ + } hdfsFileInfo; + + + /** + * hdfsListDirectory - Get list of files/directories for a given + * directory-path. hdfsFreeFileInfo should be called to deallocate memory. + * @param fs The configured filesystem handle. + * @param path The path of the directory. + * @param numEntries Set to the number of files/directories in path. + * @return Returns a dynamically-allocated array of hdfsFileInfo + * objects; NULL on error or empty directory. + * errno is set to non-zero on error or zero on success. + */ + LIBHDFS_EXTERNAL + hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, + int *numEntries); + + + /** + * hdfsGetPathInfo - Get information about a path as a (dynamically + * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be + * called when the pointer is no longer needed. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @return Returns a dynamically-allocated hdfsFileInfo object; + * NULL on error. + */ + LIBHDFS_EXTERNAL + hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path); + + + /** + * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) + * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo + * objects. + * @param numEntries The size of the array. + */ + LIBHDFS_EXTERNAL + void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries); + + /** + * hdfsFileIsEncrypted: determine if a file is encrypted based on its + * hdfsFileInfo. + * @return -1 if there was an error (errno will be set), 0 if the file is + * not encrypted, 1 if the file is encrypted. + */ + LIBHDFS_EXTERNAL + int hdfsFileIsEncrypted(hdfsFileInfo *hdfsFileInfo); + + + /** + * hdfsGetHosts - Get hostnames where a particular block (determined by + * pos & blocksize) of a file is stored. The last element in the array + * is NULL. Due to replication, a single block could be present on + * multiple hosts. + * @param fs The configured filesystem handle. + * @param path The path of the file. + * @param start The start of the block. + * @param length The length of the block. + * @return Returns a dynamically-allocated 2-d array of blocks-hosts; + * NULL on error. + */ + LIBHDFS_EXTERNAL + char*** hdfsGetHosts(hdfsFS fs, const char* path, + tOffset start, tOffset length); + + + /** + * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts + * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo + * objects. + * @param numEntries The size of the array. + */ + LIBHDFS_EXTERNAL + void hdfsFreeHosts(char ***blockHosts); + + + /** + * hdfsGetDefaultBlockSize - Get the default blocksize. + * + * @param fs The configured filesystem handle. + * @deprecated Use hdfsGetDefaultBlockSizeAtPath instead. + * + * @return Returns the default blocksize, or -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetDefaultBlockSize(hdfsFS fs); + + + /** + * hdfsGetDefaultBlockSizeAtPath - Get the default blocksize at the + * filesystem indicated by a given path. + * + * @param fs The configured filesystem handle. + * @param path The given path will be used to locate the actual + * filesystem. The full path does not have to exist. + * + * @return Returns the default blocksize, or -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetDefaultBlockSizeAtPath(hdfsFS fs, const char *path); + + + /** + * hdfsGetCapacity - Return the raw capacity of the filesystem. + * @param fs The configured filesystem handle. + * @return Returns the raw-capacity; -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetCapacity(hdfsFS fs); + + + /** + * hdfsGetUsed - Return the total raw size of all files in the filesystem. + * @param fs The configured filesystem handle. + * @return Returns the total-size; -1 on error. + */ + LIBHDFS_EXTERNAL + tOffset hdfsGetUsed(hdfsFS fs); + + /** + * Change the user and/or group of a file or directory. + * + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param owner User string. Set to NULL for 'no change' + * @param group Group string. Set to NULL for 'no change' + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsChown(hdfsFS fs, const char* path, const char *owner, + const char *group); + + /** + * hdfsChmod + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param mode the bitmask to set it to + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsChmod(hdfsFS fs, const char* path, short mode); + + /** + * hdfsUtime + * @param fs The configured filesystem handle. + * @param path the path to the file or directory + * @param mtime new modification time or -1 for no change + * @param atime new access time or -1 for no change + * @return 0 on success else -1 + */ + LIBHDFS_EXTERNAL + int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime); + + /** + * Allocate a zero-copy options structure. + * + * You must free all options structures allocated with this function using + * hadoopRzOptionsFree. + * + * @return A zero-copy options structure, or NULL if one could + * not be allocated. If NULL is returned, errno will + * contain the error number. + */ + LIBHDFS_EXTERNAL + struct hadoopRzOptions *hadoopRzOptionsAlloc(void); + + /** + * Determine whether we should skip checksums in read0. + * + * @param opts The options structure. + * @param skip Nonzero to skip checksums sometimes; zero to always + * check them. + * + * @return 0 on success; -1 plus errno on failure. + */ + LIBHDFS_EXTERNAL + int hadoopRzOptionsSetSkipChecksum( + struct hadoopRzOptions *opts, int skip); + + /** + * Set the ByteBufferPool to use with read0. + * + * @param opts The options structure. + * @param className If this is NULL, we will not use any + * ByteBufferPool. If this is non-NULL, it will be + * treated as the name of the pool class to use. + * For example, you can use + * ELASTIC_BYTE_BUFFER_POOL_CLASS. + * + * @return 0 if the ByteBufferPool class was found and + * instantiated; + * -1 plus errno otherwise. + */ + LIBHDFS_EXTERNAL + int hadoopRzOptionsSetByteBufferPool( + struct hadoopRzOptions *opts, const char *className); + + /** + * Free a hadoopRzOptionsFree structure. + * + * @param opts The options structure to free. + * Any associated ByteBufferPool will also be freed. + */ + LIBHDFS_EXTERNAL + void hadoopRzOptionsFree(struct hadoopRzOptions *opts); + + /** + * Perform a byte buffer read. + * If possible, this will be a zero-copy (mmap) read. + * + * @param file The file to read from. + * @param opts An options structure created by hadoopRzOptionsAlloc. + * @param maxLength The maximum length to read. We may read fewer bytes + * than this length. + * + * @return On success, we will return a new hadoopRzBuffer. + * This buffer will continue to be valid and readable + * until it is released by readZeroBufferFree. Failure to + * release a buffer will lead to a memory leak. + * You can access the data within the hadoopRzBuffer with + * hadoopRzBufferGet. If you have reached EOF, the data + * within the hadoopRzBuffer will be NULL. You must still + * free hadoopRzBuffer instances containing NULL. + * + * On failure, we will return NULL plus an errno code. + * errno = EOPNOTSUPP indicates that we could not do a + * zero-copy read, and there was no ByteBufferPool + * supplied. + */ + LIBHDFS_EXTERNAL + struct hadoopRzBuffer* hadoopReadZero(hdfsFile file, + struct hadoopRzOptions *opts, int32_t maxLength); + + /** + * Determine the length of the buffer returned from readZero. + * + * @param buffer a buffer returned from readZero. + * @return the length of the buffer. + */ + LIBHDFS_EXTERNAL + int32_t hadoopRzBufferLength(const struct hadoopRzBuffer *buffer); + + /** + * Get a pointer to the raw buffer returned from readZero. + * + * To find out how many bytes this buffer contains, call + * hadoopRzBufferLength. + * + * @param buffer a buffer returned from readZero. + * @return a pointer to the start of the buffer. This will be + * NULL when end-of-file has been reached. + */ + LIBHDFS_EXTERNAL + const void *hadoopRzBufferGet(const struct hadoopRzBuffer *buffer); + + /** + * Release a buffer obtained through readZero. + * + * @param file The hdfs stream that created this buffer. This must be + * the same stream you called hadoopReadZero on. + * @param buffer The buffer to release. + */ + LIBHDFS_EXTERNAL + void hadoopRzBufferFree(hdfsFile file, struct hadoopRzBuffer *buffer); + + /** + * Get the last exception root cause that happened in the context of the + * current thread, i.e. the thread that called into libHDFS. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call into libHDFS by the current thread. + * Users of this function should not free the pointer. + * + * A NULL will be returned if no exception information could be retrieved + * for the previous call. + * + * @return The root cause as a C-string. + */ + LIBHDFS_EXTERNAL + char* hdfsGetLastExceptionRootCause(); + + /** + * Get the last exception stack trace that happened in the context of the + * current thread, i.e. the thread that called into libHDFS. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call into libHDFS by the current thread. + * Users of this function should not free the pointer. + * + * A NULL will be returned if no exception information could be retrieved + * for the previous call. + * + * @return The stack trace as a C-string. + */ + LIBHDFS_EXTERNAL + char* hdfsGetLastExceptionStackTrace(); + +#ifdef __cplusplus +} +#endif + +#undef LIBHDFS_EXTERNAL +#endif /*LIBHDFS_HDFS_H*/ + +/** + * vim: ts=4: sw=4: et + */ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c new file mode 100644 index 00000000000..cf880e91b75 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.c @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "os/mutexes.h" + +#include + +/** + * Whether initCachedClasses has been called or not. Protected by the mutex + * jclassInitMutex. + */ +static int jclassesInitialized = 0; + +typedef struct { + jclass javaClass; + const char *className; +} javaClassAndName; + +/** + * A collection of commonly used jclass objects that are used throughout + * libhdfs. The jclasses are loaded immediately after the JVM is created (see + * initCachedClasses). The array is indexed using CachedJavaClass. + */ +javaClassAndName cachedJavaClasses[NUM_CACHED_CLASSES]; + +/** + * Helper method that creates and sets a jclass object given a class name. + * Returns a jthrowable on error, NULL otherwise. + */ +static jthrowable initCachedClass(JNIEnv *env, const char *className, + jclass *cachedJclass) { + assert(className != NULL && "Found a CachedJavaClass without a class " + "name"); + jthrowable jthr = NULL; + jclass tempLocalClassRef; + tempLocalClassRef = (*env)->FindClass(env, className); + if (!tempLocalClassRef) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + *cachedJclass = (jclass) (*env)->NewGlobalRef(env, tempLocalClassRef); + if (!*cachedJclass) { + jthr = getPendingExceptionAndClear(env); + goto done; + } +done: + destroyLocalReference(env, tempLocalClassRef); + return jthr; +} + +jthrowable initCachedClasses(JNIEnv* env) { + mutexLock(&jclassInitMutex); + if (!jclassesInitialized) { + // Set all the class names + cachedJavaClasses[JC_CONFIGURATION].className = + "org/apache/hadoop/conf/Configuration"; + cachedJavaClasses[JC_PATH].className = + "org/apache/hadoop/fs/Path"; + cachedJavaClasses[JC_FILE_SYSTEM].className = + "org/apache/hadoop/fs/FileSystem"; + cachedJavaClasses[JC_FS_STATUS].className = + "org/apache/hadoop/fs/FsStatus"; + cachedJavaClasses[JC_FILE_UTIL].className = + "org/apache/hadoop/fs/FileUtil"; + cachedJavaClasses[JC_BLOCK_LOCATION].className = + "org/apache/hadoop/fs/BlockLocation"; + cachedJavaClasses[JC_DFS_HEDGED_READ_METRICS].className = + "org/apache/hadoop/hdfs/DFSHedgedReadMetrics"; + cachedJavaClasses[JC_DISTRIBUTED_FILE_SYSTEM].className = + "org/apache/hadoop/hdfs/DistributedFileSystem"; + cachedJavaClasses[JC_FS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataInputStream"; + cachedJavaClasses[JC_FS_DATA_OUTPUT_STREAM].className = + "org/apache/hadoop/fs/FSDataOutputStream"; + cachedJavaClasses[JC_FILE_STATUS].className = + "org/apache/hadoop/fs/FileStatus"; + cachedJavaClasses[JC_FS_PERMISSION].className = + "org/apache/hadoop/fs/permission/FsPermission"; + cachedJavaClasses[JC_READ_STATISTICS].className = + "org/apache/hadoop/hdfs/ReadStatistics"; + cachedJavaClasses[JC_HDFS_DATA_INPUT_STREAM].className = + "org/apache/hadoop/hdfs/client/HdfsDataInputStream"; + cachedJavaClasses[JC_DOMAIN_SOCKET].className = + "org/apache/hadoop/net/unix/DomainSocket"; + cachedJavaClasses[JC_URI].className = + "java/net/URI"; + cachedJavaClasses[JC_BYTE_BUFFER].className = + "java/nio/ByteBuffer"; + cachedJavaClasses[JC_ENUM_SET].className = + "java/util/EnumSet"; + cachedJavaClasses[JC_EXCEPTION_UTILS].className = + "org/apache/commons/lang3/exception/ExceptionUtils"; + + // Create and set the jclass objects based on the class names set above + jthrowable jthr; + int numCachedClasses = + sizeof(cachedJavaClasses) / sizeof(javaClassAndName); + for (int i = 0; i < numCachedClasses; i++) { + jthr = initCachedClass(env, cachedJavaClasses[i].className, + &cachedJavaClasses[i].javaClass); + if (jthr) { + mutexUnlock(&jclassInitMutex); + return jthr; + } + } + jclassesInitialized = 1; + } + mutexUnlock(&jclassInitMutex); + return NULL; +} + +jclass getJclass(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].javaClass; +} + +const char *getClassName(CachedJavaClass cachedJavaClass) { + return cachedJavaClasses[cachedJavaClass].className; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h new file mode 100644 index 00000000000..92cdd542e23 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jclasses.h @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JCLASSES_H +#define LIBHDFS_JCLASSES_H + +#include + +/** + * Encapsulates logic to cache jclass objects so they can re-used across + * calls to FindClass. Creating jclass objects every time libhdfs has to + * invoke a method can hurt performance. By cacheing jclass objects we avoid + * this overhead. + * + * We use the term "cached" here loosely; jclasses are not truly cached, + * instead they are created once during JVM load and are kept alive until the + * process shutdowns. There is no eviction of jclass objects. + * + * @see https://www.ibm.com/developerworks/library/j-jni/index.html#notc + */ + +/** + * Each enum value represents one jclass that is cached. Enum values should + * be passed to getJclass or getName to get the jclass object or class name + * represented by the enum value. + */ +typedef enum { + JC_CONFIGURATION, + JC_PATH, + JC_FILE_SYSTEM, + JC_FS_STATUS, + JC_FILE_UTIL, + JC_BLOCK_LOCATION, + JC_DFS_HEDGED_READ_METRICS, + JC_DISTRIBUTED_FILE_SYSTEM, + JC_FS_DATA_INPUT_STREAM, + JC_FS_DATA_OUTPUT_STREAM, + JC_FILE_STATUS, + JC_FS_PERMISSION, + JC_READ_STATISTICS, + JC_HDFS_DATA_INPUT_STREAM, + JC_DOMAIN_SOCKET, + JC_URI, + JC_BYTE_BUFFER, + JC_ENUM_SET, + JC_EXCEPTION_UTILS, + // A special marker enum that counts the number of cached jclasses + NUM_CACHED_CLASSES +} CachedJavaClass; + +/** + * Internally initializes all jclass objects listed in the CachedJavaClass + * enum. This method is idempotent and thread-safe. + */ +jthrowable initCachedClasses(JNIEnv* env); + +/** + * Return the jclass object represented by the given CachedJavaClass + */ +jclass getJclass(CachedJavaClass cachedJavaClass); + +/** + * Return the class name represented by the given CachedJavaClass + */ +const char *getClassName(CachedJavaClass cachedJavaClass); + +/* Some frequently used HDFS class names */ +#define HADOOP_CONF "org/apache/hadoop/conf/Configuration" +#define HADOOP_PATH "org/apache/hadoop/fs/Path" +#define HADOOP_LOCALFS "org/apache/hadoop/fs/LocalFileSystem" +#define HADOOP_FS "org/apache/hadoop/fs/FileSystem" +#define HADOOP_FSSTATUS "org/apache/hadoop/fs/FsStatus" +#define HADOOP_FILEUTIL "org/apache/hadoop/fs/FileUtil" +#define HADOOP_BLK_LOC "org/apache/hadoop/fs/BlockLocation" +#define HADOOP_DFS_HRM "org/apache/hadoop/hdfs/DFSHedgedReadMetrics" +#define HADOOP_DFS "org/apache/hadoop/hdfs/DistributedFileSystem" +#define HADOOP_FSDISTRM "org/apache/hadoop/fs/FSDataInputStream" +#define HADOOP_FSDOSTRM "org/apache/hadoop/fs/FSDataOutputStream" +#define HADOOP_FILESTAT "org/apache/hadoop/fs/FileStatus" +#define HADOOP_FSPERM "org/apache/hadoop/fs/permission/FsPermission" +#define HADOOP_RSTAT "org/apache/hadoop/hdfs/ReadStatistics" +#define HADOOP_HDISTRM "org/apache/hadoop/hdfs/client/HdfsDataInputStream" +#define HADOOP_RO "org/apache/hadoop/fs/ReadOption" +#define HADOOP_DS "org/apache/hadoop/net/unix/DomainSocket" + +/* Some frequently used Java class names */ +#define JAVA_NET_ISA "java/net/InetSocketAddress" +#define JAVA_NET_URI "java/net/URI" +#define JAVA_BYTEBUFFER "java/nio/ByteBuffer" +#define JAVA_STRING "java/lang/String" +#define JAVA_ENUMSET "java/util/EnumSet" + +/* Some frequently used third-party class names */ + +#define EXCEPTION_UTILS "org/apache/commons/lang3/exception/ExceptionUtils" + +#endif /*LIBHDFS_JCLASSES_H*/ diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c new file mode 100644 index 00000000000..badb041df77 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.c @@ -0,0 +1,994 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The change here is that getGlobalJNIEnv reports + * whether it attached the current thread, and reuses an existing attachment + * instead of calling AttachCurrentThread on a thread that already has one. + */ + +#include "config.h" +#include "exception.h" +#include "jclasses.h" +#include "jni_helper.h" +#include "platform.h" +#include "os/mutexes.h" +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +/** The Native return types that methods could return */ +#define JVOID 'V' +#define JOBJECT 'L' +#define JARRAYOBJECT '[' +#define JBOOLEAN 'Z' +#define JBYTE 'B' +#define JCHAR 'C' +#define JSHORT 'S' +#define JINT 'I' +#define JLONG 'J' +#define JFLOAT 'F' +#define JDOUBLE 'D' + +/** + * Length of buffer for retrieving created JVMs. (We only ever create one.) + */ +#define VM_BUF_LENGTH 1 + +void destroyLocalReference(JNIEnv *env, jobject jObject) +{ + if (jObject) + (*env)->DeleteLocalRef(env, jObject); +} + +static jthrowable validateMethodType(JNIEnv *env, MethType methType) +{ + if (methType != STATIC && methType != INSTANCE) { + return newRuntimeError(env, "validateMethodType(methType=%d): " + "illegal method type.\n", methType); + } + return NULL; +} + +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out) +{ + jstring jstr; + + if (!str) { + /* Can't pass NULL to NewStringUTF: the result would be + * implementation-defined. */ + *out = NULL; + return NULL; + } + jstr = (*env)->NewStringUTF(env, str); + if (!jstr) { + /* If NewStringUTF returns NULL, an exception has been thrown, + * which we need to handle. Probaly an OOM. */ + return getPendingExceptionAndClear(env); + } + *out = jstr; + return NULL; +} + +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out) +{ + const char *tmp; + + if (!jstr) { + *out = NULL; + return NULL; + } + tmp = (*env)->GetStringUTFChars(env, jstr, NULL); + if (!tmp) { + return getPendingExceptionAndClear(env); + } + *out = strdup(tmp); + (*env)->ReleaseStringUTFChars(env, jstr, tmp); + return NULL; +} + +/** + * Does the work to actually execute a Java method. Takes in an existing jclass + * object and a va_list of arguments for the Java method to be invoked. + */ +static jthrowable invokeMethodOnJclass(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, jclass cls, const char *className, + const char *methName, const char *methSignature, va_list args) +{ + jmethodID mid; + jthrowable jthr; + const char *str; + char returnType; + + jthr = methodIdFromClass(cls, className, methName, methSignature, methType, + env, &mid); + if (jthr) + return jthr; + str = methSignature; + while (*str != ')') str++; + str++; + returnType = *str; + if (returnType == JOBJECT || returnType == JARRAYOBJECT) { + jobject jobj = NULL; + if (methType == STATIC) { + jobj = (*env)->CallStaticObjectMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jobj = (*env)->CallObjectMethodV(env, instObj, mid, args); + } + retval->l = jobj; + } + else if (returnType == JVOID) { + if (methType == STATIC) { + (*env)->CallStaticVoidMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + (*env)->CallVoidMethodV(env, instObj, mid, args); + } + } + else if (returnType == JBOOLEAN) { + jboolean jbool = 0; + if (methType == STATIC) { + jbool = (*env)->CallStaticBooleanMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jbool = (*env)->CallBooleanMethodV(env, instObj, mid, args); + } + retval->z = jbool; + } + else if (returnType == JSHORT) { + jshort js = 0; + if (methType == STATIC) { + js = (*env)->CallStaticShortMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + js = (*env)->CallShortMethodV(env, instObj, mid, args); + } + retval->s = js; + } + else if (returnType == JLONG) { + jlong jl = -1; + if (methType == STATIC) { + jl = (*env)->CallStaticLongMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + jl = (*env)->CallLongMethodV(env, instObj, mid, args); + } + retval->j = jl; + } + else if (returnType == JINT) { + jint ji = -1; + if (methType == STATIC) { + ji = (*env)->CallStaticIntMethodV(env, cls, mid, args); + } + else if (methType == INSTANCE) { + ji = (*env)->CallIntMethodV(env, instObj, mid, args); + } + retval->i = ji; + } + + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + return jthr; + } + return NULL; +} + +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...) +{ + jclass cls = NULL; + jthrowable jthr = NULL; + + va_list args; + va_start(args, methSignature); + + jthr = validateMethodType(env, methType); + if (jthr) { + goto done; + } + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, cls, + className, methName, methSignature, args); + +done: + va_end(args); + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...) +{ + jthrowable jthr; + + va_list args; + va_start(args, methSignature); + + jthr = invokeMethodOnJclass(env, retval, methType, instObj, + getJclass(class), getClassName(class), methName, methSignature, + args); + + va_end(args); + return jthr; +} + +static jthrowable constructNewObjectOfJclass(JNIEnv *env, + jobject *out, jclass cls, const char *className, + const char *ctorSignature, va_list args) { + jmethodID mid; + jobject jobj; + jthrowable jthr; + + jthr = methodIdFromClass(cls, className, "", ctorSignature, INSTANCE, + env, &mid); + if (jthr) + return jthr; + jobj = (*env)->NewObjectV(env, cls, mid, args); + if (!jobj) + return getPendingExceptionAndClear(env); + *out = jobj; + return NULL; +} + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...) +{ + va_list args; + jclass cls; + jthrowable jthr = NULL; + + cls = (*env)->FindClass(env, className); + if (!cls) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + + va_start(args, ctorSignature); + jthr = constructNewObjectOfJclass(env, out, cls, className, + ctorSignature, args); + va_end(args); +done: + destroyLocalReference(env, cls); + return jthr; +} + +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...) +{ + jthrowable jthr = NULL; + va_list args; + va_start(args, ctorSignature); + + jthr = constructNewObjectOfJclass(env, out, + getJclass(cachedJavaClass), getClassName(cachedJavaClass), + ctorSignature, args); + + va_end(args); + return jthr; +} + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out) +{ + jthrowable jthr; + jmethodID mid = 0; + + jthr = validateMethodType(env, methType); + if (jthr) + return jthr; + if (methType == STATIC) { + mid = (*env)->GetStaticMethodID(env, cls, methName, methSignature); + } + else if (methType == INSTANCE) { + mid = (*env)->GetMethodID(env, cls, methName, methSignature); + } + if (mid == NULL) { + fprintf(stderr, "could not find method %s from class %s with " + "signature %s\n", methName, className, methSignature); + return getPendingExceptionAndClear(env); + } + *out = mid; + return NULL; +} + +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name) +{ + jthrowable jthr; + jclass cls, clsClass = NULL; + jmethodID mid; + jstring str = NULL; + const char *cstr = NULL; + char *newstr; + + cls = (*env)->GetObjectClass(env, jobj); + if (cls == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + clsClass = (*env)->FindClass(env, "java/lang/Class"); + if (clsClass == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + mid = (*env)->GetMethodID(env, clsClass, "getName", "()Ljava/lang/String;"); + if (mid == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + str = (*env)->CallObjectMethod(env, cls, mid); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionClear(env); + goto done; + } + if (str == NULL) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + cstr = (*env)->GetStringUTFChars(env, str, NULL); + if (!cstr) { + jthr = getPendingExceptionAndClear(env); + goto done; + } + newstr = strdup(cstr); + if (newstr == NULL) { + jthr = newRuntimeError(env, "classNameOfObject: out of memory"); + goto done; + } + *name = newstr; + jthr = NULL; + +done: + destroyLocalReference(env, cls); + destroyLocalReference(env, clsClass); + if (str) { + if (cstr) + (*env)->ReleaseStringUTFChars(env, str, cstr); + (*env)->DeleteLocalRef(env, str); + } + return jthr; +} + +/** + * For the given path, expand it by filling in with all *.jar or *.JAR files, + * separated by PATH_SEPARATOR. Assumes that expanded is big enough to hold the + * string, eg allocated after using this function with expanded=NULL to get the + * right size. Also assumes that the path ends with a "/.". The length of the + * expanded path is returned, which includes space at the end for either a + * PATH_SEPARATOR or null terminator. + */ +static ssize_t wildcard_expandPath(const char* path, char* expanded) +{ + struct dirent* file; + char* dest = expanded; + ssize_t length = 0; + size_t pathLength = strlen(path); + DIR* dir; + + dir = opendir(path); + if (dir != NULL) { + // can open dir so try to match with all *.jar and *.JAR entries + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath: %s\n", path); +#endif + + errno = 0; + while ((file = readdir(dir)) != NULL) { + const char* filename = file->d_name; + const size_t filenameLength = strlen(filename); + const char* jarExtension; + + // If filename is smaller than 4 characters then it can not possibly + // have extension ".jar" or ".JAR" + if (filenameLength < 4) { + continue; + } + + jarExtension = &filename[filenameLength-4]; + if ((strcmp(jarExtension, ".jar") == 0) || + (strcmp(jarExtension, ".JAR") == 0)) { + + // pathLength includes an extra '.' which we'll use for either + // separator or null termination + length += pathLength + filenameLength; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_scanPath:\t%s\t:\t%zd\n", filename, length); +#endif + + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength - 1; + strncpy(dest, filename, filenameLength); + dest += filenameLength; + *dest = PATH_SEPARATOR; + dest++; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("wildcard_expandPath:\t%s\t:\t%s\n", + filename, expanded); +#endif + } + } + } + + if (errno != 0) { + fprintf(stderr, "wildcard_expandPath: on readdir %s: %s\n", + path, strerror(errno)); + length = -1; + } + + if (closedir(dir) != 0) { + fprintf(stderr, "wildcard_expandPath: on closedir %s: %s\n", + path, strerror(errno)); + } + } else if ((errno != EACCES) && (errno != ENOENT) && (errno != ENOTDIR)) { + // can not opendir due to an error we can not handle + fprintf(stderr, "wildcard_expandPath: on opendir %s: %s\n", path, + strerror(errno)); + length = -1; + } + + if (length == 0) { + // either we failed to open dir due to EACCESS, ENOENT, or ENOTDIR, or + // we did not find any file that matches *.jar or *.JAR + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + fprintf(stderr, "wildcard_expandPath: can not expand %.*s*: %s\n", + (int)(pathLength-1), path, strerror(errno)); +#endif + + // in this case, the wildcard expansion is the same as the original + // +1 for PATH_SEPARTOR or null termination + length = pathLength + 1; + if (expanded != NULL) { + // pathLength includes an extra '.' + strncpy(dest, path, pathLength-1); + dest += pathLength-1; + *dest = '*'; // restore wildcard + dest++; + *dest = PATH_SEPARATOR; + dest++; + } + } + + return length; +} + +/** + * Helper to expand classpaths. Returns the total length of the expanded + * classpath. If expandedClasspath is not NULL, then fills that with the + * expanded classpath. It assumes that expandedClasspath is of correct size, eg + * allocated after using this function with expandedClasspath=NULL to get the + * right size. + */ +static ssize_t getClassPath_helper(const char *classpath, char* expandedClasspath) +{ + ssize_t length; + ssize_t retval; + char* expandedCP_curr; + char* cp_token; + char* classpath_dup; + + classpath_dup = strdup(classpath); + if (classpath_dup == NULL) { + fprintf(stderr, "getClassPath_helper: failed strdup: %s\n", + strerror(errno)); + return -1; + } + + length = 0; + + // expandedCP_curr is the current pointer + expandedCP_curr = expandedClasspath; + + cp_token = strtok(classpath_dup, PATH_SEPARATOR_STR); + while (cp_token != NULL) { + size_t tokenlen; + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("%s\n", cp_token); +#endif + + tokenlen = strlen(cp_token); + // We only expand if token ends with "/*" + if ((tokenlen > 1) && + (cp_token[tokenlen-1] == '*') && (cp_token[tokenlen-2] == '/')) { + // replace the '*' with '.' so that we don't have to allocate another + // string for passing to opendir() in wildcard_expandPath() + cp_token[tokenlen-1] = '.'; + retval = wildcard_expandPath(cp_token, expandedCP_curr); + if (retval < 0) { + free(classpath_dup); + return -1; + } + + length += retval; + if (expandedCP_curr != NULL) { + expandedCP_curr += retval; + } + } else { + // +1 for path separator or null terminator + length += tokenlen + 1; + if (expandedCP_curr != NULL) { + strncpy(expandedCP_curr, cp_token, tokenlen); + expandedCP_curr += tokenlen; + *expandedCP_curr = PATH_SEPARATOR; + expandedCP_curr++; + } + } + + cp_token = strtok(NULL, PATH_SEPARATOR_STR); + } + + // Fix the last ':' and use it to null terminate + if (expandedCP_curr != NULL) { + expandedCP_curr--; + *expandedCP_curr = '\0'; + } + + free(classpath_dup); + return length; +} + +/** + * Gets the classpath. Wild card entries are resolved only if the entry ends + * with "/\*" (backslash to escape commenting) to match against .jar and .JAR. + * All other wild card entries (eg /path/to/dir/\*foo*) are not resolved, + * following JAVA default behavior, see: + * https://docs.oracle.com/javase/8/docs/technotes/tools/unix/classpath.html + */ +static char* getClassPath() +{ + char* classpath; + char* expandedClasspath; + ssize_t length; + ssize_t retval; + + classpath = getenv("CLASSPATH"); + if (classpath == NULL) { + return NULL; + } + + // First, get the total size of the string we will need for the expanded + // classpath + length = getClassPath_helper(classpath, NULL); + if (length < 0) { + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("+++++++++++++++++\n"); +#endif + + // we don't have to do anything if classpath has no valid wildcards + // we get length = 0 when CLASSPATH is set but empty + // if CLASSPATH is not empty, then length includes null terminator + // if length of expansion is same as original, then return a duplicate of + // original since expansion can only be longer + if ((length == 0) || ((length - 1) == strlen(classpath))) { + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + if ((length == 0) && (strlen(classpath) != 0)) { + fprintf(stderr, "Something went wrong with getting the wildcard \ + expansion length\n" ); + } +#endif + + expandedClasspath = strdup(classpath); + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; + } + + // Allocte memory for expanded classpath string + expandedClasspath = calloc(length, sizeof(char)); + if (expandedClasspath == NULL) { + fprintf(stderr, "getClassPath: failed calloc: %s\n", strerror(errno)); + return NULL; + } + + // Actual expansion + retval = getClassPath_helper(classpath, expandedClasspath); + if (retval < 0) { + free(expandedClasspath); + return NULL; + } + + // This should not happen, but dotting i's and crossing t's + if (retval != length) { + fprintf(stderr, + "Expected classpath expansion length to be %zu but instead got %zu\n", + length, retval); + free(expandedClasspath); + return NULL; + } + +#ifdef _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + printf("===============\n"); + printf("Allocated %zd for expanding classpath\n", length); + printf("Used %zu for expanding classpath\n", strlen(expandedClasspath) + 1); + printf("Expanded classpath=%s\n", expandedClasspath); +#endif + + return expandedClasspath; +} + + +/** + * Get the global JNI environemnt. + * + * We only have to create the JVM once. After that, we can use it in + * every thread. You must be holding the jvmMutex when you call this + * function. + * + * @param[out] attachedByLibhdfs Set to true if this call attached the current + * thread to the JVM, false if the thread was + * already attached by someone else. Only the + * former may be detached at thread exit. + * + * @return The JNIEnv on success; error code otherwise + */ +static JNIEnv* getGlobalJNIEnv(bool *attachedByLibhdfs) +{ + JavaVM* vmBuf[VM_BUF_LENGTH]; + JNIEnv *env; + jint rv = 0; + jint noVMs = 0; + jthrowable jthr; + char *hadoopClassPath; + const char *hadoopClassPathVMArg = "-Djava.class.path="; + size_t optHadoopClassPathLen; + char *optHadoopClassPath; + int noArgs = 1; + char *hadoopJvmArgs; + char jvmArgDelims[] = " "; + char *str, *token, *savePtr; + JavaVMInitArgs vm_args; + JavaVM *vm; + JavaVMOption *options; + + *attachedByLibhdfs = false; + rv = JNI_GetCreatedJavaVMs(&(vmBuf[0]), VM_BUF_LENGTH, &noVMs); + if (rv != 0) { + fprintf(stderr, "JNI_GetCreatedJavaVMs failed with error: %d\n", rv); + return NULL; + } + + if (noVMs == 0) { + //Get the environment variables for initializing the JVM + hadoopClassPath = getClassPath(); + if (hadoopClassPath == NULL) { + fprintf(stderr, "Environment variable CLASSPATH not set!\n"); + return NULL; + } + optHadoopClassPathLen = strlen(hadoopClassPath) + + strlen(hadoopClassPathVMArg) + 1; + optHadoopClassPath = malloc(sizeof(char)*optHadoopClassPathLen); + snprintf(optHadoopClassPath, optHadoopClassPathLen, + "%s%s", hadoopClassPathVMArg, hadoopClassPath); + + free(hadoopClassPath); + + // Determine the # of LIBHDFS_OPTS args + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + } + free(hadoopJvmArgs); + } + + // Now that we know the # args, populate the options array + options = calloc(noArgs, sizeof(JavaVMOption)); + if (!options) { + fputs("Call to calloc failed\n", stderr); + free(optHadoopClassPath); + return NULL; + } + options[0].optionString = optHadoopClassPath; + hadoopJvmArgs = getenv("LIBHDFS_OPTS"); + if (hadoopJvmArgs != NULL) { + hadoopJvmArgs = strdup(hadoopJvmArgs); + for (noArgs = 1, str = hadoopJvmArgs; ; noArgs++, str = NULL) { + token = strtok_r(str, jvmArgDelims, &savePtr); + if (NULL == token) { + break; + } + options[noArgs].optionString = token; + } + } + + //Create the VM + vm_args.version = JNI_VERSION_1_2; + vm_args.options = options; + vm_args.nOptions = noArgs; + vm_args.ignoreUnrecognized = 1; + + rv = JNI_CreateJavaVM(&vm, (void*)&env, &vm_args); + + if (hadoopJvmArgs != NULL) { + free(hadoopJvmArgs); + } + free(optHadoopClassPath); + free(options); + + if (rv != 0) { + fprintf(stderr, "Call to JNI_CreateJavaVM failed " + "with error: %d\n", rv); + return NULL; + } + *attachedByLibhdfs = true; + + // We use findClassAndInvokeMethod here because the jclasses in + // jclasses.h have not loaded yet + jthr = findClassAndInvokeMethod(env, NULL, STATIC, NULL, HADOOP_FS, + "loadFileSystems", "()V"); + if (jthr) { + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "FileSystem: loadFileSystems failed"); + return NULL; + } + } else { + vm = vmBuf[0]; + // Reuse an existing attachment rather than creating one. On a thread + // the JVM or the embedding application already attached, + // AttachCurrentThread succeeds and hands back the same JNIEnv, which + // would leave libhdfs believing it owns an attachment it did not make + // and detaching it at thread exit. Comet attaches its own Tokio worker + // threads, so that is the common case here rather than a corner one. + rv = (*vm)->GetEnv(vm, (void**)&env, JNI_VERSION_1_2); + if (rv == JNI_OK) { + return env; + } + if (rv != JNI_EDETACHED) { + fprintf(stderr, "Call to GetEnv failed with error: %d\n", rv); + return NULL; + } + //Attach this thread to the VM + rv = (*vm)->AttachCurrentThread(vm, (void*)&env, 0); + if (rv != 0) { + fprintf(stderr, "Call to AttachCurrentThread " + "failed with error: %d\n", rv); + return NULL; + } + *attachedByLibhdfs = true; + } + + return env; +} + +/** + * getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * + * Implementation note: we rely on POSIX thread-local storage (tls). + * This allows us to associate a destructor function with each thread, that + * will detach the thread from the Java VM when the thread terminates. If we + * failt to do this, it will cause a memory leak. + * + * However, POSIX TLS is not the most efficient way to do things. It requires a + * key to be initialized before it can be used. Since we don't know if this key + * is initialized at the start of this function, we have to lock a mutex first + * and check. Luckily, most operating systems support the more efficient + * __thread construct, which is initialized by the linker. + * + * @param: None. + * @return The JNIEnv* corresponding to the thread. + */ +JNIEnv* getJNIEnv(void) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (state) return state->env; + + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + if (state) { + mutexUnlock(&jvmMutex); + + // Free any stale exception strings. + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = NULL; + state->lastExceptionStackTrace = NULL; + + return state->env; + } + + /* Create a ThreadLocalState for this thread */ + state = threadLocalStorageCreate(); + if (!state) { + mutexUnlock(&jvmMutex); + fprintf(stderr, "getJNIEnv: Unable to create ThreadLocalState\n"); + return NULL; + } + if (threadLocalStorageSet(state)) { + mutexUnlock(&jvmMutex); + goto fail; + } + THREAD_LOCAL_STORAGE_SET_QUICK(state); + + state->env = getGlobalJNIEnv(&state->attachedByLibhdfs); + mutexUnlock(&jvmMutex); + + if (!state->env) { + goto fail; + } + + jthrowable jthr = NULL; + jthr = initCachedClasses(state->env); + if (jthr) { + printExceptionAndFree(state->env, jthr, PRINT_EXC_ALL, + "initCachedClasses failed"); + goto fail; + } + return state->env; + +fail: + fprintf(stderr, "getJNIEnv: getGlobalJNIEnv failed\n"); + hdfsThreadDestructor(state); + return NULL; +} + +char* getLastTLSExceptionRootCause() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionRootCause; +} + +char* getLastTLSExceptionStackTrace() +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return NULL; + } + mutexUnlock(&jvmMutex); + } + return state->lastExceptionStackTrace; +} + +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace) +{ + struct ThreadLocalState *state = NULL; + THREAD_LOCAL_STORAGE_GET_QUICK(&state); + if (!state) { + mutexLock(&jvmMutex); + if (threadLocalStorageGet(&state)) { + mutexUnlock(&jvmMutex); + return; + } + mutexUnlock(&jvmMutex); + } + + free(state->lastExceptionRootCause); + free(state->lastExceptionStackTrace); + state->lastExceptionRootCause = (char*)rootCause; + state->lastExceptionStackTrace = (char*)stackTrace; +} + +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name) +{ + jclass clazz; + int ret; + + clazz = (*env)->FindClass(env, name); + if (!clazz) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "javaObjectIsOfClass(%s)", name); + return -1; + } + ret = (*env)->IsInstanceOf(env, obj, clazz); + (*env)->DeleteLocalRef(env, clazz); + return ret == JNI_TRUE ? 1 : 0; +} + +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value) +{ + jthrowable jthr; + jstring jkey = NULL, jvalue = NULL; + + jthr = newJavaStr(env, key, &jkey); + if (jthr) + goto done; + jthr = newJavaStr(env, value, &jvalue); + if (jthr) + goto done; + jthr = invokeMethod(env, NULL, INSTANCE, jConfiguration, + JC_CONFIGURATION, "set", "(Ljava/lang/String;Ljava/lang/String;)V", + jkey, jvalue); + if (jthr) + goto done; +done: + (*env)->DeleteLocalRef(env, jkey); + (*env)->DeleteLocalRef(env, jvalue); + return jthr; +} + +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out) +{ + jclass clazz; + jfieldID fieldId; + jobject jEnum; + char prettyClass[256]; + + clazz = (*env)->FindClass(env, className); + if (!clazz) { + return getPendingExceptionAndClear(env); + } + if (snprintf(prettyClass, sizeof(prettyClass), "L%s;", className) + >= sizeof(prettyClass)) { + return newRuntimeError(env, "fetchEnum(%s, %s): class name too long.", + className, valueName); + } + fieldId = (*env)->GetStaticFieldID(env, clazz, valueName, prettyClass); + if (!fieldId) { + return getPendingExceptionAndClear(env); + } + jEnum = (*env)->GetStaticObjectField(env, clazz, fieldId); + if (!jEnum) { + return getPendingExceptionAndClear(env); + } + *out = jEnum; + return NULL; +} + diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h new file mode 100644 index 00000000000..41d6fab2a75 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/jni_helper.h @@ -0,0 +1,221 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_JNI_HELPER_H +#define LIBHDFS_JNI_HELPER_H + +#include "jclasses.h" + +#include +#include + +#include +#include +#include + +#ifdef WIN32 + #define PATH_SEPARATOR ';' + #define PATH_SEPARATOR_STR ";" +#else + #define PATH_SEPARATOR ':' + #define PATH_SEPARATOR_STR ":" +#endif + +// #define _LIBHDFS_JNI_HELPER_DEBUGGING_ON_ + +/** Denote the method we want to invoke as STATIC or INSTANCE */ +typedef enum { + STATIC, + INSTANCE +} MethType; + +/** + * Create a new malloc'ed C string from a Java string. + * + * @param env The JNI environment + * @param jstr The Java string + * @param out (out param) the malloc'ed C string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newCStr(JNIEnv *env, jstring jstr, char **out); + +/** + * Create a new Java string from a C string. + * + * @param env The JNI environment + * @param str The C string + * @param out (out param) the java string + * + * @return NULL on success; the exception otherwise + */ +jthrowable newJavaStr(JNIEnv *env, const char *str, jstring *out); + +/** + * Helper function to destroy a local reference of java.lang.Object + * @param env: The JNIEnv pointer. + * @param jFile: The local reference of java.lang.Object object + * @return None. + */ +void destroyLocalReference(JNIEnv *env, jobject jObject); + +/** invokeMethod: Invoke a Static or Instance method. + * methName: Name of the method + * methSignature: the signature of the method "(arg-types)ret-type" + * methType: The type of the method (STATIC or INSTANCE) + * instObj: Required if the methType is INSTANCE. The object to invoke + the method on. + * class: The CachedJavaClass to call the method on. + * env: The JNIEnv pointer + * retval: The pointer to a union type which will contain the result of the + method invocation, e.g. if the method returns an Object, retval will be + set to that, if the method returns boolean, retval will be set to the + value (JNI_TRUE or JNI_FALSE), etc. + * exc: If the methods throws any exception, this will contain the reference + * Arguments (the method arguments) must be passed after methSignature + * RETURNS: -1 on error and 0 on success. If -1 is returned, exc will have + a valid exception reference, and the result stored at retval is undefined. + */ +jthrowable invokeMethod(JNIEnv *env, jvalue *retval, MethType methType, + jobject instObj, CachedJavaClass class, + const char *methName, const char *methSignature, ...); + +/** + * findClassAndInvokeMethod: Same as invokeMethod, but it calls FindClass on + * the given className first and then calls invokeMethod. This method exists + * mainly for test infrastructure, any production code should use + * invokeMethod. Calling FindClass repeatedly can introduce performance + * overhead, so users should prefer invokeMethod and supply a CachedJavaClass. + */ +jthrowable findClassAndInvokeMethod(JNIEnv *env, jvalue *retval, + MethType methType, jobject instObj, const char *className, + const char *methName, const char *methSignature, ...); + +jthrowable constructNewObjectOfClass(JNIEnv *env, jobject *out, + const char *className, const char *ctorSignature, ...); + +/** + * Same as constructNewObjectOfClass but it takes in a CachedJavaClass + * rather than a className. This avoids an extra call to FindClass. + */ +jthrowable constructNewObjectOfCachedClass(JNIEnv *env, jobject *out, + CachedJavaClass cachedJavaClass, const char *ctorSignature, ...); + +jthrowable methodIdFromClass(jclass cls, const char *className, + const char *methName, const char *methSignature, MethType methType, + JNIEnv *env, jmethodID *out); + +/** classNameOfObject: Get an object's class name. + * @param jobj: The object. + * @param env: The JNIEnv pointer. + * @param name: (out param) On success, will contain a string containing the + * class name. This string must be freed by the caller. + * @return NULL on success, or the exception + */ +jthrowable classNameOfObject(jobject jobj, JNIEnv *env, char **name); + +/** getJNIEnv: A helper function to get the JNIEnv* for the given thread. + * It gets this from the ThreadLocalState if it exists. If a ThreadLocalState + * does not exist, one will be created. + * If no JVM exists, then one will be created. JVM command line arguments + * are obtained from the LIBHDFS_OPTS environment variable. + * @param: None. + * @return The JNIEnv* corresponding to the thread. + * */ +JNIEnv* getJNIEnv(void); + +/** + * Get the last exception root cause that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The root cause as a C-string. + */ +char* getLastTLSExceptionRootCause(); + +/** + * Get the last exception stack trace that happened in the context of the + * current thread. + * + * The pointer returned by this function is guaranteed to be valid until + * the next call to invokeMethod() by the current thread. + * Users of this function should not free the pointer. + * + * @return The stack trace as a C-string. + */ +char* getLastTLSExceptionStackTrace(); + +/** setTLSExceptionStrings: Sets the 'rootCause' and 'stackTrace' in the + * ThreadLocalState if one exists for the current thread. + * + * @param rootCause A string containing the root cause of an exception. + * @param stackTrace A string containing the stack trace of an exception. + * @return None. + */ +void setTLSExceptionStrings(const char *rootCause, const char *stackTrace); + +/** + * Figure out if a Java object is an instance of a particular class. + * + * @param env The Java environment. + * @param obj The object to check. + * @param name The class name to check. + * + * @return -1 if we failed to find the referenced class name. + * 0 if the object is not of the given class. + * 1 if the object is of the given class. + */ +int javaObjectIsOfClass(JNIEnv *env, jobject obj, const char *name); + +/** + * Set a value in a configuration object. + * + * @param env The JNI environment + * @param jConfiguration The configuration object to modify + * @param key The key to modify + * @param value The value to set the key to + * + * @return NULL on success; exception otherwise + */ +jthrowable hadoopConfSetStr(JNIEnv *env, jobject jConfiguration, + const char *key, const char *value); + +/** + * Fetch an instance of an Enum. + * + * @param env The JNI environment. + * @param className The enum class name. + * @param valueName The name of the enum value + * @param out (out param) on success, a local reference to an + * instance of the enum object. (Since Java enums are + * singletones, this is also the only instance.) + * + * @return NULL on success; exception otherwise + */ +jthrowable fetchEnumInstance(JNIEnv *env, const char *className, + const char *valueName, jobject *out); + +#endif /*LIBHDFS_JNI_HELPER_H*/ + +/** + * vim: ts=4: sw=4: et: + */ + diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h new file mode 100644 index 00000000000..92afabd7c75 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/mutexes.h @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_MUTEXES_H +#define LIBHDFS_MUTEXES_H + +/* + * Defines abstraction over platform-specific mutexes. libhdfs has no formal + * initialization function that users would call from a single-threaded context + * to initialize the library. This creates a challenge for bootstrapping the + * mutexes. To address this, all required mutexes are pre-defined here with + * external storage. Platform-specific implementations must guarantee that the + * mutexes are initialized via static initialization. + */ + +#include "platform.h" + +/** Mutex protecting singleton JVM instance. */ +extern mutex jvmMutex; + +/** Mutex protecting initialization of jclasses in jclasses.h. */ +extern mutex jclassInitMutex; + +/** + * Locks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexLock(mutex *m); + +/** + * Unlocks a mutex. + * + * @param m mutex + * @return 0 if successful, non-zero otherwise + */ +int mutexUnlock(mutex *m); + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c new file mode 100644 index 00000000000..5c6b429d5ec --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/mutexes.c @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/mutexes.h" + +#include +#include + +mutex jvmMutex; +mutex jclassInitMutex = PTHREAD_MUTEX_INITIALIZER; +pthread_mutexattr_t jvmMutexAttr; + +__attribute__((constructor)) static void init() { + pthread_mutexattr_init(&jvmMutexAttr); + pthread_mutexattr_settype(&jvmMutexAttr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&jvmMutex, &jvmMutexAttr); +} + +int mutexLock(mutex *m) { + int ret = pthread_mutex_lock(m); + if (ret) { + fprintf(stderr, "mutexLock: pthread_mutex_lock failed with error %d\n", + ret); + } + return ret; +} + +int mutexUnlock(mutex *m) { + int ret = pthread_mutex_unlock(m); + if (ret) { + fprintf(stderr, "mutexUnlock: pthread_mutex_unlock failed with error %d\n", + ret); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h new file mode 100644 index 00000000000..c63bbf9e0e0 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/platform.h @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_PLATFORM_H +#define LIBHDFS_PLATFORM_H + +#include + +/* Use gcc type-checked format arguments. */ +#define TYPE_CHECKED_PRINTF_FORMAT(formatArg, varArgs) \ + __attribute__((format(printf, formatArg, varArgs))) + +/* + * Mutex and thread data types defined by pthreads. + */ +typedef pthread_mutex_t mutex; +typedef pthread_t threadId; + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c new file mode 100644 index 00000000000..af0c61f03da --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread.c @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "os/thread.h" + +#include +#include + +/** + * Defines a helper function that adapts function pointer provided by caller to + * the type required by pthread_create. + * + * @param toRun thread to run + * @return void* result of running thread (always NULL) + */ +static void* runThread(void *toRun) { + const thread *t = toRun; + t->start(t->arg); + return NULL; +} + +int threadCreate(thread *t) { + int ret; + ret = pthread_create(&t->id, NULL, runThread, t); + if (ret) { + fprintf(stderr, "threadCreate: pthread_create failed with error %d\n", ret); + } + return ret; +} + +int threadJoin(const thread *t) { + int ret = pthread_join(t->id, NULL); + if (ret) { + fprintf(stderr, "threadJoin: pthread_join failed with error %d\n", ret); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c new file mode 100644 index 00000000000..0e187e98651 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/posix/thread_local_storage.c @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The changes here are the HDFS-16021 + * thread-ownership guard in hdfsThreadDestructor and the initialisation of the + * two fields threadLocalStorageCreate previously left uninitialised. + */ + +#include "os/thread_local_storage.h" + +#include +#include +#include +#include + +#include "exception.h" +#include "jni_helper.h" + +#define UNKNOWN "UNKNOWN" +#define MAXTHRID 256 + +/** Key that allows us to retrieve thread-local storage */ +static pthread_key_t gTlsKey; + +/** nonzero if we succeeded in initializing gTlsKey. Protected by the jvmMutex */ +static int gTlsKeyInitialized = 0; + +static void get_current_thread_id(JNIEnv* env, char* id, int max); + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v) +{ + JavaVM *vm; + struct ThreadLocalState *state = (struct ThreadLocalState*)v; + JNIEnv *env = state->env;; + jint ret; + jthrowable jthr; + char thr_name[MAXTHRID]; + + /* Detach only threads that libhdfs attached to the JVM. Detaching a thread + * that the JVM (or an embedding application) attached frees a JNIEnv its + * owner still holds, and by the time this destructor runs that env may + * already have been freed, so the dereference below reads freed memory. + * See HDFS-16021. */ + if (state->attachedByLibhdfs && (env != NULL) && (*env != NULL)) { + ret = (*env)->GetJavaVM(env, &vm); + + if (ret != 0) { + fprintf(stderr, "hdfsThreadDestructor: GetJavaVM failed with error %d\n", + ret); + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + } else { + ret = (*vm)->DetachCurrentThread(vm); + + if (ret != JNI_OK) { + jthr = (*env)->ExceptionOccurred(env); + if (jthr) { + (*env)->ExceptionDescribe(env); + (*env)->ExceptionClear(env); + } + get_current_thread_id(env, thr_name, MAXTHRID); + + fprintf(stderr, "hdfsThreadDestructor: Unable to detach thread %s " + "from the JVM. Error code: %d\n", thr_name, ret); + } + } + } + + /* Free exception strings */ + if (state->lastExceptionStackTrace) free(state->lastExceptionStackTrace); + if (state->lastExceptionRootCause) free(state->lastExceptionRootCause); + + /* Free the state itself */ + free(state); +} + +static void get_current_thread_id(JNIEnv* env, char* id, int max) { + jvalue jVal; + jobject thr = NULL; + jstring thr_name = NULL; + jlong thr_id = 0; + jthrowable jthr = NULL; + const char *thr_name_str; + + jthr = findClassAndInvokeMethod(env, &jVal, STATIC, NULL, "java/lang/Thread", + "currentThread", "()Ljava/lang/Thread;"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#currentThread failed: "); + goto done; + } + thr = jVal.l; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "getId", "()J"); + if (jthr) { + snprintf(id, max, "%s", UNKNOWN); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#getId failed: "); + goto done; + } + thr_id = jVal.j; + + jthr = findClassAndInvokeMethod(env, &jVal, INSTANCE, thr, + "java/lang/Thread", "toString", "()Ljava/lang/String;"); + if (jthr) { + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + printExceptionAndFree(env, jthr, PRINT_EXC_ALL, + "get_current_thread_id: Thread#toString failed: "); + goto done; + } + thr_name = jVal.l; + + thr_name_str = (*env)->GetStringUTFChars(env, thr_name, NULL); + if (!thr_name_str) { + printPendingExceptionAndFree(env, PRINT_EXC_ALL, + "get_current_thread_id: GetStringUTFChars failed: "); + snprintf(id, max, "%s:%ld", UNKNOWN, thr_id); + goto done; + } + + // Treating the jlong as a long *should* be safe + snprintf(id, max, "%s:%ld", thr_name_str, thr_id); + + // Release the char* + (*env)->ReleaseStringUTFChars(env, thr_name, thr_name_str); + +done: + destroyLocalReference(env, thr); + destroyLocalReference(env, thr_name); + + // Make sure the id is null terminated in case we overflow the max length + id[max - 1] = '\0'; +} + +struct ThreadLocalState* threadLocalStorageCreate() +{ + struct ThreadLocalState *state; + state = (struct ThreadLocalState*)malloc(sizeof(struct ThreadLocalState)); + if (state == NULL) { + fprintf(stderr, + "threadLocalStorageCreate: OOM - Unable to allocate thread local state\n"); + return NULL; + } + state->attachedByLibhdfs = false; + state->env = NULL; + state->lastExceptionStackTrace = NULL; + state->lastExceptionRootCause = NULL; + return state; +} + +int threadLocalStorageGet(struct ThreadLocalState **state) +{ + int ret = 0; + if (!gTlsKeyInitialized) { + ret = pthread_key_create(&gTlsKey, hdfsThreadDestructor); + if (ret) { + fprintf(stderr, + "threadLocalStorageGet: pthread_key_create failed with error %d\n", + ret); + return ret; + } + gTlsKeyInitialized = 1; + } + *state = pthread_getspecific(gTlsKey); + return ret; +} + +int threadLocalStorageSet(struct ThreadLocalState *state) +{ + int ret = pthread_setspecific(gTlsKey, state); + if (ret) { + fprintf(stderr, + "threadLocalStorageSet: pthread_setspecific failed with error %d\n", + ret); + hdfsThreadDestructor(state); + } + return ret; +} diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h new file mode 100644 index 00000000000..ae425d35641 --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread.h @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFS_THREAD_H +#define LIBHDFS_THREAD_H + +/* + * Defines abstraction over platform-specific threads. + */ + +#include "platform.h" + +/** Pointer to function to run in thread. */ +typedef void (*threadProcedure)(void *); + +/** Structure containing a thread's ID, starting address and argument. */ +typedef struct { + threadId id; + threadProcedure start; + void *arg; +} thread; + +/** + * Creates and immediately starts a new thread. + * + * @param t thread to create + * @return 0 if successful, non-zero otherwise + */ +int threadCreate(thread *t); + +/** + * Joins to the given thread, blocking if necessary. + * + * @param t thread to join + * @return 0 if successful, non-zero otherwise + */ +int threadJoin(const thread *t); + +#endif diff --git a/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h new file mode 100644 index 00000000000..4c7128a713f --- /dev/null +++ b/native/hdfs-sys/libhdfs/hdfs_3_3/os/thread_local_storage.h @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This file was modified by the Apache DataFusion Comet project. See + * native/hdfs-sys/README.md for the provenance of this directory and for the + * full list of modifications. The change here is the `attachedByLibhdfs` field, + * which carries the HDFS-16021 thread-ownership fix. + */ + +#ifndef LIBHDFS_THREAD_LOCAL_STORAGE_H +#define LIBHDFS_THREAD_LOCAL_STORAGE_H + +/* + * Defines abstraction over platform-specific thread-local storage. libhdfs + * currently only needs thread-local storage for a single piece of data: the + * thread's JNIEnv. For simplicity, this interface is defined in terms of + * JNIEnv, not general-purpose thread-local storage of any arbitrary data. + */ + +#include +#include + +/* + * Most operating systems support the more efficient __thread construct, which + * is initialized by the linker. The following macros use this technique on the + * operating systems that support it. + */ +#ifdef HAVE_BETTER_TLS + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) \ + static __thread struct ThreadLocalState *quickTlsEnv = NULL; \ + { \ + if (quickTlsEnv) { \ + *state = quickTlsEnv; \ + } \ + } + + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) \ + { \ + quickTlsEnv = (state); \ + } +#else + #define THREAD_LOCAL_STORAGE_GET_QUICK(state) + #define THREAD_LOCAL_STORAGE_SET_QUICK(state) +#endif + +struct ThreadLocalState { + /* Whether libhdfs attached this thread to the JVM. */ + bool attachedByLibhdfs; + /* The JNIEnv associated with the current thread */ + JNIEnv *env; + /* The last exception stack trace that occured on this thread */ + char *lastExceptionStackTrace; + /* The last exception root cause that occured on this thread */ + char *lastExceptionRootCause; +}; + +/** + * The function that is called whenever a thread with libhdfs thread local data + * is destroyed. + * + * @param v The thread-local data + */ +void hdfsThreadDestructor(void *v); + +/** + * Creates an object of ThreadLocalState. + * + * @return The newly created object if successful, NULL otherwise. + */ +struct ThreadLocalState* threadLocalStorageCreate(); + +/** + * Gets the ThreadLocalState in thread-local storage for the current thread. + * If the call succeeds, and there is a ThreadLocalState associated with this + * thread, then returns 0 and populates 'state'. If the call succeeds, but + * there is no ThreadLocalState associated with this thread, then returns 0 + * and sets ThreadLocalState to NULL. If the call fails, then returns non-zero. + * Only one thread at a time may execute this function. The caller is + * responsible for enforcing mutual exclusion. + * + * @param env ThreadLocalState out parameter + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageGet(struct ThreadLocalState **state); + +/** + * Sets the ThreadLocalState in thread-local storage for the current thread. + * + * @param env ThreadLocalState to set + * @return 0 if successful, non-zero otherwise + */ +int threadLocalStorageSet(struct ThreadLocalState *state); + +#endif diff --git a/native/hdfs-sys/src/lib.rs b/native/hdfs-sys/src/lib.rs new file mode 100644 index 00000000000..9fda3a5681a --- /dev/null +++ b/native/hdfs-sys/src/lib.rs @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Raw bindings to the `libhdfs` C API, built from the vendored Apache Hadoop +//! sources under `libhdfs/hdfs_3_3/`. +//! +//! This crate stands in for the crates.io `hdfs-sys` crate through a +//! `[patch.crates-io]` entry in `native/Cargo.toml`. It exists only so Comet can +//! carry the HDFS-16021 thread-ownership fix, which no released `hdfs-sys` +//! contains. See `README.md` for the provenance and the removal condition. +//! +//! The declarations below are transcribed from the vendored +//! `libhdfs/hdfs_3_3/include/hdfs/hdfs.h` and cover the surface `hdrs`, the only +//! consumer in Comet's dependency graph, actually uses. Names match the +//! bindgen-style spelling that `hdrs` expects, notably the flattened +//! `tObjectKind_kObjectKind*` constants. + +#![allow(non_snake_case)] +#![allow(non_camel_case_types)] +#![allow(non_upper_case_globals)] + +use std::os::raw::{c_char, c_int, c_long, c_short, c_void}; + +/// `typedef int32_t tSize` -- size of data for read/write io ops. +pub type tSize = i32; +/// `typedef time_t tTime` -- time type in seconds. +pub type tTime = c_long; +/// `typedef int64_t tOffset` -- offset within the file. +pub type tOffset = i64; +/// `typedef uint16_t tPort` -- port. +pub type tPort = u16; + +/// `typedef enum tObjectKind`. A C enum with values that fit in an `unsigned +/// int`, which is how `hdrs` consumes `hdfsFileInfo::mKind`. +pub type tObjectKind = ::std::os::raw::c_uint; +/// `kObjectKindFile = 'F'` +pub const tObjectKind_kObjectKindFile: tObjectKind = 70; +/// `kObjectKindDirectory = 'D'` +pub const tObjectKind_kObjectKindDirectory: tObjectKind = 68; + +/// Opaque `struct hdfsBuilder`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsBuilder { + _unused: [u8; 0], +} + +/// Opaque `struct hdfs_internal`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfs_internal { + _unused: [u8; 0], +} + +/// `typedef struct hdfs_internal* hdfsFS`. +pub type hdfsFS = *mut hdfs_internal; + +/// Opaque `struct hdfsFile_internal`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsFile_internal { + _unused: [u8; 0], +} + +/// `typedef struct hdfsFile_internal* hdfsFile`. +pub type hdfsFile = *mut hdfsFile_internal; + +/// Field order and widths mirror the `hdfsFileInfo` typedef in `hdfs.h`. +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct hdfsFileInfo { + /// file or directory + pub mKind: tObjectKind, + /// the name of the file + pub mName: *mut c_char, + /// the last modification time for the file in seconds + pub mLastMod: tTime, + /// the size of the file in bytes + pub mSize: tOffset, + /// the count of replicas + pub mReplication: c_short, + /// the block size for the file + pub mBlockSize: tOffset, + /// the owner of the file + pub mOwner: *mut c_char, + /// the group associated with the file + pub mGroup: *mut c_char, + /// the permissions associated with the file + pub mPermissions: c_short, + /// the last access time for the file in seconds + pub mLastAccess: tTime, +} + +unsafe extern "C" { + pub fn hdfsNewBuilder() -> *mut hdfsBuilder; + pub fn hdfsFreeBuilder(bld: *mut hdfsBuilder); + pub fn hdfsBuilderSetNameNode(bld: *mut hdfsBuilder, nn: *const c_char); + pub fn hdfsBuilderSetNameNodePort(bld: *mut hdfsBuilder, port: tPort); + pub fn hdfsBuilderSetUserName(bld: *mut hdfsBuilder, userName: *const c_char); + pub fn hdfsBuilderSetKerbTicketCachePath( + bld: *mut hdfsBuilder, + kerbTicketCachePath: *const c_char, + ); + pub fn hdfsBuilderConnect(bld: *mut hdfsBuilder) -> hdfsFS; + + pub fn hdfsConnect(nn: *const c_char, port: tPort) -> hdfsFS; + pub fn hdfsDisconnect(fs: hdfsFS) -> c_int; + + pub fn hdfsOpenFile( + fs: hdfsFS, + path: *const c_char, + flags: c_int, + bufferSize: c_int, + replication: c_short, + blocksize: tSize, + ) -> hdfsFile; + pub fn hdfsCloseFile(fs: hdfsFS, file: hdfsFile) -> c_int; + pub fn hdfsExists(fs: hdfsFS, path: *const c_char) -> c_int; + + pub fn hdfsSeek(fs: hdfsFS, file: hdfsFile, desiredPos: tOffset) -> c_int; + pub fn hdfsTell(fs: hdfsFS, file: hdfsFile) -> tOffset; + pub fn hdfsRead(fs: hdfsFS, file: hdfsFile, buffer: *mut c_void, length: tSize) -> tSize; + pub fn hdfsPread( + fs: hdfsFS, + file: hdfsFile, + position: tOffset, + buffer: *mut c_void, + length: tSize, + ) -> tSize; + pub fn hdfsWrite(fs: hdfsFS, file: hdfsFile, buffer: *const c_void, length: tSize) -> tSize; + pub fn hdfsFlush(fs: hdfsFS, file: hdfsFile) -> c_int; + + pub fn hdfsDelete(fs: hdfsFS, path: *const c_char, recursive: c_int) -> c_int; + pub fn hdfsRename(fs: hdfsFS, oldPath: *const c_char, newPath: *const c_char) -> c_int; + pub fn hdfsCreateDirectory(fs: hdfsFS, path: *const c_char) -> c_int; + + pub fn hdfsGetPathInfo(fs: hdfsFS, path: *const c_char) -> *mut hdfsFileInfo; + pub fn hdfsListDirectory( + fs: hdfsFS, + path: *const c_char, + numEntries: *mut c_int, + ) -> *mut hdfsFileInfo; + pub fn hdfsFreeFileInfo(hdfsFileInfo: *mut hdfsFileInfo, numEntries: c_int); +} + +/// `hdrs` reads `hdfsFileInfo` fields out of an array libhdfs allocated, so a layout mismatch +/// between these declarations and the C typedef they transcribe would be silent memory corruption +/// rather than a link error. These are compile-time assertions rather than `#[test]`s so that every +/// build of the crate enforces them; the crate is not a default workspace member, so `cargo test` +/// would not otherwise reach it. +/// +/// The expected offsets follow from the C field order under the usual 8-byte alignment: +/// `mKind`(u32) + 4 pad, `mName`(8), `mLastMod`(8), `mSize`(8), `mReplication`(2) + 6 pad, +/// `mBlockSize`(8), `mOwner`(8), `mGroup`(8), `mPermissions`(2) + 6 pad, `mLastAccess`(8). +const _: () = { + use std::mem::{align_of, offset_of, size_of}; + + assert!(size_of::() == 80); + assert!(align_of::() == 8); + assert!(offset_of!(hdfsFileInfo, mKind) == 0); + assert!(offset_of!(hdfsFileInfo, mName) == 8); + assert!(offset_of!(hdfsFileInfo, mLastMod) == 16); + assert!(offset_of!(hdfsFileInfo, mSize) == 24); + assert!(offset_of!(hdfsFileInfo, mReplication) == 32); + assert!(offset_of!(hdfsFileInfo, mBlockSize) == 40); + assert!(offset_of!(hdfsFileInfo, mOwner) == 48); + assert!(offset_of!(hdfsFileInfo, mGroup) == 56); + assert!(offset_of!(hdfsFileInfo, mPermissions) == 64); + assert!(offset_of!(hdfsFileInfo, mLastAccess) == 72); + + // `hdfs.h` spells these as the character literals 'F' and 'D'. + assert!(tObjectKind_kObjectKindFile == b'F' as tObjectKind); + assert!(tObjectKind_kObjectKindDirectory == b'D' as tObjectKind); + + // `tTime` is `time_t`, and `hdrs` binds the two timestamp fields to `i64`. + assert!(size_of::() == 8); +};