feat(storage): Add HDFS support via opendal services-hdfs (libhdfs)

.github/workflows/ci.yml

@@ -152,7 +152,16 @@ jobs:
           save-if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
 
       - name: Build
-        run: make build
+        shell: bash
+        run: |
+          if [ "$RUNNER_OS" = "Windows" ]; then
+            # `hdrs`, the libhdfs binding behind `opendal-hdfs`, does not
+            # compile on Windows, so build every other feature there.
+            cargo build --all-targets --all-features --workspace --exclude iceberg-storage-opendal
+            cargo build --all-targets -p iceberg-storage-opendal --features opendal-all,opendal-hdfs-native
+          else
+            make build
+          fi
 
   # Checks each crate individually to catch missing feature declarations that
   # are otherwise masked by Cargo's feature unification across the workspace.
@@ -253,6 +262,19 @@ jobs:
         if: matrix.test-suite.name == 'default'
         run: make docker-up
 
+      # `opendal-hdfs` links libjvm into every test binary. Its tests run
+      # Hadoop's Java client from the HDFS fixture's jars.
+      - name: Set up the JVM for libhdfs
+        shell: bash
+        run: |
+          echo "LD_LIBRARY_PATH=$JAVA_HOME/lib/server${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
+          if [ "${{ matrix.test-suite.name }}" = "default" ]; then
+            docker compose -f dev/docker-compose.yaml cp hdfs-namenode:/opt/hadoop/share/hadoop "$RUNNER_TEMP/hadoop"
+            # libhdfs does not expand wildcards, so list each jar.
+            jars=$(find "$RUNNER_TEMP/hadoop/common" "$RUNNER_TEMP/hadoop/hdfs" -name '*.jar' ! -name '*-tests.jar' ! -name '*-sources.jar' | sort | paste -sd: -)
+            echo "CLASSPATH=$PWD/dev/hdfs/client:$jars" >> "$GITHUB_ENV"
+          fi
+
       - name: Run tests
         shell: bash
         env:

crates/storage/opendal/Cargo.toml

@@ -42,6 +42,7 @@ opendal-all = [
 opendal-azdls = ["opendal/services-azdls"]
 opendal-fs = ["opendal/services-fs"]
 opendal-gcs = ["opendal/services-gcs"]
+opendal-hdfs = ["opendal/services-hdfs"]
 opendal-hdfs-native = [
   "opendal/services-hdfs-native",
   "dep:tokio",

crates/storage/opendal/README.md

@@ -33,13 +33,16 @@ OpenDAL-based storage backend implementations for [Apache Iceberg Rust](https://
 | Alibaba Cloud OSS    | `opendal-oss`    | 🧪 Experimental | Alibaba Cloud Object Storage Service          |
 | Azure Datalake       | `opendal-azdls`  | 🧪 Experimental | Azure Datalake Storage v2                     |
 | HDFS                 | `opendal-hdfs-native` | 🧪 Experimental | HDFS via the native Rust client (`hdfs-native`) |
+| HDFS (libhdfs)       | `opendal-hdfs`   | 🧪 Experimental | HDFS via libhdfs and Hadoop's Java client (JNI) |
 
 You can enable all stable storage backends at once using the `opendal-all` feature flag.
 
 > Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`.
 
 > `opendal-hdfs-native` is experimental and not included in `opendal-all`. A portless path authority such as `hdfs://nameservice1/warehouse` must be declared in properties, e.g. `hdfs.name-node.nameservice1 = nn1:8020,nn2:8020`: unlike Hadoop, nameservices defined only in `$HADOOP_CONF_DIR` are not resolved, and a portless host is not dialed on the default port 8020.
 
+> `opendal-hdfs` is experimental and not included in `opendal-all`. Use it through `OpenDalStorageFactory::Hdfs`, since `OpenDalResolvingStorageFactory` does not route to it. Hadoop's Java client reads its configuration from `CLASSPATH`, so nameservices, HA and Kerberos work as in Hadoop, and no `hdfs.*` or `hadoop.*` property applies. Building needs a JDK, and binaries need `$JAVA_HOME/lib/server` on the library path. It does not build on Windows, where its `hdrs` dependency does not compile. Unless the process already runs a JVM, `CLASSPATH` must list the Hadoop configuration directory and each Hadoop jar, e.g. from `hadoop classpath --glob`.
+
 ## Usage
 
 Add the crate to your `Cargo.toml` with the feature flags for the backends you need:

crates/storage/opendal/public-api.txt

@@ -8,6 +8,7 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc<ope
 pub iceberg_storage_opendal::OpenDalStorage::Gcs
 pub iceberg_storage_opendal::OpenDalStorage::Gcs::client_config: iceberg_storage_opendal::OpenDalClientConfig
 pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc<opendal_service_gcs::config::GcsConfig>
+pub iceberg_storage_opendal::OpenDalStorage::Hdfs(iceberg_storage_opendal::HdfsStorage)
 pub iceberg_storage_opendal::OpenDalStorage::HdfsNative(iceberg_storage_opendal::HdfsNativeStorage)
 pub iceberg_storage_opendal::OpenDalStorage::Hf
 pub iceberg_storage_opendal::OpenDalStorage::Hf::client_config: iceberg_storage_opendal::OpenDalClientConfig
@@ -48,6 +49,7 @@ pub enum iceberg_storage_opendal::OpenDalStorageFactory
 pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls
 pub iceberg_storage_opendal::OpenDalStorageFactory::Fs
 pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs
+pub iceberg_storage_opendal::OpenDalStorageFactory::Hdfs
 pub iceberg_storage_opendal::OpenDalStorageFactory::HdfsNative
 pub iceberg_storage_opendal::OpenDalStorageFactory::Hf
 pub iceberg_storage_opendal::OpenDalStorageFactory::Memory
@@ -80,6 +82,15 @@ impl serde_core::ser::Serialize for iceberg_storage_opendal::HdfsNativeStorage
 pub fn iceberg_storage_opendal::HdfsNativeStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
 impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::HdfsNativeStorage
 pub fn iceberg_storage_opendal::HdfsNativeStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
+pub struct iceberg_storage_opendal::HdfsStorage
+impl core::clone::Clone for iceberg_storage_opendal::HdfsStorage
+pub fn iceberg_storage_opendal::HdfsStorage::clone(&self) -> iceberg_storage_opendal::HdfsStorage
+impl core::fmt::Debug for iceberg_storage_opendal::HdfsStorage
+pub fn iceberg_storage_opendal::HdfsStorage::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+impl serde_core::ser::Serialize for iceberg_storage_opendal::HdfsStorage
+pub fn iceberg_storage_opendal::HdfsStorage::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer
+impl<'de> serde_core::de::Deserialize<'de> for iceberg_storage_opendal::HdfsStorage
+pub fn iceberg_storage_opendal::HdfsStorage::deserialize<__D>(__deserializer: __D) -> core::result::Result<Self, <__D as serde_core::de::Deserializer>::Error> where __D: serde_core::de::Deserializer<'de>
 pub struct iceberg_storage_opendal::OpenDalClientConfig
 impl iceberg_storage_opendal::OpenDalClientConfig
 pub fn iceberg_storage_opendal::OpenDalClientConfig::from_properties(properties: &std::collections::hash::map::HashMap<alloc::string::String, alloc::string::String>) -> iceberg::error::Result<Self>