feat(storage): Add HDFS support via opendal services-hdfs-native

crates/iceberg/public-api.txt

@@ -1015,6 +1015,8 @@ pub const iceberg::io::GCS_PROJECT_ID: &str
 pub const iceberg::io::GCS_SERVICE_HOST: &str
 pub const iceberg::io::GCS_TOKEN: &str
 pub const iceberg::io::GCS_USER_PROJECT: &str
+pub const iceberg::io::HDFS_HADOOP_CONF_PREFIX: &str
+pub const iceberg::io::HDFS_NAME_NODE: &str
 pub const iceberg::io::HF_ENDPOINT: &str
 pub const iceberg::io::HF_REVISION: &str
 pub const iceberg::io::HF_TOKEN: &str

crates/iceberg/src/io/storage/config/hdfs.rs

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! HDFS storage configuration.
+
+use std::collections::HashMap;
+
+use iceberg_property_macro::Properties;
+
+/// HDFS NameNode RPC endpoint(s), e.g. `hdfs://namenode:8020`; a
+/// comma-separated list enables HA failover. When unset, the NameNode is
+/// derived from the path authority.
+pub const HDFS_NAME_NODE: &str = "hdfs.name-node";
+/// Prefix for properties forwarded to the HDFS client configuration, e.g.
+/// `hadoop.dfs.client.failover.random.order`. Forwarded values (prefix
+/// stripped) override those loaded from `$HADOOP_CONF_DIR`.
+pub const HDFS_HADOOP_CONF_PREFIX: &str = "hadoop.";
+
+/// HDFS storage configuration.
+// No in-crate consumer yet: `iceberg-storage-opendal` parses the raw
+// properties itself and only shares the key constants above.
+#[allow(dead_code)]
+#[derive(Debug, Properties)]
+pub(crate) struct HdfsConfig {
+    /// NameNode endpoint(s); comma-separated for HA failover.
+    #[property(key = HDFS_NAME_NODE, default = None, getter)]
+    name_node: Option<String>,
+    /// Extra HDFS client configuration (the `hadoop.` prefix stripped).
+    #[property(prefix = HDFS_HADOOP_CONF_PREFIX, getter)]
+    options: HashMap<String, String>,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_hdfs_config_from_properties() {
+        let props = HashMap::from([
+            (
+                HDFS_NAME_NODE.to_string(),
+                "hdfs://namenode:8020".to_string(),
+            ),
+            (
+                "hadoop.dfs.client.failover.random.order".to_string(),
+                "true".to_string(),
+            ),
+            ("unrelated.key".to_string(), "ignored".to_string()),
+        ]);
+
+        let cfg = HdfsConfig::from_properties(&props).unwrap();
+        assert_eq!(cfg.name_node().as_deref(), Some("hdfs://namenode:8020"));
+        assert_eq!(
+            cfg.options().get("dfs.client.failover.random.order"),
+            Some(&"true".to_string())
+        );
+        assert!(!cfg.options().contains_key("unrelated.key"));
+    }
+
+    #[test]
+    fn test_hdfs_config_empty() {
+        let cfg = HdfsConfig::from_properties(&HashMap::new()).unwrap();
+        assert_eq!(cfg.name_node().as_deref(), None);
+        assert!(cfg.options().is_empty());
+    }
+}

crates/iceberg/src/io/storage/config/mod.rs

@@ -32,6 +32,7 @@
 
 mod azdls;
 mod gcs;
+mod hdfs;
 mod hf;
 mod oss;
 mod s3;
@@ -40,6 +41,7 @@ use std::collections::HashMap;
 
 pub use azdls::*;
 pub use gcs::*;
+pub use hdfs::*;
 pub use hf::*;
 pub use oss::*;
 pub use s3::*;

crates/storage/opendal/Cargo.toml

@@ -42,6 +42,7 @@ opendal-all = [
 opendal-azdls = ["opendal/services-azdls"]
 opendal-fs = ["opendal/services-fs"]
 opendal-gcs = ["opendal/services-gcs"]
+opendal-hdfs-native = ["opendal/services-hdfs-native"]
 opendal-hf = ["opendal/services-hf"]
 opendal-memory = ["opendal/services-memory"]
 opendal-oss = ["opendal/services-oss"]

crates/storage/opendal/README.md

@@ -32,10 +32,11 @@ OpenDAL-based storage backend implementations for [Apache Iceberg Rust](https://
 | Hugging Face         | `opendal-hf`     | ✅ Stable       | Hugging Face buckets and repositories         |
 | Alibaba Cloud OSS    | `opendal-oss`    | 🧪 Experimental | Alibaba Cloud Object Storage Service          |
 | Azure Datalake       | `opendal-azdls`  | 🧪 Experimental | Azure Datalake Storage v2                     |
+| HDFS                 | `opendal-hdfs-native` | 🧪 Experimental | HDFS via the native Rust client (`hdfs-native`) |
 
 You can enable all stable storage backends at once using the `opendal-all` feature flag.
 
-> Note that `opendal-oss` and `opendal-azdls` are currently experimental and not included in `opendal-all`.
+> Note that `opendal-oss`, `opendal-azdls` and `opendal-hdfs-native` are currently experimental and not included in `opendal-all`.
 
 ## Usage
 

crates/storage/opendal/public-api.txt

@@ -6,6 +6,9 @@ pub iceberg_storage_opendal::OpenDalStorage::Azdls
 pub iceberg_storage_opendal::OpenDalStorage::Azdls::config: alloc::sync::Arc<opendal_service_azdls::config::AzdlsConfig>
 pub iceberg_storage_opendal::OpenDalStorage::Gcs
 pub iceberg_storage_opendal::OpenDalStorage::Gcs::config: alloc::sync::Arc<opendal_service_gcs::config::GcsConfig>
+pub iceberg_storage_opendal::OpenDalStorage::Hdfs
+pub iceberg_storage_opendal::OpenDalStorage::Hdfs::config: alloc::sync::Arc<opendal_service_hdfs_native::config::HdfsNativeConfig>
+pub iceberg_storage_opendal::OpenDalStorage::Hdfs::operators: alloc::sync::Arc<std::sync::poison::rwlock::RwLock<std::collections::hash::map::HashMap<alloc::string::String, opendal_core::types::operator::operator::Operator>>>
 pub iceberg_storage_opendal::OpenDalStorage::Hf
 pub iceberg_storage_opendal::OpenDalStorage::Hf::config: alloc::sync::Arc<opendal_service_hf::config::HfConfig>
 pub iceberg_storage_opendal::OpenDalStorage::LocalFs
@@ -39,6 +42,7 @@ pub enum iceberg_storage_opendal::OpenDalStorageFactory
 pub iceberg_storage_opendal::OpenDalStorageFactory::Azdls
 pub iceberg_storage_opendal::OpenDalStorageFactory::Fs
 pub iceberg_storage_opendal::OpenDalStorageFactory::Gcs
+pub iceberg_storage_opendal::OpenDalStorageFactory::Hdfs
 pub iceberg_storage_opendal::OpenDalStorageFactory::Hf
 pub iceberg_storage_opendal::OpenDalStorageFactory::Memory
 pub iceberg_storage_opendal::OpenDalStorageFactory::Oss