You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/19 17:21:06 UTC

(opendal) branch main updated: refactor(services/hdfs): deprecate append option (#7563)

This is an automated email from the ASF dual-hosted git repository.

Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new 9ca0ad793 refactor(services/hdfs): deprecate append option (#7563)
9ca0ad793 is described below

commit 9ca0ad793d533ba35708ef9eccef010ac2af2b3c
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed May 20 01:20:14 2026 +0800

    refactor(services/hdfs): deprecate append option (#7563)
---
 .github/services/hdfs/hdfs_cluster/action.yml              |  3 +--
 .../hdfs/hdfs_cluster_with_atomic_write_dir/action.yml     |  4 ++--
 .github/services/hdfs/hdfs_default/action.yml              |  2 +-
 .github/services/hdfs/hdfs_default_gcs/action.yml          |  2 +-
 .../hdfs/hdfs_default_on_azurite_azblob/action.yml         |  2 +-
 .github/services/hdfs/hdfs_default_on_minio_s3/action.yml  |  2 +-
 .../hdfs/hdfs_default_with_atomic_write_dir/action.yml     |  4 ++--
 .../services/hdfs_native/hdfs_native_cluster/action.yml    |  3 +--
 .../OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs       |  2 +-
 .../src/main/java/org/apache/opendal/ServiceConfig.java    |  4 +++-
 bindings/python/python/opendal/operator.pyi                |  4 ++--
 bindings/python/src/services.rs                            |  6 ++++--
 core/services/hdfs-native/src/backend.rs                   | 14 +++++++-------
 core/services/hdfs-native/src/config.rs                    |  8 ++++++--
 core/services/hdfs-native/src/core.rs                      |  3 ---
 core/services/hdfs-native/src/docs.md                      |  2 +-
 core/services/hdfs/src/backend.rs                          | 13 +++++++------
 core/services/hdfs/src/config.rs                           |  8 ++++++--
 core/services/hdfs/src/docs.md                             | 11 +++--------
 19 files changed, 50 insertions(+), 47 deletions(-)

diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml
index 2047c65ff..9670d2400 100644
--- a/.github/services/hdfs/hdfs_cluster/action.yml
+++ b/.github/services/hdfs/hdfs_cluster/action.yml
@@ -35,5 +35,4 @@ runs:
         cat << EOF >> $GITHUB_ENV
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
-        OPENDAL_HDFS_ENABLE_APPEND=true
-        EOF
\ No newline at end of file
+        EOF
diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
index 6ad8894b0..804bb9507 100644
--- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
+++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
@@ -36,5 +36,5 @@ runs:
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
         OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
-        OPENDAL_HDFS_ENABLE_APPEND=false
-        EOF
\ No newline at end of file
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
+        EOF
diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml
index a6e2540dd..35edbf470 100644
--- a/.github/services/hdfs/hdfs_default/action.yml
+++ b/.github/services/hdfs/hdfs_default/action.yml
@@ -31,5 +31,5 @@ runs:
         cat << EOF >> $GITHUB_ENV
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_NAME_NODE=default
-        OPENDAL_HDFS_ENABLE_APPEND=false
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
         EOF
diff --git a/.github/services/hdfs/hdfs_default_gcs/action.yml b/.github/services/hdfs/hdfs_default_gcs/action.yml
index 7cb0e8615..c018731a1 100644
--- a/.github/services/hdfs/hdfs_default_gcs/action.yml
+++ b/.github/services/hdfs/hdfs_default_gcs/action.yml
@@ -54,7 +54,7 @@ runs:
         LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
         OPENDAL_HDFS_ROOT=${OPENDAL_GCS_ROOT}
         OPENDAL_HDFS_NAME_NODE=gs://${OPENDAL_GCS_BUCKET}
-        OPENDAL_HDFS_ENABLE_APPEND=false
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
         EOF
 
         mkdir -p /tmp/hdfs
diff --git a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
index 11ac22c2e..e888f022b 100644
--- a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
+++ b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
@@ -67,5 +67,5 @@ runs:
         LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
         OPENDAL_HDFS_ROOT=${OPENDAL_AZBLOB_ROOT}
         OPENDAL_HDFS_NAME_NODE=wasb://${OPENDAL_AZBLOB_CONTAINER}@${OPENDAL_AZBLOB_ACCOUNT_NAME}.blob.core.windows.net
-        OPENDAL_HDFS_ENABLE_APPEND=false
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
         EOF
diff --git a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
index cd8128aa9..98e909a04 100644
--- a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
+++ b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
@@ -64,5 +64,5 @@ runs:
         CLASSPATH=${CLASSPATH}
         LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
         OPENDAL_HDFS_NAME_NODE=s3a://test
-        OPENDAL_HDFS_ENABLE_APPEND=false
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
         EOF
diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
index 57372570f..8cb9f3e44 100644
--- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
+++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
@@ -32,5 +32,5 @@ runs:
         OPENDAL_HDFS_ROOT=/tmp/opendal/
         OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
         OPENDAL_HDFS_NAME_NODE=default
-        OPENDAL_HDFS_ENABLE_APPEND=false
-        EOF
\ No newline at end of file
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
+        EOF
diff --git a/.github/services/hdfs_native/hdfs_native_cluster/action.yml b/.github/services/hdfs_native/hdfs_native_cluster/action.yml
index bf0fa5502..5bf9e4e55 100644
--- a/.github/services/hdfs_native/hdfs_native_cluster/action.yml
+++ b/.github/services/hdfs_native/hdfs_native_cluster/action.yml
@@ -31,5 +31,4 @@ runs:
         cat << EOF >> $GITHUB_ENV
         OPENDAL_HDFS_NATIVE_ROOT=/tmp/opendal/
         OPENDAL_HDFS_NATIVE_NAME_NODE=hdfs://localhost:8020
-        OPENDAL_HDFS_NATIVE_ENABLE_APPEND=true
-        EOF
\ No newline at end of file
+        EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
index 47ea319c1..c873a846f 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
@@ -29,7 +29,7 @@ namespace OpenDAL.ServiceConfig
     public sealed class HdfsNativeServiceConfig : IServiceConfig
     {
         /// <summary>
-        /// enable the append capacity
+        /// Deprecated: HDFS Native append capability is enabled by default.
         /// </summary>
         public bool? EnableAppend { get; init; }
         /// <summary>
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index 5f463adf6..41bfabb69 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -1499,7 +1499,9 @@ public interface ServiceConfig {
     @RequiredArgsConstructor(access = AccessLevel.PRIVATE)
     class HdfsNative implements ServiceConfig {
         /**
-         * <p>enable the append capacity</p>
+         * <p>Deprecated: HDFS Native append capability is enabled by default.</p>
+         *
+         * @deprecated HDFS Native append capability is enabled by default and this option is no longer needed.
          */
         public final Boolean enableAppend;
         /**
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index 4beaae4c0..0f47da71c 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -1381,7 +1381,7 @@ class AsyncOperator:
         Parameters
         ----------
         enable_append : builtins.bool, optional
-            enable the append capacity
+            Deprecated: HDFS Native append capability is enabled by default.
         name_node : builtins.str, optional
             name_node of this backend
         options : builtins.dict, optional
@@ -3979,7 +3979,7 @@ class Operator:
         Parameters
         ----------
         enable_append : builtins.bool, optional
-            enable the append capacity
+            Deprecated: HDFS Native append capability is enabled by default.
         name_node : builtins.str, optional
             name_node of this backend
         options : builtins.dict, optional
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index 62787e690..a2709a70b 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -1145,7 +1145,8 @@ submit! {
                 Parameters
                 ----------
                 enable_append : builtins.bool, optional
-                    enable the append capacity
+                    Deprecated: HDFS Native append capability is enabled
+                    by default.
                 name_node : builtins.str, optional
                     name_node of this backend
                 options : builtins.dict, optional
@@ -3821,7 +3822,8 @@ submit! {
                 Parameters
                 ----------
                 enable_append : builtins.bool, optional
-                    enable the append capacity
+                    Deprecated: HDFS Native append capability is enabled
+                    by default.
                 name_node : builtins.str, optional
                     name_node of this backend
                 options : builtins.dict, optional
diff --git a/core/services/hdfs-native/src/backend.rs b/core/services/hdfs-native/src/backend.rs
index 535aa5780..1d32cd1d0 100644
--- a/core/services/hdfs-native/src/backend.rs
+++ b/core/services/hdfs-native/src/backend.rs
@@ -72,11 +72,12 @@ impl HdfsNativeBuilder {
         self
     }
 
-    /// Enable append capacity of this backend.
-    ///
-    /// This should be disabled when HDFS runs in non-distributed mode.
-    pub fn enable_append(mut self, enable_append: bool) -> Self {
-        self.config.enable_append = enable_append;
+    /// Deprecated: HDFS Native append capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "HDFS Native append capability is enabled by default and this option is no longer needed."
+    )]
+    pub fn enable_append(self, _enable_append: bool) -> Self {
         self
     }
 
@@ -130,7 +131,7 @@ impl Builder for HdfsNativeBuilder {
                             read: true,
 
                             write: true,
-                            write_can_append: self.config.enable_append,
+                            write_can_append: true,
 
                             create_dir: true,
                             delete: true,
@@ -148,7 +149,6 @@ impl Builder for HdfsNativeBuilder {
                 },
                 root,
                 client: Arc::new(client),
-                enable_append: self.config.enable_append,
             }),
         })
     }
diff --git a/core/services/hdfs-native/src/config.rs b/core/services/hdfs-native/src/config.rs
index 2ec6f1427..d2e4a5965 100644
--- a/core/services/hdfs-native/src/config.rs
+++ b/core/services/hdfs-native/src/config.rs
@@ -37,7 +37,11 @@ pub struct HdfsNativeConfig {
     pub root: Option<String>,
     /// name_node of this backend
     pub name_node: Option<String>,
-    /// enable the append capacity
+    /// Deprecated: HDFS Native append capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "HDFS Native append capability is enabled by default and this option is no longer needed."
+    )]
     pub enable_append: bool,
     /// other options for hdfs client
     pub options: Option<HashMap<String, String>>,
@@ -48,7 +52,6 @@ impl Debug for HdfsNativeConfig {
         f.debug_struct("HdfsNativeConfig")
             .field("root", &self.root)
             .field("name_node", &self.name_node)
-            .field("enable_append", &self.enable_append)
             .field("options", &self.options)
             .finish_non_exhaustive()
     }
@@ -72,6 +75,7 @@ impl opendal_core::Configurator for HdfsNativeConfig {
         Self::from_iter(map)
     }
 
+    #[allow(deprecated)]
     fn into_builder(self) -> Self::Builder {
         HdfsNativeBuilder { config: self }
     }
diff --git a/core/services/hdfs-native/src/core.rs b/core/services/hdfs-native/src/core.rs
index da2808c0f..164773907 100644
--- a/core/services/hdfs-native/src/core.rs
+++ b/core/services/hdfs-native/src/core.rs
@@ -31,14 +31,12 @@ pub struct HdfsNativeCore {
     pub info: Arc<AccessorInfo>,
     pub root: String,
     pub client: Arc<hdfs_native::Client>,
-    pub enable_append: bool,
 }
 
 impl Debug for HdfsNativeCore {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("HdfsNativeCore")
             .field("root", &self.root)
-            .field("enable_append", &self.enable_append)
             .finish_non_exhaustive()
     }
 }
@@ -116,7 +114,6 @@ impl HdfsNativeCore {
 
         let f = if target_exists {
             if args.append() {
-                assert!(self.enable_append, "append is not enabled");
                 self.client
                     .append(&target_path)
                     .await
diff --git a/core/services/hdfs-native/src/docs.md b/core/services/hdfs-native/src/docs.md
index 9d5230e82..6b2f104cb 100644
--- a/core/services/hdfs-native/src/docs.md
+++ b/core/services/hdfs-native/src/docs.md
@@ -19,4 +19,4 @@ This service can be used to:
 
 - `root`: Set the work dir for backend.
 - `name_node`: Set the name node for backend.
-- `enable_append`: enable the append capacity. Default is false.
+- `enable_append`: Deprecated. HDFS Native append capability is enabled by default and this option is no longer needed.
diff --git a/core/services/hdfs/src/backend.rs b/core/services/hdfs/src/backend.rs
index c652308a9..a5d111b00 100644
--- a/core/services/hdfs/src/backend.rs
+++ b/core/services/hdfs/src/backend.rs
@@ -82,11 +82,12 @@ impl HdfsBuilder {
         self
     }
 
-    /// Enable append capacity of this backend.
-    ///
-    /// This should be disabled when HDFS runs in non-distributed mode.
-    pub fn enable_append(mut self, enable_append: bool) -> Self {
-        self.config.enable_append = enable_append;
+    /// Deprecated: HDFS append capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "HDFS append capability is enabled by default and this option is no longer needed."
+    )]
+    pub fn enable_append(self, _enable_append: bool) -> Self {
         self
     }
 
@@ -165,7 +166,7 @@ impl Builder for HdfsBuilder {
                             read: true,
 
                             write: true,
-                            write_can_append: self.config.enable_append,
+                            write_can_append: true,
 
                             create_dir: true,
                             delete: true,
diff --git a/core/services/hdfs/src/config.rs b/core/services/hdfs/src/config.rs
index 2e086c82f..8fedb2867 100644
--- a/core/services/hdfs/src/config.rs
+++ b/core/services/hdfs/src/config.rs
@@ -37,7 +37,11 @@ pub struct HdfsConfig {
     pub kerberos_ticket_cache_path: Option<String>,
     /// user of this backend
     pub user: Option<String>,
-    /// enable the append capacity
+    /// Deprecated: HDFS append capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "HDFS append capability is enabled by default and this option is no longer needed."
+    )]
     pub enable_append: bool,
     /// atomic_write_dir of this backend
     pub atomic_write_dir: Option<String>,
@@ -53,7 +57,6 @@ impl Debug for HdfsConfig {
                 &self.kerberos_ticket_cache_path,
             )
             .field("user", &self.user)
-            .field("enable_append", &self.enable_append)
             .field("atomic_write_dir", &self.atomic_write_dir)
             .finish_non_exhaustive()
     }
@@ -77,6 +80,7 @@ impl opendal_core::Configurator for HdfsConfig {
         Self::from_iter(map)
     }
 
+    #[allow(deprecated)]
     fn into_builder(self) -> Self::Builder {
         HdfsBuilder { config: self }
     }
diff --git a/core/services/hdfs/src/docs.md b/core/services/hdfs/src/docs.md
index a410b9a5b..2ccb080a9 100644
--- a/core/services/hdfs/src/docs.md
+++ b/core/services/hdfs/src/docs.md
@@ -28,7 +28,7 @@ HDFS support needs to enable feature `services-hdfs`.
 - `name_node`: Set the name node for backend.
 - `kerberos_ticket_cache_path`: Set the kerberos ticket cache path for backend, this should be gotten by `klist` after `kinit`
 - `user`: Set the user for backend
-- `enable_append`: enable the append capacity. Default is false. 
+- `enable_append`: Deprecated. HDFS append capability is enabled by default and this option is no longer needed.
 
 Refer to [`HdfsBuilder`]'s public API docs for more information.
 
@@ -113,19 +113,14 @@ use opendal_service_hdfs::Hdfs;
 #[tokio::main]
 async fn main() -> Result<(), Box<dyn std::error::Error>> {
     // Create fs backend builder.
-    let mut builder = Hdfs::default()
+    let builder = Hdfs::default()
         // Set the name node for hdfs.
         // If the string starts with a protocol type such as file://, hdfs://, or gs://, this protocol type will be used.
         .name_node("hdfs://127.0.0.1:9000")
         // Set the root for hdfs, all operations will happen under this root.
         //
         // NOTE: the root must be absolute path.
-        .root("/tmp")
-        
-        // Enable the append capacity for hdfs. 
-        // 
-        // Note: HDFS run in non-distributed mode doesn't support append.
-        .enable_append(true);
+        .root("/tmp");
 
     // `Accessor` provides the low level APIs, we will use `Operator` normally.
     let op: Operator = Operator::new(builder)?.finish();