You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/19 17:21:06 UTC
(opendal) branch main updated: refactor(services/hdfs): deprecate append option (#7563)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 9ca0ad793 refactor(services/hdfs): deprecate append option (#7563)
9ca0ad793 is described below
commit 9ca0ad793d533ba35708ef9eccef010ac2af2b3c
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Wed May 20 01:20:14 2026 +0800
refactor(services/hdfs): deprecate append option (#7563)
---
.github/services/hdfs/hdfs_cluster/action.yml | 3 +--
.../hdfs/hdfs_cluster_with_atomic_write_dir/action.yml | 4 ++--
.github/services/hdfs/hdfs_default/action.yml | 2 +-
.github/services/hdfs/hdfs_default_gcs/action.yml | 2 +-
.../hdfs/hdfs_default_on_azurite_azblob/action.yml | 2 +-
.github/services/hdfs/hdfs_default_on_minio_s3/action.yml | 2 +-
.../hdfs/hdfs_default_with_atomic_write_dir/action.yml | 4 ++--
.../services/hdfs_native/hdfs_native_cluster/action.yml | 3 +--
.../OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs | 2 +-
.../src/main/java/org/apache/opendal/ServiceConfig.java | 4 +++-
bindings/python/python/opendal/operator.pyi | 4 ++--
bindings/python/src/services.rs | 6 ++++--
core/services/hdfs-native/src/backend.rs | 14 +++++++-------
core/services/hdfs-native/src/config.rs | 8 ++++++--
core/services/hdfs-native/src/core.rs | 3 ---
core/services/hdfs-native/src/docs.md | 2 +-
core/services/hdfs/src/backend.rs | 13 +++++++------
core/services/hdfs/src/config.rs | 8 ++++++--
core/services/hdfs/src/docs.md | 11 +++--------
19 files changed, 50 insertions(+), 47 deletions(-)
diff --git a/.github/services/hdfs/hdfs_cluster/action.yml b/.github/services/hdfs/hdfs_cluster/action.yml
index 2047c65ff..9670d2400 100644
--- a/.github/services/hdfs/hdfs_cluster/action.yml
+++ b/.github/services/hdfs/hdfs_cluster/action.yml
@@ -35,5 +35,4 @@ runs:
cat << EOF >> $GITHUB_ENV
OPENDAL_HDFS_ROOT=/tmp/opendal/
OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
- OPENDAL_HDFS_ENABLE_APPEND=true
- EOF
\ No newline at end of file
+ EOF
diff --git a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
index 6ad8894b0..804bb9507 100644
--- a/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
+++ b/.github/services/hdfs/hdfs_cluster_with_atomic_write_dir/action.yml
@@ -36,5 +36,5 @@ runs:
OPENDAL_HDFS_ROOT=/tmp/opendal/
OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
OPENDAL_HDFS_NAME_NODE=hdfs://localhost:8020
- OPENDAL_HDFS_ENABLE_APPEND=false
- EOF
\ No newline at end of file
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
+ EOF
diff --git a/.github/services/hdfs/hdfs_default/action.yml b/.github/services/hdfs/hdfs_default/action.yml
index a6e2540dd..35edbf470 100644
--- a/.github/services/hdfs/hdfs_default/action.yml
+++ b/.github/services/hdfs/hdfs_default/action.yml
@@ -31,5 +31,5 @@ runs:
cat << EOF >> $GITHUB_ENV
OPENDAL_HDFS_ROOT=/tmp/opendal/
OPENDAL_HDFS_NAME_NODE=default
- OPENDAL_HDFS_ENABLE_APPEND=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
EOF
diff --git a/.github/services/hdfs/hdfs_default_gcs/action.yml b/.github/services/hdfs/hdfs_default_gcs/action.yml
index 7cb0e8615..c018731a1 100644
--- a/.github/services/hdfs/hdfs_default_gcs/action.yml
+++ b/.github/services/hdfs/hdfs_default_gcs/action.yml
@@ -54,7 +54,7 @@ runs:
LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
OPENDAL_HDFS_ROOT=${OPENDAL_GCS_ROOT}
OPENDAL_HDFS_NAME_NODE=gs://${OPENDAL_GCS_BUCKET}
- OPENDAL_HDFS_ENABLE_APPEND=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
EOF
mkdir -p /tmp/hdfs
diff --git a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
index 11ac22c2e..e888f022b 100644
--- a/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
+++ b/.github/services/hdfs/hdfs_default_on_azurite_azblob/action.yml
@@ -67,5 +67,5 @@ runs:
LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
OPENDAL_HDFS_ROOT=${OPENDAL_AZBLOB_ROOT}
OPENDAL_HDFS_NAME_NODE=wasb://${OPENDAL_AZBLOB_CONTAINER}@${OPENDAL_AZBLOB_ACCOUNT_NAME}.blob.core.windows.net
- OPENDAL_HDFS_ENABLE_APPEND=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
EOF
diff --git a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
index cd8128aa9..98e909a04 100644
--- a/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
+++ b/.github/services/hdfs/hdfs_default_on_minio_s3/action.yml
@@ -64,5 +64,5 @@ runs:
CLASSPATH=${CLASSPATH}
LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${HADOOP_HOME}/lib/native
OPENDAL_HDFS_NAME_NODE=s3a://test
- OPENDAL_HDFS_ENABLE_APPEND=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
EOF
diff --git a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
index 57372570f..8cb9f3e44 100644
--- a/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
+++ b/.github/services/hdfs/hdfs_default_with_atomic_write_dir/action.yml
@@ -32,5 +32,5 @@ runs:
OPENDAL_HDFS_ROOT=/tmp/opendal/
OPENDAL_HDFS_ATOMIC_WRITE_DIR=/tmp/atomic_write_dir/opendal/
OPENDAL_HDFS_NAME_NODE=default
- OPENDAL_HDFS_ENABLE_APPEND=false
- EOF
\ No newline at end of file
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=write_can_append=false
+ EOF
diff --git a/.github/services/hdfs_native/hdfs_native_cluster/action.yml b/.github/services/hdfs_native/hdfs_native_cluster/action.yml
index bf0fa5502..5bf9e4e55 100644
--- a/.github/services/hdfs_native/hdfs_native_cluster/action.yml
+++ b/.github/services/hdfs_native/hdfs_native_cluster/action.yml
@@ -31,5 +31,4 @@ runs:
cat << EOF >> $GITHUB_ENV
OPENDAL_HDFS_NATIVE_ROOT=/tmp/opendal/
OPENDAL_HDFS_NATIVE_NAME_NODE=hdfs://localhost:8020
- OPENDAL_HDFS_NATIVE_ENABLE_APPEND=true
- EOF
\ No newline at end of file
+ EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
index 47ea319c1..c873a846f 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/HdfsNativeServiceConfig.cs
@@ -29,7 +29,7 @@ namespace OpenDAL.ServiceConfig
public sealed class HdfsNativeServiceConfig : IServiceConfig
{
/// <summary>
- /// enable the append capacity
+ /// Deprecated: HDFS Native append capability is enabled by default.
/// </summary>
public bool? EnableAppend { get; init; }
/// <summary>
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index 5f463adf6..41bfabb69 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -1499,7 +1499,9 @@ public interface ServiceConfig {
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
class HdfsNative implements ServiceConfig {
/**
- * <p>enable the append capacity</p>
+ * <p>Deprecated: HDFS Native append capability is enabled by default.</p>
+ *
+ * @deprecated HDFS Native append capability is enabled by default and this option is no longer needed.
*/
public final Boolean enableAppend;
/**
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index 4beaae4c0..0f47da71c 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -1381,7 +1381,7 @@ class AsyncOperator:
Parameters
----------
enable_append : builtins.bool, optional
- enable the append capacity
+ Deprecated: HDFS Native append capability is enabled by default.
name_node : builtins.str, optional
name_node of this backend
options : builtins.dict, optional
@@ -3979,7 +3979,7 @@ class Operator:
Parameters
----------
enable_append : builtins.bool, optional
- enable the append capacity
+ Deprecated: HDFS Native append capability is enabled by default.
name_node : builtins.str, optional
name_node of this backend
options : builtins.dict, optional
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index 62787e690..a2709a70b 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -1145,7 +1145,8 @@ submit! {
Parameters
----------
enable_append : builtins.bool, optional
- enable the append capacity
+ Deprecated: HDFS Native append capability is enabled
+ by default.
name_node : builtins.str, optional
name_node of this backend
options : builtins.dict, optional
@@ -3821,7 +3822,8 @@ submit! {
Parameters
----------
enable_append : builtins.bool, optional
- enable the append capacity
+ Deprecated: HDFS Native append capability is enabled
+ by default.
name_node : builtins.str, optional
name_node of this backend
options : builtins.dict, optional
diff --git a/core/services/hdfs-native/src/backend.rs b/core/services/hdfs-native/src/backend.rs
index 535aa5780..1d32cd1d0 100644
--- a/core/services/hdfs-native/src/backend.rs
+++ b/core/services/hdfs-native/src/backend.rs
@@ -72,11 +72,12 @@ impl HdfsNativeBuilder {
self
}
- /// Enable append capacity of this backend.
- ///
- /// This should be disabled when HDFS runs in non-distributed mode.
- pub fn enable_append(mut self, enable_append: bool) -> Self {
- self.config.enable_append = enable_append;
+ /// Deprecated: HDFS Native append capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "HDFS Native append capability is enabled by default and this option is no longer needed."
+ )]
+ pub fn enable_append(self, _enable_append: bool) -> Self {
self
}
@@ -130,7 +131,7 @@ impl Builder for HdfsNativeBuilder {
read: true,
write: true,
- write_can_append: self.config.enable_append,
+ write_can_append: true,
create_dir: true,
delete: true,
@@ -148,7 +149,6 @@ impl Builder for HdfsNativeBuilder {
},
root,
client: Arc::new(client),
- enable_append: self.config.enable_append,
}),
})
}
diff --git a/core/services/hdfs-native/src/config.rs b/core/services/hdfs-native/src/config.rs
index 2ec6f1427..d2e4a5965 100644
--- a/core/services/hdfs-native/src/config.rs
+++ b/core/services/hdfs-native/src/config.rs
@@ -37,7 +37,11 @@ pub struct HdfsNativeConfig {
pub root: Option<String>,
/// name_node of this backend
pub name_node: Option<String>,
- /// enable the append capacity
+ /// Deprecated: HDFS Native append capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "HDFS Native append capability is enabled by default and this option is no longer needed."
+ )]
pub enable_append: bool,
/// other options for hdfs client
pub options: Option<HashMap<String, String>>,
@@ -48,7 +52,6 @@ impl Debug for HdfsNativeConfig {
f.debug_struct("HdfsNativeConfig")
.field("root", &self.root)
.field("name_node", &self.name_node)
- .field("enable_append", &self.enable_append)
.field("options", &self.options)
.finish_non_exhaustive()
}
@@ -72,6 +75,7 @@ impl opendal_core::Configurator for HdfsNativeConfig {
Self::from_iter(map)
}
+ #[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
HdfsNativeBuilder { config: self }
}
diff --git a/core/services/hdfs-native/src/core.rs b/core/services/hdfs-native/src/core.rs
index da2808c0f..164773907 100644
--- a/core/services/hdfs-native/src/core.rs
+++ b/core/services/hdfs-native/src/core.rs
@@ -31,14 +31,12 @@ pub struct HdfsNativeCore {
pub info: Arc<AccessorInfo>,
pub root: String,
pub client: Arc<hdfs_native::Client>,
- pub enable_append: bool,
}
impl Debug for HdfsNativeCore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("HdfsNativeCore")
.field("root", &self.root)
- .field("enable_append", &self.enable_append)
.finish_non_exhaustive()
}
}
@@ -116,7 +114,6 @@ impl HdfsNativeCore {
let f = if target_exists {
if args.append() {
- assert!(self.enable_append, "append is not enabled");
self.client
.append(&target_path)
.await
diff --git a/core/services/hdfs-native/src/docs.md b/core/services/hdfs-native/src/docs.md
index 9d5230e82..6b2f104cb 100644
--- a/core/services/hdfs-native/src/docs.md
+++ b/core/services/hdfs-native/src/docs.md
@@ -19,4 +19,4 @@ This service can be used to:
- `root`: Set the work dir for backend.
- `name_node`: Set the name node for backend.
-- `enable_append`: enable the append capacity. Default is false.
+- `enable_append`: Deprecated. HDFS Native append capability is enabled by default and this option is no longer needed.
diff --git a/core/services/hdfs/src/backend.rs b/core/services/hdfs/src/backend.rs
index c652308a9..a5d111b00 100644
--- a/core/services/hdfs/src/backend.rs
+++ b/core/services/hdfs/src/backend.rs
@@ -82,11 +82,12 @@ impl HdfsBuilder {
self
}
- /// Enable append capacity of this backend.
- ///
- /// This should be disabled when HDFS runs in non-distributed mode.
- pub fn enable_append(mut self, enable_append: bool) -> Self {
- self.config.enable_append = enable_append;
+ /// Deprecated: HDFS append capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "HDFS append capability is enabled by default and this option is no longer needed."
+ )]
+ pub fn enable_append(self, _enable_append: bool) -> Self {
self
}
@@ -165,7 +166,7 @@ impl Builder for HdfsBuilder {
read: true,
write: true,
- write_can_append: self.config.enable_append,
+ write_can_append: true,
create_dir: true,
delete: true,
diff --git a/core/services/hdfs/src/config.rs b/core/services/hdfs/src/config.rs
index 2e086c82f..8fedb2867 100644
--- a/core/services/hdfs/src/config.rs
+++ b/core/services/hdfs/src/config.rs
@@ -37,7 +37,11 @@ pub struct HdfsConfig {
pub kerberos_ticket_cache_path: Option<String>,
/// user of this backend
pub user: Option<String>,
- /// enable the append capacity
+ /// Deprecated: HDFS append capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "HDFS append capability is enabled by default and this option is no longer needed."
+ )]
pub enable_append: bool,
/// atomic_write_dir of this backend
pub atomic_write_dir: Option<String>,
@@ -53,7 +57,6 @@ impl Debug for HdfsConfig {
&self.kerberos_ticket_cache_path,
)
.field("user", &self.user)
- .field("enable_append", &self.enable_append)
.field("atomic_write_dir", &self.atomic_write_dir)
.finish_non_exhaustive()
}
@@ -77,6 +80,7 @@ impl opendal_core::Configurator for HdfsConfig {
Self::from_iter(map)
}
+ #[allow(deprecated)]
fn into_builder(self) -> Self::Builder {
HdfsBuilder { config: self }
}
diff --git a/core/services/hdfs/src/docs.md b/core/services/hdfs/src/docs.md
index a410b9a5b..2ccb080a9 100644
--- a/core/services/hdfs/src/docs.md
+++ b/core/services/hdfs/src/docs.md
@@ -28,7 +28,7 @@ HDFS support needs to enable feature `services-hdfs`.
- `name_node`: Set the name node for backend.
- `kerberos_ticket_cache_path`: Set the kerberos ticket cache path for backend, this should be gotten by `klist` after `kinit`
- `user`: Set the user for backend
-- `enable_append`: enable the append capacity. Default is false.
+- `enable_append`: Deprecated. HDFS append capability is enabled by default and this option is no longer needed.
Refer to [`HdfsBuilder`]'s public API docs for more information.
@@ -113,19 +113,14 @@ use opendal_service_hdfs::Hdfs;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create fs backend builder.
- let mut builder = Hdfs::default()
+ let builder = Hdfs::default()
// Set the name node for hdfs.
// If the string starts with a protocol type such as file://, hdfs://, or gs://, this protocol type will be used.
.name_node("hdfs://127.0.0.1:9000")
// Set the root for hdfs, all operations will happen under this root.
//
// NOTE: the root must be absolute path.
- .root("/tmp")
-
- // Enable the append capacity for hdfs.
- //
- // Note: HDFS run in non-distributed mode doesn't support append.
- .enable_append(true);
+ .root("/tmp");
// `Accessor` provides the low level APIs, we will use `Operator` normally.
let op: Operator = Operator::new(builder)?.finish();