You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/18 07:36:59 UTC

(opendal) branch main updated: refactor(services/s3): complete capability override migration (#7546)

This is an automated email from the ASF dual-hosted git repository.

Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git


The following commit(s) were added to refs/heads/main by this push:
     new f9cc3edf9 refactor(services/s3): complete capability override migration (#7546)
f9cc3edf9 is described below

commit f9cc3edf97bc6a31b64b261c41fb6018188def58
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon May 18 15:36:54 2026 +0800

    refactor(services/s3): complete capability override migration (#7546)
---
 .github/services/s3/r2/disabled_action.yml         |  6 +--
 .../OpenDAL/ServiceConfig/S3ServiceConfig.cs       |  6 +--
 .../java/org/apache/opendal/ServiceConfig.java     | 19 ++++-----
 bindings/python/python/opendal/operator.pyi        | 44 ++++++--------------
 bindings/python/src/services.rs                    | 44 ++++++--------------
 core/services/s3/src/backend.rs                    | 48 +++++++++++-----------
 core/services/s3/src/compatible_services.md        |  7 +++-
 core/services/s3/src/config.rs                     | 30 ++++++--------
 core/services/s3/src/docs.md                       |  3 ++
 9 files changed, 82 insertions(+), 125 deletions(-)

diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml
index fee1b3988..dd09ff264 100644
--- a/.github/services/s3/r2/disabled_action.yml
+++ b/.github/services/s3/r2/disabled_action.yml
@@ -31,14 +31,12 @@ runs:
         OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
         OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
 
-    # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation
+    # R2 has a lower delete batch limit and doesn't support stat override response queries.
     # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
     - name: Add extra settings
       shell: bash
       run: |
         cat << EOF >> $GITHUB_ENV
         OPENDAL_S3_REGION=auto
-        OPENDAL_S3_DELETE_MAX_SIZE=700
-        OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true
-        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
+        OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
         EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
index 36d4666be..b71dcc398 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
@@ -42,7 +42,7 @@ namespace OpenDAL.ServiceConfig
         [System.Obsolete("Please use SkipSignature instead of AllowAnonymous")]
         public bool? AllowAnonymous { get; init; }
         /// <summary>
-        /// Set maximum batch operations of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+        /// Deprecated: S3 delete batch capability is enabled by default.
         /// </summary>
         public long? BatchMaxOperations { get; init; }
         /// <summary>
@@ -58,7 +58,7 @@ namespace OpenDAL.ServiceConfig
         /// </summary>
         public string? DefaultStorageClass { get; init; }
         /// <summary>
-        /// Set the maximum delete size of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+        /// Deprecated: S3 delete batch capability is enabled by default.
         /// </summary>
         public long? DeleteMaxSize { get; init; }
         /// <summary>
@@ -74,7 +74,7 @@ namespace OpenDAL.ServiceConfig
         /// </summary>
         public bool? DisableListObjectsV2 { get; init; }
         /// <summary>
-        /// Disable stat with override so that opendal will not send stat request with override queries. For example, R2 doesn't support stat with response_content_type query.
+        /// Deprecated: S3 stat override capabilities are enabled by default.
         /// </summary>
         public bool? DisableStatWithOverride { get; init; }
         /// <summary>
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index 60c763bbc..5c434679a 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -2838,12 +2838,9 @@ public interface ServiceConfig {
          */
         public final Map<String, String> assumeRoleSessionTags;
         /**
-         * <p>Set maximum batch operations of this backend.</p>
-         * <p>Some compatible services have a limit on the number of operations in a batch request.
-         * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
-         * <p>Please tune this value based on services' document.</p>
+         * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
          *
-         * @deprecated Please use `delete_max_size` instead of `batch_max_operations`
+         * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
          */
         public final Long batchMaxOperations;
         /**
@@ -2885,10 +2882,9 @@ public interface ServiceConfig {
          */
         public final String defaultStorageClass;
         /**
-         * <p>Set the maximum delete size of this backend.</p>
-         * <p>Some compatible services have a limit on the number of operations in a batch request.
-         * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
-         * <p>Please tune this value based on services' document.</p>
+         * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
+         *
+         * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
          */
         public final Long deleteMaxSize;
         /**
@@ -2914,8 +2910,9 @@ public interface ServiceConfig {
          */
         public final Boolean disableListObjectsV2;
         /**
-         * <p>Disable stat with override so that opendal will not send stat request with override queries.</p>
-         * <p>For example, R2 doesn't support stat with <code>response_content_type</code> query.</p>
+         * <p>Deprecated: S3 stat override capabilities are enabled by default.</p>
+         *
+         * @deprecated S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints.
          */
         public final Boolean disableStatWithOverride;
         /**
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index d36612010..d2060aa0a 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -2150,12 +2150,8 @@ class AsyncOperator:
         assume_role_session_tags : builtins.dict, optional
             assume_role_session_tags for this backend.
         batch_max_operations : builtins.int, optional
-            Set maximum batch operations of this backend.
-            Some compatible services have a limit on the number
-            of operations in a batch request.
-            For example, R2 could return `Internal Error` while
-            batch delete 1000 files.
-            Please tune this value based on services' document.
+            Deprecated: S3 delete batch capability is enabled by
+            default.
         bucket : builtins.str
             bucket name of this backend.
             required.
@@ -2177,12 +2173,8 @@ class AsyncOperator:
             `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
             compatible services don't support all of them
         delete_max_size : builtins.int, optional
-            Set the maximum delete size of this backend.
-            Some compatible services have a limit on the number
-            of operations in a batch request.
-            For example, R2 could return `Internal Error` while
-            batch delete 1000 files.
-            Please tune this value based on services' document.
+            Deprecated: S3 delete batch capability is enabled by
+            default.
         disable_config_load : builtins.bool, optional
             Disable config load so that opendal will not load
             config from environment.
@@ -2200,10 +2192,8 @@ class AsyncOperator:
             This option allows users to switch back to the older
             List Objects V1.
         disable_stat_with_override : builtins.bool, optional
-            Disable stat with override so that opendal will not
-            send stat request with override queries.
-            For example, R2 doesn't support stat with
-            `response_content_type` query.
+            Deprecated: S3 stat override capabilities are
+            enabled by default.
         disable_write_with_if_match : builtins.bool, optional
             Disable write with if match so that opendal will not
             send write request with if match headers.
@@ -4762,12 +4752,8 @@ class Operator:
         assume_role_session_tags : builtins.dict, optional
             assume_role_session_tags for this backend.
         batch_max_operations : builtins.int, optional
-            Set maximum batch operations of this backend.
-            Some compatible services have a limit on the number
-            of operations in a batch request.
-            For example, R2 could return `Internal Error` while
-            batch delete 1000 files.
-            Please tune this value based on services' document.
+            Deprecated: S3 delete batch capability is enabled by
+            default.
         bucket : builtins.str
             bucket name of this backend.
             required.
@@ -4789,12 +4775,8 @@ class Operator:
             `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
             compatible services don't support all of them
         delete_max_size : builtins.int, optional
-            Set the maximum delete size of this backend.
-            Some compatible services have a limit on the number
-            of operations in a batch request.
-            For example, R2 could return `Internal Error` while
-            batch delete 1000 files.
-            Please tune this value based on services' document.
+            Deprecated: S3 delete batch capability is enabled by
+            default.
         disable_config_load : builtins.bool, optional
             Disable config load so that opendal will not load
             config from environment.
@@ -4812,10 +4794,8 @@ class Operator:
             This option allows users to switch back to the older
             List Objects V1.
         disable_stat_with_override : builtins.bool, optional
-            Disable stat with override so that opendal will not
-            send stat request with override queries.
-            For example, R2 doesn't support stat with
-            `response_content_type` query.
+            Deprecated: S3 stat override capabilities are
+            enabled by default.
         disable_write_with_if_match : builtins.bool, optional
             Disable write with if match so that opendal will not
             send write request with if match headers.
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index b62cebdd9..36b718e75 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -2085,12 +2085,8 @@ submit! {
                 assume_role_session_tags : builtins.dict, optional
                     assume_role_session_tags for this backend.
                 batch_max_operations : builtins.int, optional
-                    Set maximum batch operations of this backend.
-                    Some compatible services have a limit on the number
-                    of operations in a batch request.
-                    For example, R2 could return `Internal Error` while
-                    batch delete 1000 files.
-                    Please tune this value based on services' document.
+                    Deprecated: S3 delete batch capability is enabled by
+                    default.
                 bucket : builtins.str
                     bucket name of this backend.
                     required.
@@ -2112,12 +2108,8 @@ submit! {
                     `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
                     compatible services don't support all of them
                 delete_max_size : builtins.int, optional
-                    Set the maximum delete size of this backend.
-                    Some compatible services have a limit on the number
-                    of operations in a batch request.
-                    For example, R2 could return `Internal Error` while
-                    batch delete 1000 files.
-                    Please tune this value based on services' document.
+                    Deprecated: S3 delete batch capability is enabled by
+                    default.
                 disable_config_load : builtins.bool, optional
                     Disable config load so that opendal will not load
                     config from environment.
@@ -2135,10 +2127,8 @@ submit! {
                     This option allows users to switch back to the older
                     List Objects V1.
                 disable_stat_with_override : builtins.bool, optional
-                    Disable stat with override so that opendal will not
-                    send stat request with override queries.
-                    For example, R2 doesn't support stat with
-                    `response_content_type` query.
+                    Deprecated: S3 stat override capabilities are
+                    enabled by default.
                 disable_write_with_if_match : builtins.bool, optional
                     Deprecated: S3 write with If-Match capability is
                     enabled by default.
@@ -4775,12 +4765,8 @@ submit! {
                 assume_role_session_tags : builtins.dict, optional
                     assume_role_session_tags for this backend.
                 batch_max_operations : builtins.int, optional
-                    Set maximum batch operations of this backend.
-                    Some compatible services have a limit on the number
-                    of operations in a batch request.
-                    For example, R2 could return `Internal Error` while
-                    batch delete 1000 files.
-                    Please tune this value based on services' document.
+                    Deprecated: S3 delete batch capability is enabled by
+                    default.
                 bucket : builtins.str
                     bucket name of this backend.
                     required.
@@ -4802,12 +4788,8 @@ submit! {
                     `REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
                     compatible services don't support all of them
                 delete_max_size : builtins.int, optional
-                    Set the maximum delete size of this backend.
-                    Some compatible services have a limit on the number
-                    of operations in a batch request.
-                    For example, R2 could return `Internal Error` while
-                    batch delete 1000 files.
-                    Please tune this value based on services' document.
+                    Deprecated: S3 delete batch capability is enabled by
+                    default.
                 disable_config_load : builtins.bool, optional
                     Disable config load so that opendal will not load
                     config from environment.
@@ -4825,10 +4807,8 @@ submit! {
                     This option allows users to switch back to the older
                     List Objects V1.
                 disable_stat_with_override : builtins.bool, optional
-                    Disable stat with override so that opendal will not
-                    send stat request with override queries.
-                    For example, R2 doesn't support stat with
-                    `response_content_type` query.
+                    Deprecated: S3 stat override capabilities are
+                    enabled by default.
                 disable_write_with_if_match : builtins.bool, optional
                     Deprecated: S3 write with If-Match capability is
                     enabled by default.
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index ab20d802d..87c67812f 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -457,11 +457,12 @@ impl S3Builder {
         self
     }
 
-    /// Disable stat with override so that opendal will not send stat request with override queries.
-    ///
-    /// For example, R2 doesn't support stat with `response_content_type` query.
-    pub fn disable_stat_with_override(mut self) -> Self {
-        self.config.disable_stat_with_override = true;
+    /// Deprecated: S3 stat override capabilities are enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "S3 stat override capabilities are enabled by default and this option is no longer needed."
+    )]
+    pub fn disable_stat_with_override(self) -> Self {
         self
     }
 
@@ -544,21 +545,21 @@ impl S3Builder {
         endpoint
     }
 
-    /// Set maximum batch operations of this backend.
+    /// Deprecated: S3 delete batch capability is enabled by default.
     #[deprecated(
-        since = "0.52.0",
-        note = "Please use `delete_max_size` instead of `batch_max_operations`"
+        since = "0.57.0",
+        note = "S3 delete batch capability is enabled by default and this option is no longer needed."
     )]
-    pub fn batch_max_operations(mut self, batch_max_operations: usize) -> Self {
-        self.config.delete_max_size = Some(batch_max_operations);
-
+    pub fn batch_max_operations(self, _batch_max_operations: usize) -> Self {
         self
     }
 
-    /// Set maximum delete operations of this backend.
-    pub fn delete_max_size(mut self, delete_max_size: usize) -> Self {
-        self.config.delete_max_size = Some(delete_max_size);
-
+    /// Deprecated: S3 delete batch capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "S3 delete batch capability is enabled by default and this option is no longer needed."
+    )]
+    pub fn delete_max_size(self, _delete_max_size: usize) -> Self {
         self
     }
 
@@ -655,6 +656,10 @@ impl S3Builder {
         }
 
         // If this bucket is AWS, we can try to match the endpoint.
+        if endpoint == "https://s3.amazonaws.com" {
+            return Some("us-east-1".to_string());
+        }
+
         if let Some(region) = endpoint
             .strip_prefix("https://s3.")
             .and_then(|v| v.strip_suffix(".amazonaws.com"))
@@ -909,10 +914,6 @@ impl Builder for S3Builder {
         // Create the signer
         let signer = Signer::new(ctx, provider, request_signer);
 
-        let delete_max_size = config
-            .delete_max_size
-            .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
-
         Ok(S3Backend {
             core: Arc::new(S3Core {
                 info: {
@@ -925,10 +926,9 @@ impl Builder for S3Builder {
                             stat_with_if_none_match: true,
                             stat_with_if_modified_since: true,
                             stat_with_if_unmodified_since: true,
-                            stat_with_override_cache_control: !config.disable_stat_with_override,
-                            stat_with_override_content_disposition: !config
-                                .disable_stat_with_override,
-                            stat_with_override_content_type: !config.disable_stat_with_override,
+                            stat_with_override_cache_control: true,
+                            stat_with_override_content_disposition: true,
+                            stat_with_override_content_type: true,
                             stat_with_version: true,
 
                             read: true,
@@ -968,7 +968,7 @@ impl Builder for S3Builder {
                             },
 
                             delete: true,
-                            delete_max_size: Some(delete_max_size),
+                            delete_max_size: Some(DEFAULT_BATCH_MAX_OPERATIONS),
                             delete_with_version: true,
 
                             copy: true,
diff --git a/core/services/s3/src/compatible_services.md b/core/services/s3/src/compatible_services.md
index 12a6e270e..27af1d4f4 100644
--- a/core/services/s3/src/compatible_services.md
+++ b/core/services/s3/src/compatible_services.md
@@ -110,9 +110,13 @@ To connect to r2, we need to set:
 - `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
 - `bucket`: The bucket name of r2.
 - `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
-- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.
 - `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`.
 
+R2 has the following capability differences from S3:
+
+- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`.
+- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`.
+
 ### Google Cloud Storage XML API
 [Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API.
 - `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com`
@@ -123,4 +127,3 @@ To connect to r2, we need to set:
 Ceph supports a RESTful API that is compatible with the basic data access model of the Amazon S3 API.
 
 For more information, refer: <https://docs.ceph.com/en/latest/radosgw/s3/>
-
diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs
index 1b6e33ff3..912a6eefc 100644
--- a/core/services/s3/src/config.rs
+++ b/core/services/s3/src/config.rs
@@ -190,27 +190,23 @@ pub struct S3Config {
         alias = "virtual_hosted_style_request"
     )]
     pub enable_virtual_host_style: bool,
-    /// Set maximum batch operations of this backend.
-    ///
-    /// Some compatible services have a limit on the number of operations in a batch request.
-    /// For example, R2 could return `Internal Error` while batch delete 1000 files.
-    ///
-    /// Please tune this value based on services' document.
+    /// Deprecated: S3 delete batch capability is enabled by default.
     #[deprecated(
-        since = "0.52.0",
-        note = "Please use `delete_max_size` instead of `batch_max_operations`"
+        since = "0.57.0",
+        note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
     )]
     pub batch_max_operations: Option<usize>,
-    /// Set the maximum delete size of this backend.
-    ///
-    /// Some compatible services have a limit on the number of operations in a batch request.
-    /// For example, R2 could return `Internal Error` while batch delete 1000 files.
-    ///
-    /// Please tune this value based on services' document.
+    /// Deprecated: S3 delete batch capability is enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
+    )]
     pub delete_max_size: Option<usize>,
-    /// Disable stat with override so that opendal will not send stat request with override queries.
-    ///
-    /// For example, R2 doesn't support stat with `response_content_type` query.
+    /// Deprecated: S3 stat override capabilities are enabled by default.
+    #[deprecated(
+        since = "0.57.0",
+        note = "S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints."
+    )]
     pub disable_stat_with_override: bool,
     /// Checksum Algorithm to use when sending checksums in HTTP headers.
     /// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md
index 8b3fc6024..469dd5006 100644
--- a/core/services/s3/src/docs.md
+++ b/core/services/s3/src/docs.md
@@ -30,6 +30,9 @@ This service can be used to:
 - `disable_config_load`: Disable aws config load from env.
 - `enable_virtual_host_style`: Enable virtual host style.
 - `enable_request_payer`: Enable the request payer for backend.
+- `batch_max_operations`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `delete_max_size`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `disable_stat_with_override`: Deprecated. S3 stat override capabilities are enabled by default and this option is no longer needed.
 - `disable_write_with_if_match`: Deprecated. S3 write with If-Match capability is enabled by default and this option is no longer needed.
 - `enable_versioning`: Deprecated. S3 versioning capability is enabled by default and this option is no longer needed.
 - `enable_write_with_append`: Deprecated. S3 append capability is enabled by default and this option is no longer needed.