You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/18 07:36:59 UTC
(opendal) branch main updated: refactor(services/s3): complete capability override migration (#7546)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new f9cc3edf9 refactor(services/s3): complete capability override migration (#7546)
f9cc3edf9 is described below
commit f9cc3edf97bc6a31b64b261c41fb6018188def58
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon May 18 15:36:54 2026 +0800
refactor(services/s3): complete capability override migration (#7546)
---
.github/services/s3/r2/disabled_action.yml | 6 +--
.../OpenDAL/ServiceConfig/S3ServiceConfig.cs | 6 +--
.../java/org/apache/opendal/ServiceConfig.java | 19 ++++-----
bindings/python/python/opendal/operator.pyi | 44 ++++++--------------
bindings/python/src/services.rs | 44 ++++++--------------
core/services/s3/src/backend.rs | 48 +++++++++++-----------
core/services/s3/src/compatible_services.md | 7 +++-
core/services/s3/src/config.rs | 30 ++++++--------
core/services/s3/src/docs.md | 3 ++
9 files changed, 82 insertions(+), 125 deletions(-)
diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml
index fee1b3988..dd09ff264 100644
--- a/.github/services/s3/r2/disabled_action.yml
+++ b/.github/services/s3/r2/disabled_action.yml
@@ -31,14 +31,12 @@ runs:
OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
- # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation
+ # R2 has a lower delete batch limit and doesn't support stat override response queries.
# Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
- name: Add extra settings
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
OPENDAL_S3_REGION=auto
- OPENDAL_S3_DELETE_MAX_SIZE=700
- OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true
- OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
index 36d4666be..b71dcc398 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
@@ -42,7 +42,7 @@ namespace OpenDAL.ServiceConfig
[System.Obsolete("Please use SkipSignature instead of AllowAnonymous")]
public bool? AllowAnonymous { get; init; }
/// <summary>
- /// Set maximum batch operations of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
/// </summary>
public long? BatchMaxOperations { get; init; }
/// <summary>
@@ -58,7 +58,7 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public string? DefaultStorageClass { get; init; }
/// <summary>
- /// Set the maximum delete size of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
/// </summary>
public long? DeleteMaxSize { get; init; }
/// <summary>
@@ -74,7 +74,7 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public bool? DisableListObjectsV2 { get; init; }
/// <summary>
- /// Disable stat with override so that opendal will not send stat request with override queries. For example, R2 doesn't support stat with response_content_type query.
+ /// Deprecated: S3 stat override capabilities are enabled by default.
/// </summary>
public bool? DisableStatWithOverride { get; init; }
/// <summary>
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index 60c763bbc..5c434679a 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -2838,12 +2838,9 @@ public interface ServiceConfig {
*/
public final Map<String, String> assumeRoleSessionTags;
/**
- * <p>Set maximum batch operations of this backend.</p>
- * <p>Some compatible services have a limit on the number of operations in a batch request.
- * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
- * <p>Please tune this value based on services' document.</p>
+ * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
*
- * @deprecated Please use `delete_max_size` instead of `batch_max_operations`
+ * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
*/
public final Long batchMaxOperations;
/**
@@ -2885,10 +2882,9 @@ public interface ServiceConfig {
*/
public final String defaultStorageClass;
/**
- * <p>Set the maximum delete size of this backend.</p>
- * <p>Some compatible services have a limit on the number of operations in a batch request.
- * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
- * <p>Please tune this value based on services' document.</p>
+ * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
+ *
+ * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
*/
public final Long deleteMaxSize;
/**
@@ -2914,8 +2910,9 @@ public interface ServiceConfig {
*/
public final Boolean disableListObjectsV2;
/**
- * <p>Disable stat with override so that opendal will not send stat request with override queries.</p>
- * <p>For example, R2 doesn't support stat with <code>response_content_type</code> query.</p>
+ * <p>Deprecated: S3 stat override capabilities are enabled by default.</p>
+ *
+ * @deprecated S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints.
*/
public final Boolean disableStatWithOverride;
/**
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index d36612010..d2060aa0a 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -2150,12 +2150,8 @@ class AsyncOperator:
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -2177,12 +2173,8 @@ class AsyncOperator:
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -2200,10 +2192,8 @@ class AsyncOperator:
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Disable write with if match so that opendal will not
send write request with if match headers.
@@ -4762,12 +4752,8 @@ class Operator:
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -4789,12 +4775,8 @@ class Operator:
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -4812,10 +4794,8 @@ class Operator:
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Disable write with if match so that opendal will not
send write request with if match headers.
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index b62cebdd9..36b718e75 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -2085,12 +2085,8 @@ submit! {
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -2112,12 +2108,8 @@ submit! {
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -2135,10 +2127,8 @@ submit! {
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Deprecated: S3 write with If-Match capability is
enabled by default.
@@ -4775,12 +4765,8 @@ submit! {
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -4802,12 +4788,8 @@ submit! {
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -4825,10 +4807,8 @@ submit! {
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Deprecated: S3 write with If-Match capability is
enabled by default.
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index ab20d802d..87c67812f 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -457,11 +457,12 @@ impl S3Builder {
self
}
- /// Disable stat with override so that opendal will not send stat request with override queries.
- ///
- /// For example, R2 doesn't support stat with `response_content_type` query.
- pub fn disable_stat_with_override(mut self) -> Self {
- self.config.disable_stat_with_override = true;
+ /// Deprecated: S3 stat override capabilities are enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 stat override capabilities are enabled by default and this option is no longer needed."
+ )]
+ pub fn disable_stat_with_override(self) -> Self {
self
}
@@ -544,21 +545,21 @@ impl S3Builder {
endpoint
}
- /// Set maximum batch operations of this backend.
+ /// Deprecated: S3 delete batch capability is enabled by default.
#[deprecated(
- since = "0.52.0",
- note = "Please use `delete_max_size` instead of `batch_max_operations`"
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default and this option is no longer needed."
)]
- pub fn batch_max_operations(mut self, batch_max_operations: usize) -> Self {
- self.config.delete_max_size = Some(batch_max_operations);
-
+ pub fn batch_max_operations(self, _batch_max_operations: usize) -> Self {
self
}
- /// Set maximum delete operations of this backend.
- pub fn delete_max_size(mut self, delete_max_size: usize) -> Self {
- self.config.delete_max_size = Some(delete_max_size);
-
+ /// Deprecated: S3 delete batch capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default and this option is no longer needed."
+ )]
+ pub fn delete_max_size(self, _delete_max_size: usize) -> Self {
self
}
@@ -655,6 +656,10 @@ impl S3Builder {
}
// If this bucket is AWS, we can try to match the endpoint.
+ if endpoint == "https://s3.amazonaws.com" {
+ return Some("us-east-1".to_string());
+ }
+
if let Some(region) = endpoint
.strip_prefix("https://s3.")
.and_then(|v| v.strip_suffix(".amazonaws.com"))
@@ -909,10 +914,6 @@ impl Builder for S3Builder {
// Create the signer
let signer = Signer::new(ctx, provider, request_signer);
- let delete_max_size = config
- .delete_max_size
- .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
-
Ok(S3Backend {
core: Arc::new(S3Core {
info: {
@@ -925,10 +926,9 @@ impl Builder for S3Builder {
stat_with_if_none_match: true,
stat_with_if_modified_since: true,
stat_with_if_unmodified_since: true,
- stat_with_override_cache_control: !config.disable_stat_with_override,
- stat_with_override_content_disposition: !config
- .disable_stat_with_override,
- stat_with_override_content_type: !config.disable_stat_with_override,
+ stat_with_override_cache_control: true,
+ stat_with_override_content_disposition: true,
+ stat_with_override_content_type: true,
stat_with_version: true,
read: true,
@@ -968,7 +968,7 @@ impl Builder for S3Builder {
},
delete: true,
- delete_max_size: Some(delete_max_size),
+ delete_max_size: Some(DEFAULT_BATCH_MAX_OPERATIONS),
delete_with_version: true,
copy: true,
diff --git a/core/services/s3/src/compatible_services.md b/core/services/s3/src/compatible_services.md
index 12a6e270e..27af1d4f4 100644
--- a/core/services/s3/src/compatible_services.md
+++ b/core/services/s3/src/compatible_services.md
@@ -110,9 +110,13 @@ To connect to r2, we need to set:
- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
- `bucket`: The bucket name of r2.
- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
-- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.
- `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`.
+R2 has the following capability differences from S3:
+
+- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`.
+- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`.
+
### Google Cloud Storage XML API
[Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API.
- `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com`
@@ -123,4 +127,3 @@ To connect to r2, we need to set:
Ceph supports a RESTful API that is compatible with the basic data access model of the Amazon S3 API.
For more information, refer: <https://docs.ceph.com/en/latest/radosgw/s3/>
-
diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs
index 1b6e33ff3..912a6eefc 100644
--- a/core/services/s3/src/config.rs
+++ b/core/services/s3/src/config.rs
@@ -190,27 +190,23 @@ pub struct S3Config {
alias = "virtual_hosted_style_request"
)]
pub enable_virtual_host_style: bool,
- /// Set maximum batch operations of this backend.
- ///
- /// Some compatible services have a limit on the number of operations in a batch request.
- /// For example, R2 could return `Internal Error` while batch delete 1000 files.
- ///
- /// Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
#[deprecated(
- since = "0.52.0",
- note = "Please use `delete_max_size` instead of `batch_max_operations`"
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
)]
pub batch_max_operations: Option<usize>,
- /// Set the maximum delete size of this backend.
- ///
- /// Some compatible services have a limit on the number of operations in a batch request.
- /// For example, R2 could return `Internal Error` while batch delete 1000 files.
- ///
- /// Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
+ )]
pub delete_max_size: Option<usize>,
- /// Disable stat with override so that opendal will not send stat request with override queries.
- ///
- /// For example, R2 doesn't support stat with `response_content_type` query.
+ /// Deprecated: S3 stat override capabilities are enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints."
+ )]
pub disable_stat_with_override: bool,
/// Checksum Algorithm to use when sending checksums in HTTP headers.
/// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md
index 8b3fc6024..469dd5006 100644
--- a/core/services/s3/src/docs.md
+++ b/core/services/s3/src/docs.md
@@ -30,6 +30,9 @@ This service can be used to:
- `disable_config_load`: Disable aws config load from env.
- `enable_virtual_host_style`: Enable virtual host style.
- `enable_request_payer`: Enable the request payer for backend.
+- `batch_max_operations`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `delete_max_size`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `disable_stat_with_override`: Deprecated. S3 stat override capabilities are enabled by default and this option is no longer needed.
- `disable_write_with_if_match`: Deprecated. S3 write with If-Match capability is enabled by default and this option is no longer needed.
- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default and this option is no longer needed.
- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default and this option is no longer needed.