You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/05/18 05:22:31 UTC
(opendal) branch xuanwo/s3-capability-override-cleanup created (now 668ed56b3)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a change to branch xuanwo/s3-capability-override-cleanup
in repository https://gitbox.apache.org/repos/asf/opendal.git
at 668ed56b3 refactor(services/s3): complete capability override migration
This branch includes the following new commits:
new 668ed56b3 refactor(services/s3): complete capability override migration
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: refactor(services/s3): complete capability override migration
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch xuanwo/s3-capability-override-cleanup
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit 668ed56b38979520c6a0182d694ffb5b32363b36
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon May 18 13:22:18 2026 +0800
refactor(services/s3): complete capability override migration
---
.github/services/s3/r2/disabled_action.yml | 6 +--
.../OpenDAL/ServiceConfig/S3ServiceConfig.cs | 6 +--
.../java/org/apache/opendal/ServiceConfig.java | 19 ++++-----
bindings/python/python/opendal/operator.pyi | 44 ++++++--------------
bindings/python/src/services.rs | 44 ++++++--------------
core/services/s3/src/backend.rs | 48 +++++++++++-----------
core/services/s3/src/compatible_services.md | 7 +++-
core/services/s3/src/config.rs | 30 ++++++--------
core/services/s3/src/docs.md | 3 ++
9 files changed, 82 insertions(+), 125 deletions(-)
diff --git a/.github/services/s3/r2/disabled_action.yml b/.github/services/s3/r2/disabled_action.yml
index fee1b3988..dd09ff264 100644
--- a/.github/services/s3/r2/disabled_action.yml
+++ b/.github/services/s3/r2/disabled_action.yml
@@ -31,14 +31,12 @@ runs:
OPENDAL_S3_ACCESS_KEY_ID: op://services/r2/access_key_id
OPENDAL_S3_SECRET_ACCESS_KEY: op://services/r2/secret_access_key
- # OPENDAL_S3_BATCH_MAX_OPERATIONS is the R2's limitation
+ # R2 has a lower delete batch limit and doesn't support stat override response queries.
# Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
- name: Add extra settings
shell: bash
run: |
cat << EOF >> $GITHUB_ENV
OPENDAL_S3_REGION=auto
- OPENDAL_S3_DELETE_MAX_SIZE=700
- OPENDAL_S3_DISABLE_STAT_WITH_OVERRIDE=true
- OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false
+ OPENDAL_TEST_CAPABILITY_OVERRIDES=stat_with_version=false,read_with_version=false,delete_with_version=false,list_with_versions=false,list_with_deleted=false,write_can_append=false,delete_max_size=700,stat_with_override_cache_control=false,stat_with_override_content_disposition=false,stat_with_override_content_type=false
EOF
diff --git a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
index 1ad0cf942..acf0310df 100644
--- a/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
+++ b/bindings/dotnet/OpenDAL/ServiceConfig/S3ServiceConfig.cs
@@ -37,7 +37,7 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public bool? AllowAnonymous { get; init; }
/// <summary>
- /// Set maximum batch operations of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
/// </summary>
public long? BatchMaxOperations { get; init; }
/// <summary>
@@ -53,7 +53,7 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public string? DefaultStorageClass { get; init; }
/// <summary>
- /// Set the maximum delete size of this backend. Some compatible services have a limit on the number of operations in a batch request. For example, R2 could return Internal Error while batch delete 1000 files. Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
/// </summary>
public long? DeleteMaxSize { get; init; }
/// <summary>
@@ -69,7 +69,7 @@ namespace OpenDAL.ServiceConfig
/// </summary>
public bool? DisableListObjectsV2 { get; init; }
/// <summary>
- /// Disable stat with override so that opendal will not send stat request with override queries. For example, R2 doesn't support stat with response_content_type query.
+ /// Deprecated: S3 stat override capabilities are enabled by default.
/// </summary>
public bool? DisableStatWithOverride { get; init; }
/// <summary>
diff --git a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
index d08e9300a..f00344d08 100644
--- a/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
+++ b/bindings/java/src/main/java/org/apache/opendal/ServiceConfig.java
@@ -2836,12 +2836,9 @@ public interface ServiceConfig {
*/
public final Map<String, String> assumeRoleSessionTags;
/**
- * <p>Set maximum batch operations of this backend.</p>
- * <p>Some compatible services have a limit on the number of operations in a batch request.
- * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
- * <p>Please tune this value based on services' document.</p>
+ * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
*
- * @deprecated Please use `delete_max_size` instead of `batch_max_operations`
+ * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
*/
public final Long batchMaxOperations;
/**
@@ -2883,10 +2880,9 @@ public interface ServiceConfig {
*/
public final String defaultStorageClass;
/**
- * <p>Set the maximum delete size of this backend.</p>
- * <p>Some compatible services have a limit on the number of operations in a batch request.
- * For example, R2 could return <code>Internal Error</code> while batch delete 1000 files.</p>
- * <p>Please tune this value based on services' document.</p>
+ * <p>Deprecated: S3 delete batch capability is enabled by default.</p>
+ *
+ * @deprecated S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
*/
public final Long deleteMaxSize;
/**
@@ -2912,8 +2908,9 @@ public interface ServiceConfig {
*/
public final Boolean disableListObjectsV2;
/**
- * <p>Disable stat with override so that opendal will not send stat request with override queries.</p>
- * <p>For example, R2 doesn't support stat with <code>response_content_type</code> query.</p>
+ * <p>Deprecated: S3 stat override capabilities are enabled by default.</p>
+ *
+ * @deprecated S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints.
*/
public final Boolean disableStatWithOverride;
/**
diff --git a/bindings/python/python/opendal/operator.pyi b/bindings/python/python/opendal/operator.pyi
index 2a35b53d8..32a90030b 100644
--- a/bindings/python/python/opendal/operator.pyi
+++ b/bindings/python/python/opendal/operator.pyi
@@ -2145,12 +2145,8 @@ class AsyncOperator:
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -2172,12 +2168,8 @@ class AsyncOperator:
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -2195,10 +2187,8 @@ class AsyncOperator:
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Disable write with if match so that opendal will not
send write request with if match headers.
@@ -4749,12 +4739,8 @@ class Operator:
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -4776,12 +4762,8 @@ class Operator:
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -4799,10 +4781,8 @@ class Operator:
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Disable write with if match so that opendal will not
send write request with if match headers.
diff --git a/bindings/python/src/services.rs b/bindings/python/src/services.rs
index c29b2e1d4..ea87826c5 100644
--- a/bindings/python/src/services.rs
+++ b/bindings/python/src/services.rs
@@ -2084,12 +2084,8 @@ submit! {
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -2111,12 +2107,8 @@ submit! {
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -2134,10 +2126,8 @@ submit! {
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Deprecated: S3 write with If-Match capability is
enabled by default.
@@ -4770,12 +4760,8 @@ submit! {
assume_role_session_tags : builtins.dict, optional
assume_role_session_tags for this backend.
batch_max_operations : builtins.int, optional
- Set maximum batch operations of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
bucket : builtins.str
bucket name of this backend.
required.
@@ -4797,12 +4783,8 @@ submit! {
`REDUCED_REDUNDANCY` - `STANDARD` - `STANDARD_IA` S3
compatible services don't support all of them
delete_max_size : builtins.int, optional
- Set the maximum delete size of this backend.
- Some compatible services have a limit on the number
- of operations in a batch request.
- For example, R2 could return `Internal Error` while
- batch delete 1000 files.
- Please tune this value based on services' document.
+ Deprecated: S3 delete batch capability is enabled by
+ default.
disable_config_load : builtins.bool, optional
Disable config load so that opendal will not load
config from environment.
@@ -4820,10 +4802,8 @@ submit! {
This option allows users to switch back to the older
List Objects V1.
disable_stat_with_override : builtins.bool, optional
- Disable stat with override so that opendal will not
- send stat request with override queries.
- For example, R2 doesn't support stat with
- `response_content_type` query.
+ Deprecated: S3 stat override capabilities are
+ enabled by default.
disable_write_with_if_match : builtins.bool, optional
Deprecated: S3 write with If-Match capability is
enabled by default.
diff --git a/core/services/s3/src/backend.rs b/core/services/s3/src/backend.rs
index c99581d5c..3d8704f2c 100644
--- a/core/services/s3/src/backend.rs
+++ b/core/services/s3/src/backend.rs
@@ -448,11 +448,12 @@ impl S3Builder {
self
}
- /// Disable stat with override so that opendal will not send stat request with override queries.
- ///
- /// For example, R2 doesn't support stat with `response_content_type` query.
- pub fn disable_stat_with_override(mut self) -> Self {
- self.config.disable_stat_with_override = true;
+ /// Deprecated: S3 stat override capabilities are enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 stat override capabilities are enabled by default and this option is no longer needed."
+ )]
+ pub fn disable_stat_with_override(self) -> Self {
self
}
@@ -535,21 +536,21 @@ impl S3Builder {
endpoint
}
- /// Set maximum batch operations of this backend.
+ /// Deprecated: S3 delete batch capability is enabled by default.
#[deprecated(
- since = "0.52.0",
- note = "Please use `delete_max_size` instead of `batch_max_operations`"
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default and this option is no longer needed."
)]
- pub fn batch_max_operations(mut self, batch_max_operations: usize) -> Self {
- self.config.delete_max_size = Some(batch_max_operations);
-
+ pub fn batch_max_operations(self, _batch_max_operations: usize) -> Self {
self
}
- /// Set maximum delete operations of this backend.
- pub fn delete_max_size(mut self, delete_max_size: usize) -> Self {
- self.config.delete_max_size = Some(delete_max_size);
-
+ /// Deprecated: S3 delete batch capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default and this option is no longer needed."
+ )]
+ pub fn delete_max_size(self, _delete_max_size: usize) -> Self {
self
}
@@ -646,6 +647,10 @@ impl S3Builder {
}
// If this bucket is AWS, we can try to match the endpoint.
+ if endpoint == "https://s3.amazonaws.com" {
+ return Some("us-east-1".to_string());
+ }
+
if let Some(region) = endpoint
.strip_prefix("https://s3.")
.and_then(|v| v.strip_suffix(".amazonaws.com"))
@@ -895,10 +900,6 @@ impl Builder for S3Builder {
// Create the signer
let signer = Signer::new(ctx, provider, request_signer);
- let delete_max_size = config
- .delete_max_size
- .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS);
-
Ok(S3Backend {
core: Arc::new(S3Core {
info: {
@@ -911,10 +912,9 @@ impl Builder for S3Builder {
stat_with_if_none_match: true,
stat_with_if_modified_since: true,
stat_with_if_unmodified_since: true,
- stat_with_override_cache_control: !config.disable_stat_with_override,
- stat_with_override_content_disposition: !config
- .disable_stat_with_override,
- stat_with_override_content_type: !config.disable_stat_with_override,
+ stat_with_override_cache_control: true,
+ stat_with_override_content_disposition: true,
+ stat_with_override_content_type: true,
stat_with_version: true,
read: true,
@@ -954,7 +954,7 @@ impl Builder for S3Builder {
},
delete: true,
- delete_max_size: Some(delete_max_size),
+ delete_max_size: Some(DEFAULT_BATCH_MAX_OPERATIONS),
delete_with_version: true,
copy: true,
diff --git a/core/services/s3/src/compatible_services.md b/core/services/s3/src/compatible_services.md
index 12a6e270e..27af1d4f4 100644
--- a/core/services/s3/src/compatible_services.md
+++ b/core/services/s3/src/compatible_services.md
@@ -110,9 +110,13 @@ To connect to r2, we need to set:
- `endpoint`: The endpoint of r2, for example: `https://<account_id>.r2.cloudflarestorage.com`
- `bucket`: The bucket name of r2.
- `region`: When you create a new bucket, the data location is set to Automatic by default. So please use `auto` for region.
-- `batch_max_operations`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please set this value `<= 700` to make sure batch delete work as expected.
- `enable_exact_buf_write`: R2 requires the non-tailing parts size to be exactly the same. Please enable this option to avoid the error `All non-trailing parts must have the same length`.
+R2 has the following capability differences from S3:
+
+- `delete_max_size`: R2's delete objects will return `Internal Error` if the batch is larger than `700`. Please override `delete_max_size` to `700`.
+- `stat_with_override_cache_control`, `stat_with_override_content_disposition`, `stat_with_override_content_type`: R2 doesn't support stat with response override queries. Please override them to `false`.
+
### Google Cloud Storage XML API
[Google Cloud Storage XML API](https://cloud.google.com/storage/docs/xml-api/overview) provides s3 compatible API.
- `endpoint`: The endpoint of Google Cloud Storage XML API, for example: `https://storage.googleapis.com`
@@ -123,4 +127,3 @@ To connect to r2, we need to set:
Ceph supports a RESTful API that is compatible with the basic data access model of the Amazon S3 API.
For more information, refer: <https://docs.ceph.com/en/latest/radosgw/s3/>
-
diff --git a/core/services/s3/src/config.rs b/core/services/s3/src/config.rs
index d47233e1e..b04d1345b 100644
--- a/core/services/s3/src/config.rs
+++ b/core/services/s3/src/config.rs
@@ -184,27 +184,23 @@ pub struct S3Config {
alias = "virtual_hosted_style_request"
)]
pub enable_virtual_host_style: bool,
- /// Set maximum batch operations of this backend.
- ///
- /// Some compatible services have a limit on the number of operations in a batch request.
- /// For example, R2 could return `Internal Error` while batch delete 1000 files.
- ///
- /// Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
#[deprecated(
- since = "0.52.0",
- note = "Please use `delete_max_size` instead of `batch_max_operations`"
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
)]
pub batch_max_operations: Option<usize>,
- /// Set the maximum delete size of this backend.
- ///
- /// Some compatible services have a limit on the number of operations in a batch request.
- /// For example, R2 could return `Internal Error` while batch delete 1000 files.
- ///
- /// Please tune this value based on services' document.
+ /// Deprecated: S3 delete batch capability is enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints."
+ )]
pub delete_max_size: Option<usize>,
- /// Disable stat with override so that opendal will not send stat request with override queries.
- ///
- /// For example, R2 doesn't support stat with `response_content_type` query.
+ /// Deprecated: S3 stat override capabilities are enabled by default.
+ #[deprecated(
+ since = "0.57.0",
+ note = "S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints."
+ )]
pub disable_stat_with_override: bool,
/// Checksum Algorithm to use when sending checksums in HTTP headers.
/// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
diff --git a/core/services/s3/src/docs.md b/core/services/s3/src/docs.md
index 8b3fc6024..469dd5006 100644
--- a/core/services/s3/src/docs.md
+++ b/core/services/s3/src/docs.md
@@ -30,6 +30,9 @@ This service can be used to:
- `disable_config_load`: Disable aws config load from env.
- `enable_virtual_host_style`: Enable virtual host style.
- `enable_request_payer`: Enable the request payer for backend.
+- `batch_max_operations`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `delete_max_size`: Deprecated. S3 delete batch capability is enabled by default and this option is no longer needed.
+- `disable_stat_with_override`: Deprecated. S3 stat override capabilities are enabled by default and this option is no longer needed.
- `disable_write_with_if_match`: Deprecated. S3 write with If-Match capability is enabled by default and this option is no longer needed.
- `enable_versioning`: Deprecated. S3 versioning capability is enabled by default and this option is no longer needed.
- `enable_write_with_append`: Deprecated. S3 append capability is enabled by default and this option is no longer needed.