You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2026/04/20 01:10:20 UTC
(opendal) branch main updated: feat(s3): by default quite batch delete s3 objects (#7415)
This is an automated email from the ASF dual-hosted git repository.
Xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 28789836b feat(s3): by default quite batch delete s3 objects (#7415)
28789836b is described below
commit 28789836b400c4f2e8ca9c951f2648c4874b3d4e
Author: dentiny <de...@gmail.com>
AuthorDate: Sun Apr 19 18:10:14 2026 -0700
feat(s3): by default quite batch delete s3 objects (#7415)
---
core/services/s3/src/core.rs | 47 ++++++-----------------------------------
core/services/s3/src/deleter.rs | 36 +++++++++++++++----------------
2 files changed, 24 insertions(+), 59 deletions(-)
diff --git a/core/services/s3/src/core.rs b/core/services/s3/src/core.rs
index 643c95dc9..9bd98cf92 100644
--- a/core/services/s3/src/core.rs
+++ b/core/services/s3/src/core.rs
@@ -975,17 +975,18 @@ impl S3Core {
pub async fn s3_delete_objects(
&self,
- paths: Vec<(String, OpDelete)>,
+ paths: &[(String, OpDelete)],
) -> Result<Response<Buffer>> {
let url = format!("{}/?delete", self.endpoint);
let mut req = Request::post(&url);
let content = quick_xml::se::to_string(&DeleteObjectsRequest {
+ quiet: true,
object: paths
- .into_iter()
+ .iter()
.map(|(path, op)| DeleteObjectsRequestObject {
- key: build_abs_path(&self.root, &path),
+ key: build_abs_path(&self.root, path),
version_id: op.version().map(|v| v.to_owned()),
})
.collect(),
@@ -1144,6 +1145,7 @@ pub struct CopyObjectResult {
#[derive(Default, Debug, Serialize)]
#[serde(default, rename = "Delete", rename_all = "PascalCase")]
pub struct DeleteObjectsRequest {
+ pub quiet: bool,
pub object: Vec<DeleteObjectsRequestObject>,
}
@@ -1159,17 +1161,9 @@ pub struct DeleteObjectsRequestObject {
#[derive(Default, Debug, Deserialize)]
#[serde(default, rename = "DeleteResult", rename_all = "PascalCase")]
pub struct DeleteObjectsResult {
- pub deleted: Vec<DeleteObjectsResultDeleted>,
pub error: Vec<DeleteObjectsResultError>,
}
-#[derive(Default, Debug, Deserialize)]
-#[serde(rename_all = "PascalCase")]
-pub struct DeleteObjectsResultDeleted {
- pub key: String,
- pub version_id: Option<String>,
-}
-
#[derive(Default, Debug, Deserialize)]
#[serde(default, rename_all = "PascalCase")]
pub struct DeleteObjectsResultError {
@@ -1418,6 +1412,7 @@ mod tests {
#[test]
fn test_serialize_delete_objects_request() {
let req = DeleteObjectsRequest {
+ quiet: true,
object: vec![
DeleteObjectsRequestObject {
key: "sample1.txt".to_string(),
@@ -1435,6 +1430,7 @@ mod tests {
pretty_assertions::assert_eq!(
actual,
r#"<Delete>
+ <Quiet>true</Quiet>
<Object>
<Key>sample1.txt</Key>
</Object>
@@ -1454,9 +1450,6 @@ mod tests {
let bs = Bytes::from(
r#"<?xml version="1.0" encoding="UTF-8"?>
<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
- <Deleted>
- <Key>sample1.txt</Key>
- </Deleted>
<Error>
<Key>sample2.txt</Key>
<Code>AccessDenied</Code>
@@ -1468,38 +1461,12 @@ mod tests {
let out: DeleteObjectsResult =
quick_xml::de::from_reader(bs.reader()).expect("must success");
- assert_eq!(out.deleted.len(), 1);
- assert_eq!(out.deleted[0].key, "sample1.txt");
assert_eq!(out.error.len(), 1);
assert_eq!(out.error[0].key, "sample2.txt");
assert_eq!(out.error[0].code, "AccessDenied");
assert_eq!(out.error[0].message, "Access Denied");
}
- #[test]
- fn test_deserialize_delete_objects_with_version_id() {
- let bs = Bytes::from(
- r#"<?xml version="1.0" encoding="UTF-8"?>
- <DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
- <Deleted>
- <Key>SampleDocument.txt</Key>
- <VersionId>OYcLXagmS.WaD..oyH4KRguB95_YhLs7</VersionId>
- </Deleted>
- </DeleteResult>"#,
- );
-
- let out: DeleteObjectsResult =
- quick_xml::de::from_reader(bs.reader()).expect("must success");
-
- assert_eq!(out.deleted.len(), 1);
- assert_eq!(out.deleted[0].key, "SampleDocument.txt");
- assert_eq!(
- out.deleted[0].version_id,
- Some("OYcLXagmS.WaD..oyH4KRguB95_YhLs7".to_owned())
- );
- assert_eq!(out.error.len(), 0);
- }
-
/// This example is from https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html#API_ListObjects_Examples
#[test]
fn test_parse_list_output_v1() {
diff --git a/core/services/s3/src/deleter.rs b/core/services/s3/src/deleter.rs
index feb9ff90b..48831ffbb 100644
--- a/core/services/s3/src/deleter.rs
+++ b/core/services/s3/src/deleter.rs
@@ -59,7 +59,7 @@ impl oio::BatchDelete for S3Deleter {
}
async fn delete_batch(&self, batch: Vec<(String, OpDelete)>) -> Result<BatchDeleteResult> {
- let resp = self.core.s3_delete_objects(batch).await?;
+ let resp = self.core.s3_delete_objects(&batch).await?;
let status = resp.status();
if status != StatusCode::OK {
@@ -71,27 +71,25 @@ impl oio::BatchDelete for S3Deleter {
let result: DeleteObjectsResult =
quick_xml::de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+ let mut errors = result.error;
let mut batched_result = BatchDeleteResult {
- succeeded: Vec::with_capacity(result.deleted.len()),
- failed: Vec::with_capacity(result.error.len()),
+ succeeded: Vec::with_capacity(batch.len() - errors.len()),
+ failed: Vec::with_capacity(errors.len()),
};
- for i in result.deleted {
- let path = build_rel_path(&self.core.root, &i.key);
- let mut op = OpDelete::new();
- if let Some(version_id) = i.version_id {
- op = op.with_version(version_id.as_str());
+ for (path, op) in batch {
+ let abs_path = build_abs_path(&self.core.root, &path);
+ // Assume errors are rare, so lookup and erase is acceptable.
+ if let Some(idx) = errors
+ .iter()
+ .position(|e| e.key == abs_path && e.version_id.as_deref() == op.version())
+ {
+ let error = errors.swap_remove(idx);
+ batched_result
+ .failed
+ .push((path, op, parse_delete_objects_result_error(error)));
+ } else {
+ batched_result.succeeded.push((path, op));
}
- batched_result.succeeded.push((path, op));
- }
- for i in result.error {
- let path = build_rel_path(&self.core.root, &i.key);
- let mut op = OpDelete::new();
- if let Some(version_id) = &i.version_id {
- op = op.with_version(version_id.as_str());
- }
- batched_result
- .failed
- .push((path, op, parse_delete_objects_result_error(i)));
}
Ok(batched_result)