You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2024/10/14 06:28:46 UTC
(opendal) branch allow-retry-on-incomplete-xml created (now c7686c35b0)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a change to branch allow-retry-on-incomplete-xml
in repository https://gitbox.apache.org/repos/asf/opendal.git
at c7686c35b0 fix(services/s3): Mark xml deserialize error as temporary during list
This branch includes the following new commits:
new c7686c35b0 fix(services/s3): Mark xml deserialize error as temporary during list
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(opendal) 01/01: fix(services/s3): Mark xml deserialize error as temporary during list
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch allow-retry-on-incomplete-xml
in repository https://gitbox.apache.org/repos/asf/opendal.git
commit c7686c35b03d9a41d289ce9ee39fb62d4b042ae3
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Mon Oct 14 14:28:03 2024 +0800
fix(services/s3): Mark xml deserialize error as temporary during list
Signed-off-by: Xuanwo <gi...@xuanwo.io>
---
core/src/services/s3/lister.rs | 21 +++++++++++++++++----
1 file changed, 17 insertions(+), 4 deletions(-)
diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs
index 621d82c224..9063e61044 100644
--- a/core/src/services/s3/lister.rs
+++ b/core/src/services/s3/lister.rs
@@ -23,6 +23,7 @@ use super::error::parse_error;
use crate::raw::oio::PageContext;
use crate::raw::*;
use crate::EntryMode;
+use crate::Error;
use crate::Metadata;
use crate::Result;
use bytes::Buf;
@@ -84,8 +85,14 @@ impl oio::PageList for S3Lister {
}
let bs = resp.into_body();
- let output: ListObjectsOutput =
- de::from_reader(bs.reader()).map_err(new_xml_deserialize_error)?;
+ let output: ListObjectsOutput = de::from_reader(bs.reader())
+ .map_err(new_xml_deserialize_error)
+ // Allow S3 list to retry on XML deserialization errors.
+ //
+ // This is because the S3 list API may return incomplete XML data under high load.
+ // We are confident that our XML decoding logic is correct. When this error occurs,
+ // we allow retries to obtain the correct data.
+ .map_err(Error::set_temporary)?;
// Try our best to check whether this list is done.
//
@@ -196,8 +203,14 @@ impl oio::PageList for S3ObjectVersionsLister {
}
let body = resp.into_body();
- let output: ListObjectVersionsOutput =
- de::from_reader(body.reader()).map_err(new_xml_deserialize_error)?;
+ let output: ListObjectVersionsOutput = de::from_reader(body.reader())
+ .map_err(new_xml_deserialize_error)
+ // Allow S3 list to retry on XML deserialization errors.
+ //
+ // This is because the S3 list API may return incomplete XML data under high load.
+ // We are confident that our XML decoding logic is correct. When this error occurs,
+ // we allow retries to obtain the correct data.
+ .map_err(Error::set_temporary)?;
ctx.done = if let Some(is_truncated) = output.is_truncated {
!is_truncated