You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/28 05:09:25 UTC
(doris) 19/22: [enhance](S3) Init default retry strategy for aws s3 sdk (#31329)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f18c85349589e911410da895ffba9e7d9252d3be
Author: AlexYue <yj...@gmail.com>
AuthorDate: Wed Feb 28 12:19:54 2024 +0800
[enhance](S3) Init default retry strategy for aws s3 sdk (#31329)
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 3 +++
be/src/util/s3_util.cpp | 4 ++++
3 files changed, 9 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index cd31a8a0136..46c98bb763b 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1175,6 +1175,8 @@ DEFINE_Int32(partition_disk_index_lru_size, "10000");
DEFINE_mBool(check_segment_when_build_rowset_meta, "false");
+DEFINE_mInt32(max_s3_client_retry, "10");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 0230b2d768c..8878cba0e64 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1251,6 +1251,9 @@ DECLARE_Int32(partition_disk_index_lru_size);
DECLARE_mBool(check_segment_when_build_rowset_meta);
+// max s3 client retry times
+DECLARE_mInt32(max_s3_client_retry);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 82a1b4097c6..d09b808868b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -20,6 +20,7 @@
#include <aws/core/auth/AWSAuthSigner.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
+#include <aws/core/client/DefaultRetryStrategy.h>
#include <aws/core/utils/logging/LogLevel.h>
#include <aws/core/utils/logging/LogSystemInterface.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
@@ -30,6 +31,7 @@
#include <atomic>
#include <cstdlib>
#include <functional>
+#include <memory>
#include <ostream>
#include <utility>
@@ -174,6 +176,8 @@ std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf
if (s3_conf.connect_timeout_ms > 0) {
aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms;
}
+ aws_config.retryStrategy =
+ std::make_shared<Aws::Client::DefaultRetryStrategy>(config::max_s3_client_retry);
std::shared_ptr<Aws::S3::S3Client> new_client;
if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org