You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/28 05:09:25 UTC

(doris) 19/22: [enhance](S3) Init default retry strategy for aws s3 sdk (#31329)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git

commit f18c85349589e911410da895ffba9e7d9252d3be
Author: AlexYue <yj...@gmail.com>
AuthorDate: Wed Feb 28 12:19:54 2024 +0800

    [enhance](S3) Init default retry strategy for aws s3 sdk (#31329)
---
 be/src/common/config.cpp | 2 ++
 be/src/common/config.h   | 3 +++
 be/src/util/s3_util.cpp  | 4 ++++
 3 files changed, 9 insertions(+)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index cd31a8a0136..46c98bb763b 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1175,6 +1175,8 @@ DEFINE_Int32(partition_disk_index_lru_size, "10000");
 
 DEFINE_mBool(check_segment_when_build_rowset_meta, "false");
 
+DEFINE_mInt32(max_s3_client_retry, "10");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 0230b2d768c..8878cba0e64 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1251,6 +1251,9 @@ DECLARE_Int32(partition_disk_index_lru_size);
 
 DECLARE_mBool(check_segment_when_build_rowset_meta);
 
+// max s3 client retry times
+DECLARE_mInt32(max_s3_client_retry);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 82a1b4097c6..d09b808868b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -20,6 +20,7 @@
 #include <aws/core/auth/AWSAuthSigner.h>
 #include <aws/core/auth/AWSCredentials.h>
 #include <aws/core/auth/AWSCredentialsProviderChain.h>
+#include <aws/core/client/DefaultRetryStrategy.h>
 #include <aws/core/utils/logging/LogLevel.h>
 #include <aws/core/utils/logging/LogSystemInterface.h>
 #include <aws/core/utils/memory/stl/AWSStringStream.h>
@@ -30,6 +31,7 @@
 #include <atomic>
 #include <cstdlib>
 #include <functional>
+#include <memory>
 #include <ostream>
 #include <utility>
 
@@ -174,6 +176,8 @@ std::shared_ptr<Aws::S3::S3Client> S3ClientFactory::create(const S3Conf& s3_conf
     if (s3_conf.connect_timeout_ms > 0) {
         aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms;
     }
+    aws_config.retryStrategy =
+            std::make_shared<Aws::Client::DefaultRetryStrategy>(config::max_s3_client_retry);
     std::shared_ptr<Aws::S3::S3Client> new_client;
     if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
         Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org