You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2025/05/17 03:02:02 UTC

(doris) branch branch-3.0 updated: branch-3.0: [enhancement] Optimize the retry policy of backend request meta service #50957 (#50987)

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new d0323c09142 branch-3.0: [enhancement] Optimize the retry policy of backend request meta service #50957 (#50987)
d0323c09142 is described below

commit d0323c09142b9b2152ae1ac73f6c87484f2aeb6e
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat May 17 11:01:56 2025 +0800

    branch-3.0: [enhancement] Optimize the retry policy of backend request meta service #50957 (#50987)
    
    Cherry-picked from #50957
    
    Co-authored-by: Luwei <lu...@selectdb.com>
---
 be/src/cloud/cloud_meta_mgr.cpp | 10 +++++++++-
 be/src/cloud/config.cpp         |  5 ++++-
 be/src/cloud/config.h           |  3 +++
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index f7bd3218fee..7435dfa5263 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -18,6 +18,7 @@
 
 #include <brpc/channel.h>
 #include <brpc/controller.h>
+#include <brpc/errno.pb.h>
 #include <bthread/bthread.h>
 #include <bthread/condition_variable.h>
 #include <bthread/mutex.h>
@@ -385,9 +386,11 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res,
         }
         cntl.set_max_retry(kBrpcRetryTimes);
         res->Clear();
+        int error_code = 0;
         (stub.get()->*method)(&cntl, &req, res, nullptr);
         if (cntl.Failed()) [[unlikely]] {
             error_msg = cntl.ErrorText();
+            error_code = cntl.ErrorCode();
             proxy->set_unhealthy();
         } else if (res->status().code() == MetaServiceCode::OK) {
             return Status::OK();
@@ -401,7 +404,12 @@ Status retry_rpc(std::string_view op_name, const Request& req, Response* res,
             error_msg = res->status().msg();
         }
 
-        if (++retry_times > config::meta_service_rpc_retry_times) {
+        ++retry_times;
+        if (retry_times > config::meta_service_rpc_retry_times ||
+            (retry_times > config::meta_service_rpc_timeout_retry_times &&
+             error_code == brpc::ERPCTIMEDOUT) ||
+            (retry_times > config::meta_service_conflict_error_retry_times &&
+             res->status().code() == MetaServiceCode::KV_TXN_CONFLICT)) {
             break;
         }
 
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index baae4faf494..157780da073 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -32,8 +32,9 @@ DEFINE_Bool(meta_service_connection_pooled, "true");
 DEFINE_mInt64(meta_service_connection_pool_size, "20");
 DEFINE_mInt32(meta_service_connection_age_base_seconds, "30");
 DEFINE_mInt32(meta_service_idle_connection_timeout_ms, "0");
-DEFINE_mInt32(meta_service_rpc_retry_times, "200");
+DEFINE_mInt32(meta_service_rpc_retry_times, "20");
 DEFINE_mInt32(meta_service_brpc_timeout_ms, "10000");
+DEFINE_mInt32(meta_service_rpc_timeout_retry_times, "2");
 
 DEFINE_Int64(tablet_cache_capacity, "100000");
 DEFINE_Int64(tablet_cache_shards, "16");
@@ -82,5 +83,7 @@ DEFINE_mBool(enable_cloud_tablet_report, "true");
 DEFINE_mInt32(delete_bitmap_rpc_retry_times, "25");
 
 DEFINE_mInt64(meta_service_rpc_reconnect_interval_ms, "5000");
+
+DEFINE_mInt32(meta_service_conflict_error_retry_times, "10");
 #include "common/compile_check_end.h"
 } // namespace doris::config
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index 62d442945ea..9d2027b965b 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -63,6 +63,7 @@ DECLARE_mInt32(meta_service_rpc_timeout_ms);
 DECLARE_mInt32(meta_service_rpc_retry_times);
 // default brpc timeout
 DECLARE_mInt32(meta_service_brpc_timeout_ms);
+DECLARE_mInt32(meta_service_rpc_timeout_retry_times);
 
 // CloudTabletMgr config
 DECLARE_Int64(tablet_cache_capacity);
@@ -119,5 +120,7 @@ DECLARE_mInt32(delete_bitmap_rpc_retry_times);
 
 DECLARE_mInt64(meta_service_rpc_reconnect_interval_ms);
 
+DECLARE_mInt32(meta_service_conflict_error_retry_times);
+
 #include "common/compile_check_end.h"
 } // namespace doris::config


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org