You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/06 10:56:17 UTC

(doris) branch branch-4.1 updated: branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new 757a401aad4 branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)
757a401aad4 is described below

commit 757a401aad429a43562b047d1a1d714e1209966c
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed May 6 18:56:09 2026 +0800

    branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)
    
    Cherry-picked from #62870
    
    Co-authored-by: wudi <wu...@selectdb.com>
---
 .../main/java/org/apache/doris/common/Config.java  |  6 +++
 .../insert/streaming/StreamingMultiTblTask.java    | 34 ++++++++-----
 .../job/offset/jdbc/JdbcSourceOffsetProvider.java  | 57 ++++++++++++++++------
 .../offset/jdbc/JdbcTvfSourceOffsetProvider.java   | 14 ++++--
 .../org/apache/doris/rpc/BackendServiceClient.java |  5 ++
 .../org/apache/doris/rpc/BackendServiceProxy.java  | 11 +++++
 6 files changed, 98 insertions(+), 29 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 1a650003415..a86bbe5682e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1296,6 +1296,12 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true, masterOnly = true)
     public static int streaming_task_timeout_multiplier = 10;
 
+    @ConfField(mutable = true, masterOnly = true)
+    public static int streaming_cdc_light_rpc_timeout_sec = 90;
+
+    @ConfField(mutable = true, masterOnly = true)
+    public static int streaming_cdc_heavy_rpc_timeout_sec = 600;
+
     /**
      * the max timeout of get kafka meta.
      */
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
index ac5efe51542..1d3fdf66f04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
@@ -60,6 +60,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 /**
  * In PostgreSQL/MySQL, multi-table writes are performed by tasks that only make calls.
@@ -137,9 +139,9 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.error("Failed to send write records request, {}", result.getStatus().getErrorMsgs(0));
@@ -163,6 +165,11 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
                 throw new JobException("Failed to parse write records response: " + response);
             }
             throw new JobException("Failed to send write records request , error message: " + response);
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/writeRecords taskId={} jobId={} backend={}:{} timeout_sec={}",
+                    taskId, getJobId(), backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_heavy_rpc_timeout_sec);
+            throw new JobException("cdc_client RPC timeout: /api/writeRecords taskId=" + taskId);
         } catch (ExecutionException | InterruptedException ex) {
             log.error("Send write request failed: ", ex);
             throw new JobException(ex);
@@ -334,20 +341,20 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
      * such as a data quality error, and needs to expose it to the user.
      */
     public String getTimeoutReason() {
+        if (runningBackendId <= 0) {
+            log.info("No running backend for task {}", runningBackendId);
+            return "";
+        }
+        Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId);
         try {
-            if (runningBackendId <= 0) {
-                log.info("No running backend for task {}", runningBackendId);
-                return "";
-            }
-            Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId);
             InternalService.PRequestCdcClientRequest request = InternalService.PRequestCdcClientRequest.newBuilder()
                     .setApi("/api/getFailReason/" + getTaskId())
                     .build();
             TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
             InternalService.PRequestCdcClientResult result = null;
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to get task timeout reason, {}", result.getStatus().getErrorMsgs(0));
@@ -366,6 +373,11 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
             } catch (JsonProcessingException e) {
                 log.warn("Failed to get task timeout reason, response: {}", response);
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/getFailReason jobId={} taskId={} backend={}:{} "
+                            + "timeout_sec={}",
+                    getJobId(), getTaskId(), backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_light_rpc_timeout_sec);
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("Send get task fail reason request failed: ", ex);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
index 3209b190538..bada3a1d7a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
@@ -19,6 +19,7 @@ package org.apache.doris.job.offset.jdbc;
 
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
 import org.apache.doris.httpv2.entity.ResponseBody;
 import org.apache.doris.httpv2.rest.RestApiStatusCode;
 import org.apache.doris.job.cdc.DataSourceConfigKeys;
@@ -64,6 +65,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
 @Getter
@@ -231,9 +234,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to get end offset from backend, {}", result.getStatus().getErrorMsgs(0));
@@ -258,6 +261,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
                 log.warn("Failed to parse end offset response: {}", response);
                 throw new JobException(response);
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/fetchEndOffset jobId={} backend={}:{} timeout_sec={}",
+                    getJobId(), backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_light_rpc_timeout_sec);
+            throw new JobException("cdc_client RPC timeout: /api/fetchEndOffset jobId=" + getJobId());
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("Get end offset error: ", ex);
             throw new JobException(ex);
@@ -318,9 +326,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to compare offset , {}", result.getStatus().getErrorMsgs(0));
@@ -340,6 +348,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
                 log.warn("Failed to parse compare offset response: {}", response);
                 throw new JobException("Failed to parse compare offset response: " + response);
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/compareOffset jobId={} backend={}:{} timeout_sec={}",
+                    getJobId(), backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_light_rpc_timeout_sec);
+            throw new JobException("cdc_client RPC timeout: /api/compareOffset jobId=" + getJobId());
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("Compare offset error: ", ex);
             throw new JobException(ex);
@@ -561,9 +574,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to get split from backend, {}", result.getStatus().getErrorMsgs(0));
@@ -584,6 +597,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
                 log.warn("Failed to parse split response: {}", response);
                 throw new JobException("Failed to parse split response: " + response);
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/fetchSplits jobId={} backend={}:{} table={} timeout_sec={}",
+                    getJobId(), backend.getHost(), backend.getBrpcPort(), table,
+                    Config.streaming_cdc_heavy_rpc_timeout_sec);
+            throw new JobException("cdc_client RPC timeout: /api/fetchSplits jobId=" + getJobId() + " table=" + table);
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("Get splits error: ", ex);
             throw new JobException(ex);
@@ -675,9 +693,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to init job {} reader, {}", getJobId(), result.getStatus().getErrorMsgs(0));
@@ -705,6 +723,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
                 log.warn("Failed to init {} source reader, {}", getJobId(), response);
                 throw new JobException("Failed to init source reader, cause " + e.getMessage());
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/initReader jobId={} backend={}:{} timeout_sec={}",
+                    getJobId(), backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_heavy_rpc_timeout_sec);
+            throw new JobException("cdc_client RPC timeout: /api/initReader jobId=" + getJobId());
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("init source reader: ", ex);
             throw new JobException(ex);
@@ -723,13 +746,17 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
         TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
         InternalService.PRequestCdcClientResult result = null;
         try {
-            Future<PRequestCdcClientResult> future =
-                    BackendServiceProxy.getInstance().requestCdcClient(address, request);
-            result = future.get();
+            Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                    .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+            result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
             TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
             if (code != TStatusCode.OK) {
                 log.warn("Failed to close job {} source {}", jobId, result.getStatus().getErrorMsgs(0));
             }
+        } catch (TimeoutException te) {
+            log.warn("cdc_client RPC timeout api=/api/close jobId={} backend={}:{} timeout_sec={}",
+                    jobId, backend.getHost(), backend.getBrpcPort(),
+                    Config.streaming_cdc_light_rpc_timeout_sec);
         } catch (ExecutionException | InterruptedException ex) {
             log.warn("Close job error: ", ex);
         }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
index 8e2e5eff9df..749b592c775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
@@ -58,6 +58,8 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.stream.Collectors;
 
 /**
@@ -203,9 +205,10 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
             String rawResponse = null;
             try {
                 TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
-                Future<PRequestCdcClientResult> future =
-                        BackendServiceProxy.getInstance().requestCdcClient(address, request);
-                InternalService.PRequestCdcClientResult result = future.get();
+                Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+                        .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+                InternalService.PRequestCdcClientResult result =
+                        future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
                 TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
                 if (code != TStatusCode.OK) {
                     log.warn("Failed to get task {} offset from BE {}: {}", taskId,
@@ -221,6 +224,11 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
                     log.info("Fetched task {} offset from BE {}: {}", taskId, backend.getHost(), data);
                     return data;
                 }
+            } catch (TimeoutException te) {
+                log.warn("cdc_client RPC timeout api=/api/getTaskOffset jobId={} taskId={} backend={}:{} "
+                                + "timeout_sec={}",
+                        jobId, taskId, backend.getHost(), backend.getBrpcPort(),
+                        Config.streaming_cdc_light_rpc_timeout_sec);
             } catch (Exception ex) {
                 log.warn("Get task offset error for task {} from BE {}, raw response: {}",
                         taskId, backend.getHost(), rawResponse, ex);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
index fc3dac0c214..456ec6a4620 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
@@ -217,6 +217,11 @@ public class BackendServiceClient {
         return stub.requestCdcClient(request);
     }
 
+    public Future<InternalService.PRequestCdcClientResult> requestCdcClient(
+            InternalService.PRequestCdcClientRequest request, int timeoutSec) {
+        return stub.withDeadlineAfter(timeoutSec, TimeUnit.SECONDS).requestCdcClient(request);
+    }
+
     public void shutdown() {
         ConnectivityState state = channel.getState(false);
         LOG.warn("shut down backend service client: {}, channel state: {}", address, state);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
index fbdfb3cf223..2cd756ee486 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
@@ -608,4 +608,15 @@ public class BackendServiceProxy {
         }
         return null;
     }
+
+    public Future<InternalService.PRequestCdcClientResult> requestCdcClient(TNetworkAddress address,
+            InternalService.PRequestCdcClientRequest request, int timeoutSec) {
+        try {
+            final BackendServiceClient client = getProxy(address);
+            return client.requestCdcClient(request, timeoutSec);
+        } catch (Throwable e) {
+            LOG.warn("request cdc client failed, address={}:{}", address.getHostname(), address.getPort(), e);
+        }
+        return null;
+    }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org