You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/06 10:56:17 UTC
(doris) branch branch-4.1 updated: branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 757a401aad4 branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)
757a401aad4 is described below
commit 757a401aad429a43562b047d1a1d714e1209966c
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Wed May 6 18:56:09 2026 +0800
branch-4.1: [fix](streaming-job) bound cdc_client RPCs with per-category timeouts #62870 (#62983)
Cherry-picked from #62870
Co-authored-by: wudi <wu...@selectdb.com>
---
.../main/java/org/apache/doris/common/Config.java | 6 +++
.../insert/streaming/StreamingMultiTblTask.java | 34 ++++++++-----
.../job/offset/jdbc/JdbcSourceOffsetProvider.java | 57 ++++++++++++++++------
.../offset/jdbc/JdbcTvfSourceOffsetProvider.java | 14 ++++--
.../org/apache/doris/rpc/BackendServiceClient.java | 5 ++
.../org/apache/doris/rpc/BackendServiceProxy.java | 11 +++++
6 files changed, 98 insertions(+), 29 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 1a650003415..a86bbe5682e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1296,6 +1296,12 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static int streaming_task_timeout_multiplier = 10;
+ @ConfField(mutable = true, masterOnly = true)
+ public static int streaming_cdc_light_rpc_timeout_sec = 90;
+
+ @ConfField(mutable = true, masterOnly = true)
+ public static int streaming_cdc_heavy_rpc_timeout_sec = 600;
+
/**
* the max timeout of get kafka meta.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
index ac5efe51542..1d3fdf66f04 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/insert/streaming/StreamingMultiTblTask.java
@@ -60,6 +60,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
/**
* In PostgreSQL/MySQL, multi-table writes are performed by tasks that only make calls.
@@ -137,9 +139,9 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.error("Failed to send write records request, {}", result.getStatus().getErrorMsgs(0));
@@ -163,6 +165,11 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
throw new JobException("Failed to parse write records response: " + response);
}
throw new JobException("Failed to send write records request , error message: " + response);
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/writeRecords taskId={} jobId={} backend={}:{} timeout_sec={}",
+ taskId, getJobId(), backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_heavy_rpc_timeout_sec);
+ throw new JobException("cdc_client RPC timeout: /api/writeRecords taskId=" + taskId);
} catch (ExecutionException | InterruptedException ex) {
log.error("Send write request failed: ", ex);
throw new JobException(ex);
@@ -334,20 +341,20 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
* such as a data quality error, and needs to expose it to the user.
*/
public String getTimeoutReason() {
+ if (runningBackendId <= 0) {
+ log.info("No running backend for task {}", runningBackendId);
+ return "";
+ }
+ Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId);
try {
- if (runningBackendId <= 0) {
- log.info("No running backend for task {}", runningBackendId);
- return "";
- }
- Backend backend = Env.getCurrentSystemInfo().getBackend(runningBackendId);
InternalService.PRequestCdcClientRequest request = InternalService.PRequestCdcClientRequest.newBuilder()
.setApi("/api/getFailReason/" + getTaskId())
.build();
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to get task timeout reason, {}", result.getStatus().getErrorMsgs(0));
@@ -366,6 +373,11 @@ public class StreamingMultiTblTask extends AbstractStreamingTask {
} catch (JsonProcessingException e) {
log.warn("Failed to get task timeout reason, response: {}", response);
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/getFailReason jobId={} taskId={} backend={}:{} "
+ + "timeout_sec={}",
+ getJobId(), getTaskId(), backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_light_rpc_timeout_sec);
} catch (ExecutionException | InterruptedException ex) {
log.warn("Send get task fail reason request failed: ", ex);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
index 3209b190538..bada3a1d7a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcSourceOffsetProvider.java
@@ -19,6 +19,7 @@ package org.apache.doris.job.offset.jdbc;
import org.apache.doris.catalog.Env;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.httpv2.entity.ResponseBody;
import org.apache.doris.httpv2.rest.RestApiStatusCode;
import org.apache.doris.job.cdc.DataSourceConfigKeys;
@@ -64,6 +65,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
@Getter
@@ -231,9 +234,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to get end offset from backend, {}", result.getStatus().getErrorMsgs(0));
@@ -258,6 +261,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
log.warn("Failed to parse end offset response: {}", response);
throw new JobException(response);
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/fetchEndOffset jobId={} backend={}:{} timeout_sec={}",
+ getJobId(), backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_light_rpc_timeout_sec);
+ throw new JobException("cdc_client RPC timeout: /api/fetchEndOffset jobId=" + getJobId());
} catch (ExecutionException | InterruptedException ex) {
log.warn("Get end offset error: ", ex);
throw new JobException(ex);
@@ -318,9 +326,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to compare offset , {}", result.getStatus().getErrorMsgs(0));
@@ -340,6 +348,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
log.warn("Failed to parse compare offset response: {}", response);
throw new JobException("Failed to parse compare offset response: " + response);
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/compareOffset jobId={} backend={}:{} timeout_sec={}",
+ getJobId(), backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_light_rpc_timeout_sec);
+ throw new JobException("cdc_client RPC timeout: /api/compareOffset jobId=" + getJobId());
} catch (ExecutionException | InterruptedException ex) {
log.warn("Compare offset error: ", ex);
throw new JobException(ex);
@@ -561,9 +574,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to get split from backend, {}", result.getStatus().getErrorMsgs(0));
@@ -584,6 +597,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
log.warn("Failed to parse split response: {}", response);
throw new JobException("Failed to parse split response: " + response);
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/fetchSplits jobId={} backend={}:{} table={} timeout_sec={}",
+ getJobId(), backend.getHost(), backend.getBrpcPort(), table,
+ Config.streaming_cdc_heavy_rpc_timeout_sec);
+ throw new JobException("cdc_client RPC timeout: /api/fetchSplits jobId=" + getJobId() + " table=" + table);
} catch (ExecutionException | InterruptedException ex) {
log.warn("Get splits error: ", ex);
throw new JobException(ex);
@@ -675,9 +693,9 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_heavy_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_heavy_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to init job {} reader, {}", getJobId(), result.getStatus().getErrorMsgs(0));
@@ -705,6 +723,11 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
log.warn("Failed to init {} source reader, {}", getJobId(), response);
throw new JobException("Failed to init source reader, cause " + e.getMessage());
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/initReader jobId={} backend={}:{} timeout_sec={}",
+ getJobId(), backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_heavy_rpc_timeout_sec);
+ throw new JobException("cdc_client RPC timeout: /api/initReader jobId=" + getJobId());
} catch (ExecutionException | InterruptedException ex) {
log.warn("init source reader: ", ex);
throw new JobException(ex);
@@ -723,13 +746,17 @@ public class JdbcSourceOffsetProvider implements SourceOffsetProvider {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
InternalService.PRequestCdcClientResult result = null;
try {
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+ result = future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to close job {} source {}", jobId, result.getStatus().getErrorMsgs(0));
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/close jobId={} backend={}:{} timeout_sec={}",
+ jobId, backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_light_rpc_timeout_sec);
} catch (ExecutionException | InterruptedException ex) {
log.warn("Close job error: ", ex);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
index 8e2e5eff9df..749b592c775 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/job/offset/jdbc/JdbcTvfSourceOffsetProvider.java
@@ -58,6 +58,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
/**
@@ -203,9 +205,10 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
String rawResponse = null;
try {
TNetworkAddress address = new TNetworkAddress(backend.getHost(), backend.getBrpcPort());
- Future<PRequestCdcClientResult> future =
- BackendServiceProxy.getInstance().requestCdcClient(address, request);
- InternalService.PRequestCdcClientResult result = future.get();
+ Future<PRequestCdcClientResult> future = BackendServiceProxy.getInstance()
+ .requestCdcClient(address, request, Config.streaming_cdc_light_rpc_timeout_sec);
+ InternalService.PRequestCdcClientResult result =
+ future.get(Config.streaming_cdc_light_rpc_timeout_sec, TimeUnit.SECONDS);
TStatusCode code = TStatusCode.findByValue(result.getStatus().getStatusCode());
if (code != TStatusCode.OK) {
log.warn("Failed to get task {} offset from BE {}: {}", taskId,
@@ -221,6 +224,11 @@ public class JdbcTvfSourceOffsetProvider extends JdbcSourceOffsetProvider {
log.info("Fetched task {} offset from BE {}: {}", taskId, backend.getHost(), data);
return data;
}
+ } catch (TimeoutException te) {
+ log.warn("cdc_client RPC timeout api=/api/getTaskOffset jobId={} taskId={} backend={}:{} "
+ + "timeout_sec={}",
+ jobId, taskId, backend.getHost(), backend.getBrpcPort(),
+ Config.streaming_cdc_light_rpc_timeout_sec);
} catch (Exception ex) {
log.warn("Get task offset error for task {} from BE {}, raw response: {}",
taskId, backend.getHost(), rawResponse, ex);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
index fc3dac0c214..456ec6a4620 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceClient.java
@@ -217,6 +217,11 @@ public class BackendServiceClient {
return stub.requestCdcClient(request);
}
+ public Future<InternalService.PRequestCdcClientResult> requestCdcClient(
+ InternalService.PRequestCdcClientRequest request, int timeoutSec) {
+ return stub.withDeadlineAfter(timeoutSec, TimeUnit.SECONDS).requestCdcClient(request);
+ }
+
public void shutdown() {
ConnectivityState state = channel.getState(false);
LOG.warn("shut down backend service client: {}, channel state: {}", address, state);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
index fbdfb3cf223..2cd756ee486 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/rpc/BackendServiceProxy.java
@@ -608,4 +608,15 @@ public class BackendServiceProxy {
}
return null;
}
+
+ public Future<InternalService.PRequestCdcClientResult> requestCdcClient(TNetworkAddress address,
+ InternalService.PRequestCdcClientRequest request, int timeoutSec) {
+ try {
+ final BackendServiceClient client = getProxy(address);
+ return client.requestCdcClient(request, timeoutSec);
+ } catch (Throwable e) {
+ LOG.warn("request cdc client failed, address={}:{}", address.getHostname(), address.getPort(), e);
+ }
+ return null;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org