You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2024/07/12 14:46:40 UTC
(doris) branch master updated: [fix](split) remove retry when fetch split batch failed (#37636)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 83a734e762c [fix](split) remove retry when fetch split batch failed (#37636)
83a734e762c is described below
commit 83a734e762cfa2c6497288d3e5fe96d77c34b455
Author: Ashin Gau <As...@users.noreply.github.com>
AuthorDate: Fri Jul 12 22:46:33 2024 +0800
[fix](split) remove retry when fetch split batch failed (#37636)
## Proposed changes
We need to remove the retry logic for failed to fetch split batch.
Originally, this was implemented to handle cases where the cached client
connection might have been lost and needed to be reestablished. However,
this retry mechanism can lead to data loss. For instance, if a batch of
data has already been sent, retrying can cause this batch to be lost
without the receiver being aware of it.
---
be/src/vec/exec/scan/split_source_connector.cpp | 10 ++--------
.../java/org/apache/doris/datasource/SplitSourceManager.java | 7 ++++++-
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/be/src/vec/exec/scan/split_source_connector.cpp b/be/src/vec/exec/scan/split_source_connector.cpp
index 9bba44b4e76..6533ae2bfe0 100644
--- a/be/src/vec/exec/scan/split_source_connector.cpp
+++ b/be/src/vec/exec/scan/split_source_connector.cpp
@@ -56,14 +56,8 @@ Status RemoteSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* rang
TFetchSplitBatchResult result;
try {
coord->fetchSplitBatch(result, request);
- } catch (std::exception& e1) {
- LOG(WARNING) << "Failed to get batch of split source: {}, try to reopen" << e1.what();
- RETURN_IF_ERROR(coord.reopen());
- try {
- coord->fetchSplitBatch(result, request);
- } catch (std::exception& e2) {
- return Status::IOError("Failed to get batch of split source: {}", e2.what());
- }
+ } catch (std::exception& e) {
+ return Status::IOError<false>("Failed to get batch of split source: {}", e.what());
}
_last_batch = result.splits.empty();
_scan_ranges = result.splits;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
index 83a7436df9a..6d4b06e0e7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java
@@ -61,7 +61,12 @@ public class SplitSourceManager extends MasterDaemon {
}
public SplitSource getSplitSource(long uniqueId) {
- return splits.get(uniqueId).get();
+ WeakReference<SplitSource> ref = splits.get(uniqueId);
+ if (ref == null) {
+ return null;
+ } else {
+ return ref.get();
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org