You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ga...@apache.org on 2024/09/28 19:07:47 UTC
(doris) 03/13: [bugfix](hive)Handle exceptions when submitting tasks fails to prevent dead loop (#40708)
This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 0ad3b1b5eca17d604546957ff71510f1dc947ac7
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Fri Sep 13 11:05:11 2024 +0800
[bugfix](hive)Handle exceptions when submitting tasks fails to prevent dead loop (#40708)
## Proposed changes
When the thread pool is full, an exception will be thrown and we need to
handle this exception
---
.../doris/datasource/hive/source/HiveScanNode.java | 44 +++++++++++-----------
1 file changed, 23 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
index be722b31c7b..e09161fcf74 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java
@@ -260,30 +260,32 @@ public class HiveScanNode extends FileQueryScanNode {
}
try {
splittersOnFlight.acquire();
- } catch (InterruptedException e) {
+ CompletableFuture.runAsync(() -> {
+ try {
+ List<Split> allFiles = Lists.newArrayList();
+ getFileSplitByPartitions(
+ cache, Collections.singletonList(partition), allFiles, bindBrokerName);
+ if (allFiles.size() > numSplitsPerPartition.get()) {
+ numSplitsPerPartition.set(allFiles.size());
+ }
+ splitAssignment.addToQueue(allFiles);
+ } catch (IOException e) {
+ batchException.set(new UserException(e.getMessage(), e));
+ } finally {
+ splittersOnFlight.release();
+ if (batchException.get() != null) {
+ splitAssignment.setException(batchException.get());
+ }
+ if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) {
+ splitAssignment.finishSchedule();
+ }
+ }
+ }, scheduleExecutor);
+ } catch (Exception e) {
+ // When submitting a task, an exception will be thrown if the task pool(scheduleExecutor) is full
batchException.set(new UserException(e.getMessage(), e));
break;
}
- CompletableFuture.runAsync(() -> {
- try {
- List<Split> allFiles = Lists.newArrayList();
- getFileSplitByPartitions(cache, Collections.singletonList(partition), allFiles, bindBrokerName);
- if (allFiles.size() > numSplitsPerPartition.get()) {
- numSplitsPerPartition.set(allFiles.size());
- }
- splitAssignment.addToQueue(allFiles);
- } catch (IOException e) {
- batchException.set(new UserException(e.getMessage(), e));
- } finally {
- splittersOnFlight.release();
- if (batchException.get() != null) {
- splitAssignment.setException(batchException.get());
- }
- if (numFinishedPartitions.incrementAndGet() == prunedPartitions.size()) {
- splitAssignment.finishSchedule();
- }
- }
- }, scheduleExecutor);
}
if (batchException.get() != null) {
splitAssignment.setException(batchException.get());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org