[GitHub] [druid] panhongan commented on a diff in pull request #11296: Fix future control bug for taskClient.pause

[GitBox <gi...@apache.org>]

panhongan commented on code in PR #11296:
URL: https://github.com/apache/druid/pull/11296#discussion_r975276162


##########
indexing-service/src/main/java/org/apache/druid/indexing/seekablestream/supervisor/SeekableStreamSupervisor.java:
##########
@@ -2824,60 +2824,81 @@ private void checkTaskDuration() throws ExecutionException, InterruptedException
       }
     }
 
-    List<Map<PartitionIdType, SequenceOffsetType>> results = Futures.successfulAsList(futures)
-                                                                    .get(futureTimeoutInSeconds, TimeUnit.SECONDS);
-    for (int j = 0; j < results.size(); j++) {
-      Integer groupId = futureGroupIds.get(j);
-      TaskGroup group = activelyReadingTaskGroups.get(groupId);
-      Map<PartitionIdType, SequenceOffsetType> endOffsets = results.get(j);
-
-      if (endOffsets != null) {
-        // set a timeout and put this group in pendingCompletionTaskGroups so that it can be monitored for completion
-        group.completionTimeout = DateTimes.nowUtc().plus(ioConfig.getCompletionTimeout());
-        pendingCompletionTaskGroups.computeIfAbsent(groupId, k -> new CopyOnWriteArrayList<>()).add(group);
+    try {
+      Futures.successfulAsList(futures).get(futureTimeoutInSeconds, TimeUnit.SECONDS);
+    }
+    finally {
+      for (int i = 0; i < futureGroupIds.size(); ++i) {
+        Integer groupId = futureGroupIds.get(i);
+        ListenableFuture<Map<PartitionIdType, SequenceOffsetType>> future = futures.get(i);
 
+        if (future.isDone()) {
+          log.info("GroupId [%d] checking task finished", groupId);
 
-        boolean endOffsetsAreInvalid = false;
-        for (Entry<PartitionIdType, SequenceOffsetType> entry : endOffsets.entrySet()) {
-          if (entry.getValue().equals(getEndOfPartitionMarker())) {
-            log.info(
-                "Got end of partition marker for partition [%s] in checkTaskDuration, not updating partition offset.",
-                entry.getKey()
-            );
-            endOffsetsAreInvalid = true;
+          try {
+            this.moveGroupFromReadingToPending(groupId, future.get());
           }
-        }
-
-        // set endOffsets as the next startOffsets
-        // If we received invalid endOffset values, we clear the known offset to refetch the last committed offset
-        // from metadata. If any endOffset values are invalid, we treat the entire set as invalid as a safety measure.
-        if (!endOffsetsAreInvalid) {
-          for (Entry<PartitionIdType, SequenceOffsetType> entry : endOffsets.entrySet()) {
-            partitionOffsets.put(entry.getKey(), entry.getValue());
+          catch (Exception e1) {
+            log.warn(e1, "Get future result failed for groupId[%d]", groupId);
           }
         } else {
-          for (Entry<PartitionIdType, SequenceOffsetType> entry : endOffsets.entrySet()) {
-            partitionOffsets.put(entry.getKey(), getNotSetMarker());
-          }
+          log.warn("GroupId [%d] checking task not finished", groupId);
         }
-      } else {
-        for (String id : group.taskIds()) {
-          killTask(
-              id,
-              "All tasks in group [%s] failed to transition to publishing state",
-              groupId
+      }
+
+      taskClient.cancelTaskPauseRequests();
+    }
+  }
+
+  protected void moveGroupFromReadingToPending(Integer groupId, Map<PartitionIdType, SequenceOffsetType> endOffsets)

Review Comment:
   done



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org