You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by kf...@apache.org on 2022/03/22 13:58:27 UTC

[druid] branch master updated: Fix OOM failures in dimension distribution phase of parallel indexing (#12331)

This is an automated email from the ASF dual-hosted git repository.

kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new 0867ca7  Fix OOM failures in dimension distribution phase of parallel indexing (#12331)
0867ca7 is described below

commit 0867ca75e12f20934c92a8a3ed02bad3e740684f
Author: Kashif Faraz <ka...@gmail.com>
AuthorDate: Tue Mar 22 19:28:15 2022 +0530

    Fix OOM failures in dimension distribution phase of parallel indexing (#12331)
    
    Parallel indexing with range partitioning can often cause OOM in the
    `ParallelIndexSupervisorTask` during the dimension distribution phase.
    This typically happens because of too many `StringSketch` objects
    obtained from the different `partial_dimension_distribution` sub-tasks.
    
    We need not keep any of the sketches in memory until we need to compute
    the PartitionBoundaries for the respective interval.
    
    Changes
    - Extract `StringDistribution` from `DimensionDistributionReport`s when they are received
      and write to disk inside the task/temp/distributions
    - After all the subtasks have finished, iterate over all the intervals one by one
    - For each interval, read the distributions from disk, merge them and create `PartitionBoundaries`.
    - Cleanup task/temp/distributions directory when all `PartitionBoundaries` have been determined
---
 .../parallel/ParallelIndexSupervisorTask.java      |  67 ++----
 ...mensionDistributionParallelIndexTaskRunner.java | 227 +++++++++++++++++++++
 .../common/task/batch/parallel/TaskMonitor.java    |   1 +
 .../batch/parallel/RangePartitionTaskKillTest.java | 185 ++++-------------
 4 files changed, 289 insertions(+), 191 deletions(-)

diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
index c1911c9..fe5c9dc 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
@@ -25,9 +25,7 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Multimap;
 import org.apache.datasketches.hll.HllSketch;
 import org.apache.datasketches.hll.Union;
 import org.apache.datasketches.memory.Memory;
@@ -57,9 +55,6 @@ import org.apache.druid.indexing.common.task.TaskResource;
 import org.apache.druid.indexing.common.task.Tasks;
 import org.apache.druid.indexing.common.task.batch.MaxAllowedLocksExceededException;
 import org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexTaskRunner.SubTaskSpecStatus;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistributionMerger;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketchMerger;
 import org.apache.druid.indexing.worker.shuffle.IntermediaryDataManager;
 import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Pair;
@@ -786,7 +781,8 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
   TaskStatus runRangePartitionMultiPhaseParallel(TaskToolbox toolbox) throws Exception
   {
     ParallelIndexIngestionSpec ingestionSchemaToUse = ingestionSchema;
-    ParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport> distributionRunner =
+    PartialDimensionDistributionParallelIndexTaskRunner distributionRunner =
+        (PartialDimensionDistributionParallelIndexTaskRunner)
         createRunner(
             toolbox,
             this::createPartialDimensionDistributionRunner
@@ -798,14 +794,26 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
       return TaskStatus.failure(getId(), errMsg);
     }
 
-    Map<Interval, PartitionBoundaries> intervalToPartitions =
-        determineAllRangePartitions(distributionRunner.getReports().values());
-
-    if (intervalToPartitions.isEmpty()) {
-      String msg = "No valid rows for single dimension partitioning."
-                   + " All rows may have invalid timestamps or multiple dimension values.";
-      LOG.warn(msg);
-      return TaskStatus.success(getId(), msg);
+    // Get the partition boundaries for each interval
+    final Map<Interval, PartitionBoundaries> intervalToPartitions;
+    try {
+      intervalToPartitions = distributionRunner.getIntervalToPartitionBoundaries(
+          (DimensionRangePartitionsSpec) ingestionSchema.getTuningConfig().getGivenOrDefaultPartitionsSpec()
+      );
+      if (intervalToPartitions.isEmpty()) {
+        String msg = "No valid rows for range partitioning."
+                     + " All rows may have invalid timestamps or multiple dimension values.";
+        LOG.warn(msg);
+        return TaskStatus.success(getId(), msg);
+      }
+    }
+    catch (Exception e) {
+      String errorMsg = "Error creating partition boundaries.";
+      if (distributionRunner.getStopReason() != null) {
+        errorMsg += " " + distributionRunner.getStopReason();
+      }
+      LOG.error(e, errorMsg);
+      return TaskStatus.failure(getId(), errorMsg);
     }
 
     ingestionSchemaToUse = rewriteIngestionSpecWithIntervalsIfMissing(
@@ -910,37 +918,6 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
     );
   }
 
-  private Map<Interval, PartitionBoundaries> determineAllRangePartitions(Collection<DimensionDistributionReport> reports)
-  {
-    Multimap<Interval, StringDistribution> intervalToDistributions = ArrayListMultimap.create();
-    reports.forEach(report -> {
-      Map<Interval, StringDistribution> intervalToDistribution = report.getIntervalToDistribution();
-      intervalToDistribution.forEach(intervalToDistributions::put);
-    });
-
-    return CollectionUtils.mapValues(intervalToDistributions.asMap(), this::determineRangePartition);
-  }
-
-  private PartitionBoundaries determineRangePartition(Collection<StringDistribution> distributions)
-  {
-    StringDistributionMerger distributionMerger = new StringSketchMerger();
-    distributions.forEach(distributionMerger::merge);
-    StringDistribution mergedDistribution = distributionMerger.getResult();
-
-    DimensionRangePartitionsSpec partitionsSpec =
-        (DimensionRangePartitionsSpec) ingestionSchema.getTuningConfig().getGivenOrDefaultPartitionsSpec();
-
-    final PartitionBoundaries partitions;
-    Integer targetRowsPerSegment = partitionsSpec.getTargetRowsPerSegment();
-    if (targetRowsPerSegment == null) {
-      partitions = mergedDistribution.getEvenPartitionsByMaxSize(partitionsSpec.getMaxRowsPerSegment());
-    } else {
-      partitions = mergedDistribution.getEvenPartitionsByTargetSize(targetRowsPerSegment);
-    }
-
-    return partitions;
-  }
-
   /**
    * Creates a map from partition (interval + bucketId) to the corresponding
    * PartitionLocations. Note that the bucketId maybe different from the final
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
index ad73b7d..a6c332b 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
@@ -20,9 +20,28 @@
 package org.apache.druid.indexing.common.task.batch.parallel;
 
 import org.apache.druid.data.input.InputSplit;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
 import org.apache.druid.indexing.common.TaskToolbox;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistributionMerger;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketchMerger;
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.timeline.partition.PartitionBoundaries;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
 
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Phaser;
 
 /**
  * {@link ParallelIndexTaskRunner} for the phase to determine distribution of dimension values in
@@ -31,8 +50,19 @@ import java.util.Map;
 class PartialDimensionDistributionParallelIndexTaskRunner
     extends InputSourceSplitParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport>
 {
+  private static final Logger log = new Logger(PartialDimensionDistributionParallelIndexTaskRunner.class);
   private static final String PHASE_NAME = "partial dimension distribution";
 
+  private final ExecutorService executor = Execs.singleThreaded("DimDistributionWriter-%s");
+
+  /**
+   * Phaser to await the processing of all reports submitted to the {@link #executor}.
+   */
+  private final Phaser allReportsProcessedPhaser = new Phaser(1);
+
+  private final Map<Interval, Set<String>> intervalToTaskIds = new HashMap<>();
+  private final File tempDistributionsDir;
+
   PartialDimensionDistributionParallelIndexTaskRunner(
       TaskToolbox toolbox,
       String taskId,
@@ -50,6 +80,7 @@ class PartialDimensionDistributionParallelIndexTaskRunner
         ingestionSchema,
         context
     );
+    this.tempDistributionsDir = createDistributionsDir();
   }
 
   @Override
@@ -59,6 +90,202 @@ class PartialDimensionDistributionParallelIndexTaskRunner
   }
 
   @Override
+  public void collectReport(DimensionDistributionReport report)
+  {
+    // Send an empty report to TaskMonitor
+    super.collectReport(new DimensionDistributionReport(report.getTaskId(), null));
+
+    // Extract the distributions in a separate thread to unblock HTTP requests
+    if (executor.isShutdown()) {
+      // this should never happen
+      throw new ISE("Executor is already shutdown. Cannot process more reports.");
+    }
+
+    allReportsProcessedPhaser.register();
+    executor.submit(() -> extractDistributionsFromReport(report));
+  }
+
+  /**
+   * Map from an interval to PartitionBoundaries calculated by applying the target
+   * row size on the final StringDistribution. The final distribution for an
+   * interval is obtained by merging the distributions reported by all the
+   * sub-tasks for that interval.
+   */
+  public Map<Interval, PartitionBoundaries> getIntervalToPartitionBoundaries(
+      DimensionRangePartitionsSpec partitionsSpec
+  )
+  {
+    waitToProcessPendingReports();
+
+    // Do not proceed if a shutdown has been requested
+    if (getStopReason() != null) {
+      throw new ISE("DimensionDistributionPhaseRunner has been stopped. %s", getStopReason());
+    }
+
+    // Merge distributions only from succeeded sub-tasks
+    final Set<String> succeededTaskIds = super.getReports().keySet();
+    final Map<Interval, PartitionBoundaries> intervalToPartitions = new HashMap<>();
+    intervalToTaskIds.forEach(
+        (interval, subTaskIds) -> {
+          final File intervalDir = getIntervalDistributionDir(interval);
+          final StringDistributionMerger merger = new StringSketchMerger();
+          subTaskIds
+              .stream()
+              .filter(succeededTaskIds::contains)
+              .map(subTaskId -> readDistributionFromFile(intervalDir, subTaskId))
+              .forEach(merger::merge);
+          final StringDistribution mergedDistribution = merger.getResult();
+
+          final PartitionBoundaries partitions;
+          Integer targetRowsPerSegment = partitionsSpec.getTargetRowsPerSegment();
+          if (targetRowsPerSegment == null) {
+            partitions = mergedDistribution.getEvenPartitionsByMaxSize(partitionsSpec.getMaxRowsPerSegment());
+          } else {
+            partitions = mergedDistribution.getEvenPartitionsByTargetSize(targetRowsPerSegment);
+          }
+
+          intervalToPartitions.put(interval, partitions);
+        }
+    );
+
+    cleanupDistributionsDir();
+    return intervalToPartitions;
+  }
+
+  /**
+   * Extracts the distributions from the given report and writes them to the task
+   * temp directory.
+   * <p>
+   * This method is not thread-safe as the reports are processed by a single-threaded executor.
+   */
+  private void extractDistributionsFromReport(DimensionDistributionReport report)
+  {
+    try {
+      log.debug("Started writing distributions from task [%s]", report.getTaskId());
+
+      final Map<Interval, StringDistribution> distributions = report.getIntervalToDistribution();
+      if (distributions == null || distributions.isEmpty()) {
+        log.debug("No dimension distribution received from task [%s]", report.getTaskId());
+        return;
+      }
+      distributions.forEach(
+          (interval, distribution) -> {
+            Set<String> taskIds = intervalToTaskIds.computeIfAbsent(interval, i -> new HashSet<>());
+            final String subTaskId = report.getTaskId();
+            if (!taskIds.contains(subTaskId)) {
+              writeDistributionToFile(interval, subTaskId, distribution);
+              taskIds.add(subTaskId);
+            }
+          }
+      );
+
+      log.debug("Finished writing distributions from task [%s]", report.getTaskId());
+    }
+    finally {
+      allReportsProcessedPhaser.arriveAndDeregister();
+    }
+  }
+
+  /**
+   * Writes the given distribution to the task temp directory.
+   * <p>
+   * If this operation fails, it requests a graceful shutdown of the runner via
+   * {@link #stopGracefully(String)}.
+   */
+  private void writeDistributionToFile(
+      Interval interval,
+      String subTaskId,
+      StringDistribution distribution
+  )
+  {
+    try {
+      File intervalDir = getIntervalDistributionDir(interval);
+      FileUtils.mkdirp(intervalDir);
+
+      File distributionJsonFile = getDistributionJsonFile(intervalDir, subTaskId);
+      getToolbox().getJsonMapper().writeValue(distributionJsonFile, distribution);
+    }
+    catch (IOException e) {
+      String errorMsg = StringUtils.format(
+          "Exception while writing distribution file for interval [%s], task [%s]",
+          interval,
+          subTaskId
+      );
+      stopGracefully(errorMsg);
+      throw new ISE(e, errorMsg);
+    }
+  }
+
+  private StringDistribution readDistributionFromFile(File intervalDir, String subTaskId)
+  {
+    try {
+      File distributionJsonFile = getDistributionJsonFile(intervalDir, subTaskId);
+      return getToolbox().getJsonMapper().readValue(distributionJsonFile, StringDistribution.class);
+    }
+    catch (IOException e) {
+      throw new ISE(e, "Error while reading distribution for interval [%s], task [%s]",
+                    intervalDir.getName(), subTaskId
+      );
+    }
+  }
+
+  private File getIntervalDistributionDir(Interval interval)
+  {
+    return new File(tempDistributionsDir, toIntervalString(interval));
+  }
+
+  private File getDistributionJsonFile(File intervalDir, String subTaskId)
+  {
+    return new File(intervalDir, subTaskId);
+  }
+
+  /**
+   * Waits for distributions from pending reports (if any) to be extracted.
+   */
+  private void waitToProcessPendingReports()
+  {
+    log.info("Waiting to extract distributions from sub-task reports.");
+    try {
+      allReportsProcessedPhaser.arriveAndAwaitAdvance();
+      executor.shutdownNow();
+    }
+    catch (Exception e) {
+      throw new ISE(e, "Exception while waiting to extract distributions.");
+    }
+  }
+
+  private File createDistributionsDir()
+  {
+    File taskTempDir = getToolbox().getConfig().getTaskTempDir(getTaskId());
+    File distributionsDir = new File(taskTempDir, "dimension_distributions");
+    try {
+      FileUtils.mkdirp(distributionsDir);
+      return distributionsDir;
+    }
+    catch (IOException e) {
+      throw new ISE(e, "Could not create temp distribution directory.");
+    }
+  }
+
+  private void cleanupDistributionsDir()
+  {
+    try {
+      FileUtils.deleteDirectory(tempDistributionsDir);
+    }
+    catch (IOException e) {
+      log.warn(e, "Could not delete temp distribution directory.");
+    }
+  }
+
+  private String toIntervalString(Interval interval)
+  {
+    return
+        new DateTime(interval.getStartMillis(), interval.getChronology())
+        + "_"
+        + new DateTime(interval.getEndMillis(), interval.getChronology());
+  }
+
+  @Override
   SubTaskSpec<PartialDimensionDistributionTask> createSubTaskSpec(
       String id,
       String groupId,
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
index fe55db8..5f00c28 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
@@ -264,6 +264,7 @@ public class TaskMonitor<T extends Task, SubTaskReportType extends SubTaskReport
     // Here, we simply make sure the current report is exactly the same as the previous one.
     reportsMap.compute(report.getTaskId(), (taskId, prevReport) -> {
       if (prevReport != null) {
+        log.warn("Received duplicate report for task [%s]", taskId);
         Preconditions.checkState(
             prevReport.equals(report),
             "task[%s] sent two or more reports and previous report[%s] is different from the current one[%s]",
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
index 2e6fdb1..ce112aa 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.indexing.common.task.batch.parallel;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
 import org.apache.druid.data.input.InputFormat;
 import org.apache.druid.data.input.impl.CsvInputFormat;
 import org.apache.druid.data.input.impl.DimensionsSpec;
@@ -35,30 +36,28 @@ import org.apache.druid.indexing.common.TaskToolbox;
 import org.apache.druid.indexing.common.actions.TaskActionClient;
 import org.apache.druid.indexing.common.task.Task;
 import org.apache.druid.indexing.common.task.TaskResource;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketch;
+import org.apache.druid.java.util.common.ISE;
 import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.query.aggregation.AggregatorFactory;
 import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
 import org.apache.druid.segment.indexing.DataSchema;
 import org.apache.druid.segment.indexing.granularity.GranularitySpec;
 import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
+import org.apache.druid.timeline.partition.PartitionBoundaries;
 import org.joda.time.Interval;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
 
 import javax.annotation.Nullable;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.function.Function;
 
 /**
@@ -225,14 +224,14 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
 
   static class ParallelIndexSupervisorTaskTest extends ParallelIndexSupervisorTask
   {
-    // These variables control how many runners get created until it fails:
-    private final int succeedsBeforeFailing;
-    private int numRuns;
+    // These variables control how many phases pass until the task fails:
+    private final int failInPhase;
+    private int currentPhase;
 
     // These maps are a hacky way to provide some sort of mock object in the runner to make the run continue
     // until it fails (whatever they contain is nonsense other that it allows the code to make progress):
-    private final Map<String, DimensionDistributionReport> firstMap;
-    private final Map<String, DimensionDistributionReport> secondMap;
+    private static final Map<Interval, PartitionBoundaries> INTERVAL_TO_PARTITIONS
+        = ImmutableMap.of(Intervals.of("2011-04-01/2011-04-02"), new PartitionBoundaries());
 
     public ParallelIndexSupervisorTaskTest(
         String id,
@@ -242,19 +241,10 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
         @Nullable String baseSubtaskSpecName,
         Map<String, Object> context,
         int succedsBeforeFailing
-
     )
     {
       super(id, groupId, taskResource, ingestionSchema, baseSubtaskSpecName, context);
-      this.succeedsBeforeFailing = succedsBeforeFailing;
-
-      this.firstMap = new HashMap<>();
-      Map<Interval, StringDistribution> intervalToDistribution = new HashMap<>();
-      intervalToDistribution.put(Intervals.of("2011-04-01/2011-04-02"), new StringSketch());
-      this.firstMap.put("A", new DimensionDistributionReport("id", intervalToDistribution));
-
-      this.secondMap = Collections.emptyMap();
-
+      this.failInPhase = succedsBeforeFailing;
     }
 
     @Override
@@ -263,141 +253,44 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
         Function<TaskToolbox, ParallelIndexTaskRunner<T, R>> runnerCreator
     )
     {
-
       // Below are the conditions to determine phase:
-      ParallelIndexTaskRunner<T, R> retVal;
-      if (succeedsBeforeFailing == 0) {
-        retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, firstMap, "PHASE-1");
-      } else if (succeedsBeforeFailing == 1
-                 && numRuns == 1) {
-        retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, secondMap, "PHASE-2");
-      } else if (succeedsBeforeFailing == 2
-                 && numRuns == 2) {
-        retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, secondMap, "PHASE-3");
+      final PartialDimensionDistributionParallelIndexTaskRunner retVal;
+      if (failInPhase == 0) {
+        retVal = createMockDistributionRunner(false, "PHASE-1");
+      } else if (failInPhase == 1 && currentPhase == 1) {
+        retVal = createMockDistributionRunner(false, "PHASE-2");
+      } else if (failInPhase == 2 && currentPhase == 2) {
+        retVal = createMockDistributionRunner(false, "PHASE-3");
       } else {
-        numRuns++;
-        Map<String, DimensionDistributionReport> map;
-        if (numRuns < 2) {
-          map = firstMap;
-        } else {
-          map = secondMap;
-        }
-        retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(true, map, "SUCCESFUL-PHASE");
+        currentPhase++;
+        retVal = createMockDistributionRunner(true, "SUCCESFUL-PHASE");
       }
-      return retVal;
-    }
-  }
-
-  static class TestRunner
-      implements ParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport>
-  {
 
-    private final boolean succeeds;
-    private final String phase;
-
-    private final Map<String, DimensionDistributionReport> distributionMap;
-
-    TestRunner(boolean succeeds, Map<String, DimensionDistributionReport> distributionMap, String phase)
-    {
-      this.succeeds = succeeds;
-      this.distributionMap = distributionMap;
-      this.phase = phase;
+      return (ParallelIndexTaskRunner<T, R>) retVal;
     }
 
-    @Override
-    public String getName()
+    private PartialDimensionDistributionParallelIndexTaskRunner createMockDistributionRunner(
+        boolean succeeds,
+        String phase
+    )
     {
-      if (succeeds) {
-        return StringUtils.format(phase);
-      } else {
-        return StringUtils.format(phase);
+      try {
+        final PartialDimensionDistributionParallelIndexTaskRunner runner = Mockito.mock(
+            PartialDimensionDistributionParallelIndexTaskRunner.class
+        );
+        Mockito.when(runner.getName()).thenReturn(phase);
+        Mockito.when(runner.run()).thenReturn(succeeds ? TaskState.SUCCESS : TaskState.FAILED);
+        Mockito.when(runner.getStopReason()).thenReturn(null);
+        Mockito.when(runner.getProgress()).thenReturn(null);
+        Mockito.when(runner.getIntervalToPartitionBoundaries(ArgumentMatchers.any()))
+               .thenReturn(INTERVAL_TO_PARTITIONS);
+
+        return runner;
       }
-    }
-
-    @Override
-    public TaskState run() 
-    {
-      if (succeeds) {
-        return TaskState.SUCCESS;
+      catch (Exception e) {
+        throw new ISE(e, "Error while mocking distribution phase runner");
       }
-      return TaskState.FAILED;
-    }
-
-    @Override
-    public void stopGracefully(String stopReason)
-    {
-
-    }
-
-    @Override
-    public String getStopReason()
-    {
-      return null;
     }
-
-    @Override
-    public void collectReport(DimensionDistributionReport report)
-    {
-
-    }
-
-    @Override
-    public Map<String, DimensionDistributionReport> getReports()
-    {
-      return distributionMap;
-    }
-
-    @Override
-    public ParallelIndexingPhaseProgress getProgress()
-    {
-      return null;
-    }
-
-    @Override
-    public Set<String> getRunningTaskIds()
-    {
-      return null;
-    }
-
-    @Override
-    public List<SubTaskSpec<PartialDimensionDistributionTask>> getSubTaskSpecs()
-    {
-      return null;
-    }
-
-    @Override
-    public List<SubTaskSpec<PartialDimensionDistributionTask>> getRunningSubTaskSpecs()
-    {
-      return null;
-    }
-
-    @Override
-    public List<SubTaskSpec<PartialDimensionDistributionTask>> getCompleteSubTaskSpecs()
-    {
-      return null;
-    }
-
-    @Nullable
-    @Override
-    public SubTaskSpec<PartialDimensionDistributionTask> getSubTaskSpec(String subTaskSpecId)
-    {
-      return null;
-    }
-
-    @Nullable
-    @Override
-    public SubTaskSpecStatus getSubTaskState(String subTaskSpecId)
-    {
-      return null;
-    }
-
-    @Nullable
-    @Override
-    public TaskHistory<PartialDimensionDistributionTask> getCompleteSubTaskSpecAttemptHistory(String subTaskSpecId)
-    {
-      return null;
-    }
-
   }
 
   protected ParallelIndexSupervisorTask newTask(

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org