You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by kf...@apache.org on 2022/03/22 13:58:27 UTC
[druid] branch master updated: Fix OOM failures in dimension distribution phase of parallel indexing (#12331)
This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 0867ca7 Fix OOM failures in dimension distribution phase of parallel indexing (#12331)
0867ca7 is described below
commit 0867ca75e12f20934c92a8a3ed02bad3e740684f
Author: Kashif Faraz <ka...@gmail.com>
AuthorDate: Tue Mar 22 19:28:15 2022 +0530
Fix OOM failures in dimension distribution phase of parallel indexing (#12331)
Parallel indexing with range partitioning can often cause OOM in the
`ParallelIndexSupervisorTask` during the dimension distribution phase.
This typically happens because of too many `StringSketch` objects
obtained from the different `partial_dimension_distribution` sub-tasks.
We need not keep any of the sketches in memory until we need to compute
the PartitionBoundaries for the respective interval.
Changes
- Extract `StringDistribution` from `DimensionDistributionReport`s when they are received
and write to disk inside the task/temp/distributions
- After all the subtasks have finished, iterate over all the intervals one by one
- For each interval, read the distributions from disk, merge them and create `PartitionBoundaries`.
- Cleanup task/temp/distributions directory when all `PartitionBoundaries` have been determined
---
.../parallel/ParallelIndexSupervisorTask.java | 67 ++----
...mensionDistributionParallelIndexTaskRunner.java | 227 +++++++++++++++++++++
.../common/task/batch/parallel/TaskMonitor.java | 1 +
.../batch/parallel/RangePartitionTaskKillTest.java | 185 ++++-------------
4 files changed, 289 insertions(+), 191 deletions(-)
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
index c1911c9..fe5c9dc 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java
@@ -25,9 +25,7 @@ import com.fasterxml.jackson.jaxrs.smile.SmileMediaTypes;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
-import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Multimap;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.Union;
import org.apache.datasketches.memory.Memory;
@@ -57,9 +55,6 @@ import org.apache.druid.indexing.common.task.TaskResource;
import org.apache.druid.indexing.common.task.Tasks;
import org.apache.druid.indexing.common.task.batch.MaxAllowedLocksExceededException;
import org.apache.druid.indexing.common.task.batch.parallel.ParallelIndexTaskRunner.SubTaskSpecStatus;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistributionMerger;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketchMerger;
import org.apache.druid.indexing.worker.shuffle.IntermediaryDataManager;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Pair;
@@ -786,7 +781,8 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
TaskStatus runRangePartitionMultiPhaseParallel(TaskToolbox toolbox) throws Exception
{
ParallelIndexIngestionSpec ingestionSchemaToUse = ingestionSchema;
- ParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport> distributionRunner =
+ PartialDimensionDistributionParallelIndexTaskRunner distributionRunner =
+ (PartialDimensionDistributionParallelIndexTaskRunner)
createRunner(
toolbox,
this::createPartialDimensionDistributionRunner
@@ -798,14 +794,26 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
return TaskStatus.failure(getId(), errMsg);
}
- Map<Interval, PartitionBoundaries> intervalToPartitions =
- determineAllRangePartitions(distributionRunner.getReports().values());
-
- if (intervalToPartitions.isEmpty()) {
- String msg = "No valid rows for single dimension partitioning."
- + " All rows may have invalid timestamps or multiple dimension values.";
- LOG.warn(msg);
- return TaskStatus.success(getId(), msg);
+ // Get the partition boundaries for each interval
+ final Map<Interval, PartitionBoundaries> intervalToPartitions;
+ try {
+ intervalToPartitions = distributionRunner.getIntervalToPartitionBoundaries(
+ (DimensionRangePartitionsSpec) ingestionSchema.getTuningConfig().getGivenOrDefaultPartitionsSpec()
+ );
+ if (intervalToPartitions.isEmpty()) {
+ String msg = "No valid rows for range partitioning."
+ + " All rows may have invalid timestamps or multiple dimension values.";
+ LOG.warn(msg);
+ return TaskStatus.success(getId(), msg);
+ }
+ }
+ catch (Exception e) {
+ String errorMsg = "Error creating partition boundaries.";
+ if (distributionRunner.getStopReason() != null) {
+ errorMsg += " " + distributionRunner.getStopReason();
+ }
+ LOG.error(e, errorMsg);
+ return TaskStatus.failure(getId(), errorMsg);
}
ingestionSchemaToUse = rewriteIngestionSpecWithIntervalsIfMissing(
@@ -910,37 +918,6 @@ public class ParallelIndexSupervisorTask extends AbstractBatchIndexTask implemen
);
}
- private Map<Interval, PartitionBoundaries> determineAllRangePartitions(Collection<DimensionDistributionReport> reports)
- {
- Multimap<Interval, StringDistribution> intervalToDistributions = ArrayListMultimap.create();
- reports.forEach(report -> {
- Map<Interval, StringDistribution> intervalToDistribution = report.getIntervalToDistribution();
- intervalToDistribution.forEach(intervalToDistributions::put);
- });
-
- return CollectionUtils.mapValues(intervalToDistributions.asMap(), this::determineRangePartition);
- }
-
- private PartitionBoundaries determineRangePartition(Collection<StringDistribution> distributions)
- {
- StringDistributionMerger distributionMerger = new StringSketchMerger();
- distributions.forEach(distributionMerger::merge);
- StringDistribution mergedDistribution = distributionMerger.getResult();
-
- DimensionRangePartitionsSpec partitionsSpec =
- (DimensionRangePartitionsSpec) ingestionSchema.getTuningConfig().getGivenOrDefaultPartitionsSpec();
-
- final PartitionBoundaries partitions;
- Integer targetRowsPerSegment = partitionsSpec.getTargetRowsPerSegment();
- if (targetRowsPerSegment == null) {
- partitions = mergedDistribution.getEvenPartitionsByMaxSize(partitionsSpec.getMaxRowsPerSegment());
- } else {
- partitions = mergedDistribution.getEvenPartitionsByTargetSize(targetRowsPerSegment);
- }
-
- return partitions;
- }
-
/**
* Creates a map from partition (interval + bucketId) to the corresponding
* PartitionLocations. Note that the bucketId maybe different from the final
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
index ad73b7d..a6c332b 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/PartialDimensionDistributionParallelIndexTaskRunner.java
@@ -20,9 +20,28 @@
package org.apache.druid.indexing.common.task.batch.parallel;
import org.apache.druid.data.input.InputSplit;
+import org.apache.druid.indexer.partitions.DimensionRangePartitionsSpec;
import org.apache.druid.indexing.common.TaskToolbox;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistributionMerger;
+import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketchMerger;
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.timeline.partition.PartitionBoundaries;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Phaser;
/**
* {@link ParallelIndexTaskRunner} for the phase to determine distribution of dimension values in
@@ -31,8 +50,19 @@ import java.util.Map;
class PartialDimensionDistributionParallelIndexTaskRunner
extends InputSourceSplitParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport>
{
+ private static final Logger log = new Logger(PartialDimensionDistributionParallelIndexTaskRunner.class);
private static final String PHASE_NAME = "partial dimension distribution";
+ private final ExecutorService executor = Execs.singleThreaded("DimDistributionWriter-%s");
+
+ /**
+ * Phaser to await the processing of all reports submitted to the {@link #executor}.
+ */
+ private final Phaser allReportsProcessedPhaser = new Phaser(1);
+
+ private final Map<Interval, Set<String>> intervalToTaskIds = new HashMap<>();
+ private final File tempDistributionsDir;
+
PartialDimensionDistributionParallelIndexTaskRunner(
TaskToolbox toolbox,
String taskId,
@@ -50,6 +80,7 @@ class PartialDimensionDistributionParallelIndexTaskRunner
ingestionSchema,
context
);
+ this.tempDistributionsDir = createDistributionsDir();
}
@Override
@@ -59,6 +90,202 @@ class PartialDimensionDistributionParallelIndexTaskRunner
}
@Override
+ public void collectReport(DimensionDistributionReport report)
+ {
+ // Send an empty report to TaskMonitor
+ super.collectReport(new DimensionDistributionReport(report.getTaskId(), null));
+
+ // Extract the distributions in a separate thread to unblock HTTP requests
+ if (executor.isShutdown()) {
+ // this should never happen
+ throw new ISE("Executor is already shutdown. Cannot process more reports.");
+ }
+
+ allReportsProcessedPhaser.register();
+ executor.submit(() -> extractDistributionsFromReport(report));
+ }
+
+ /**
+ * Map from an interval to PartitionBoundaries calculated by applying the target
+ * row size on the final StringDistribution. The final distribution for an
+ * interval is obtained by merging the distributions reported by all the
+ * sub-tasks for that interval.
+ */
+ public Map<Interval, PartitionBoundaries> getIntervalToPartitionBoundaries(
+ DimensionRangePartitionsSpec partitionsSpec
+ )
+ {
+ waitToProcessPendingReports();
+
+ // Do not proceed if a shutdown has been requested
+ if (getStopReason() != null) {
+ throw new ISE("DimensionDistributionPhaseRunner has been stopped. %s", getStopReason());
+ }
+
+ // Merge distributions only from succeeded sub-tasks
+ final Set<String> succeededTaskIds = super.getReports().keySet();
+ final Map<Interval, PartitionBoundaries> intervalToPartitions = new HashMap<>();
+ intervalToTaskIds.forEach(
+ (interval, subTaskIds) -> {
+ final File intervalDir = getIntervalDistributionDir(interval);
+ final StringDistributionMerger merger = new StringSketchMerger();
+ subTaskIds
+ .stream()
+ .filter(succeededTaskIds::contains)
+ .map(subTaskId -> readDistributionFromFile(intervalDir, subTaskId))
+ .forEach(merger::merge);
+ final StringDistribution mergedDistribution = merger.getResult();
+
+ final PartitionBoundaries partitions;
+ Integer targetRowsPerSegment = partitionsSpec.getTargetRowsPerSegment();
+ if (targetRowsPerSegment == null) {
+ partitions = mergedDistribution.getEvenPartitionsByMaxSize(partitionsSpec.getMaxRowsPerSegment());
+ } else {
+ partitions = mergedDistribution.getEvenPartitionsByTargetSize(targetRowsPerSegment);
+ }
+
+ intervalToPartitions.put(interval, partitions);
+ }
+ );
+
+ cleanupDistributionsDir();
+ return intervalToPartitions;
+ }
+
+ /**
+ * Extracts the distributions from the given report and writes them to the task
+ * temp directory.
+ * <p>
+ * This method is not thread-safe as the reports are processed by a single-threaded executor.
+ */
+ private void extractDistributionsFromReport(DimensionDistributionReport report)
+ {
+ try {
+ log.debug("Started writing distributions from task [%s]", report.getTaskId());
+
+ final Map<Interval, StringDistribution> distributions = report.getIntervalToDistribution();
+ if (distributions == null || distributions.isEmpty()) {
+ log.debug("No dimension distribution received from task [%s]", report.getTaskId());
+ return;
+ }
+ distributions.forEach(
+ (interval, distribution) -> {
+ Set<String> taskIds = intervalToTaskIds.computeIfAbsent(interval, i -> new HashSet<>());
+ final String subTaskId = report.getTaskId();
+ if (!taskIds.contains(subTaskId)) {
+ writeDistributionToFile(interval, subTaskId, distribution);
+ taskIds.add(subTaskId);
+ }
+ }
+ );
+
+ log.debug("Finished writing distributions from task [%s]", report.getTaskId());
+ }
+ finally {
+ allReportsProcessedPhaser.arriveAndDeregister();
+ }
+ }
+
+ /**
+ * Writes the given distribution to the task temp directory.
+ * <p>
+ * If this operation fails, it requests a graceful shutdown of the runner via
+ * {@link #stopGracefully(String)}.
+ */
+ private void writeDistributionToFile(
+ Interval interval,
+ String subTaskId,
+ StringDistribution distribution
+ )
+ {
+ try {
+ File intervalDir = getIntervalDistributionDir(interval);
+ FileUtils.mkdirp(intervalDir);
+
+ File distributionJsonFile = getDistributionJsonFile(intervalDir, subTaskId);
+ getToolbox().getJsonMapper().writeValue(distributionJsonFile, distribution);
+ }
+ catch (IOException e) {
+ String errorMsg = StringUtils.format(
+ "Exception while writing distribution file for interval [%s], task [%s]",
+ interval,
+ subTaskId
+ );
+ stopGracefully(errorMsg);
+ throw new ISE(e, errorMsg);
+ }
+ }
+
+ private StringDistribution readDistributionFromFile(File intervalDir, String subTaskId)
+ {
+ try {
+ File distributionJsonFile = getDistributionJsonFile(intervalDir, subTaskId);
+ return getToolbox().getJsonMapper().readValue(distributionJsonFile, StringDistribution.class);
+ }
+ catch (IOException e) {
+ throw new ISE(e, "Error while reading distribution for interval [%s], task [%s]",
+ intervalDir.getName(), subTaskId
+ );
+ }
+ }
+
+ private File getIntervalDistributionDir(Interval interval)
+ {
+ return new File(tempDistributionsDir, toIntervalString(interval));
+ }
+
+ private File getDistributionJsonFile(File intervalDir, String subTaskId)
+ {
+ return new File(intervalDir, subTaskId);
+ }
+
+ /**
+ * Waits for distributions from pending reports (if any) to be extracted.
+ */
+ private void waitToProcessPendingReports()
+ {
+ log.info("Waiting to extract distributions from sub-task reports.");
+ try {
+ allReportsProcessedPhaser.arriveAndAwaitAdvance();
+ executor.shutdownNow();
+ }
+ catch (Exception e) {
+ throw new ISE(e, "Exception while waiting to extract distributions.");
+ }
+ }
+
+ private File createDistributionsDir()
+ {
+ File taskTempDir = getToolbox().getConfig().getTaskTempDir(getTaskId());
+ File distributionsDir = new File(taskTempDir, "dimension_distributions");
+ try {
+ FileUtils.mkdirp(distributionsDir);
+ return distributionsDir;
+ }
+ catch (IOException e) {
+ throw new ISE(e, "Could not create temp distribution directory.");
+ }
+ }
+
+ private void cleanupDistributionsDir()
+ {
+ try {
+ FileUtils.deleteDirectory(tempDistributionsDir);
+ }
+ catch (IOException e) {
+ log.warn(e, "Could not delete temp distribution directory.");
+ }
+ }
+
+ private String toIntervalString(Interval interval)
+ {
+ return
+ new DateTime(interval.getStartMillis(), interval.getChronology())
+ + "_"
+ + new DateTime(interval.getEndMillis(), interval.getChronology());
+ }
+
+ @Override
SubTaskSpec<PartialDimensionDistributionTask> createSubTaskSpec(
String id,
String groupId,
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
index fe55db8..5f00c28 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/TaskMonitor.java
@@ -264,6 +264,7 @@ public class TaskMonitor<T extends Task, SubTaskReportType extends SubTaskReport
// Here, we simply make sure the current report is exactly the same as the previous one.
reportsMap.compute(report.getTaskId(), (taskId, prevReport) -> {
if (prevReport != null) {
+ log.warn("Received duplicate report for task [%s]", taskId);
Preconditions.checkState(
prevReport.equals(report),
"task[%s] sent two or more reports and previous report[%s] is different from the current one[%s]",
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
index 2e6fdb1..ce112aa 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/RangePartitionTaskKillTest.java
@@ -20,6 +20,7 @@
package org.apache.druid.indexing.common.task.batch.parallel;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
import org.apache.druid.data.input.InputFormat;
import org.apache.druid.data.input.impl.CsvInputFormat;
import org.apache.druid.data.input.impl.DimensionsSpec;
@@ -35,30 +36,28 @@ import org.apache.druid.indexing.common.TaskToolbox;
import org.apache.druid.indexing.common.actions.TaskActionClient;
import org.apache.druid.indexing.common.task.Task;
import org.apache.druid.indexing.common.task.TaskResource;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringDistribution;
-import org.apache.druid.indexing.common.task.batch.parallel.distribution.StringSketch;
+import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
import org.apache.druid.segment.indexing.DataSchema;
import org.apache.druid.segment.indexing.granularity.GranularitySpec;
import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
+import org.apache.druid.timeline.partition.PartitionBoundaries;
import org.joda.time.Interval;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+import org.mockito.Mockito;
import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.function.Function;
/**
@@ -225,14 +224,14 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
static class ParallelIndexSupervisorTaskTest extends ParallelIndexSupervisorTask
{
- // These variables control how many runners get created until it fails:
- private final int succeedsBeforeFailing;
- private int numRuns;
+ // These variables control how many phases pass until the task fails:
+ private final int failInPhase;
+ private int currentPhase;
// These maps are a hacky way to provide some sort of mock object in the runner to make the run continue
// until it fails (whatever they contain is nonsense other that it allows the code to make progress):
- private final Map<String, DimensionDistributionReport> firstMap;
- private final Map<String, DimensionDistributionReport> secondMap;
+ private static final Map<Interval, PartitionBoundaries> INTERVAL_TO_PARTITIONS
+ = ImmutableMap.of(Intervals.of("2011-04-01/2011-04-02"), new PartitionBoundaries());
public ParallelIndexSupervisorTaskTest(
String id,
@@ -242,19 +241,10 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
@Nullable String baseSubtaskSpecName,
Map<String, Object> context,
int succedsBeforeFailing
-
)
{
super(id, groupId, taskResource, ingestionSchema, baseSubtaskSpecName, context);
- this.succeedsBeforeFailing = succedsBeforeFailing;
-
- this.firstMap = new HashMap<>();
- Map<Interval, StringDistribution> intervalToDistribution = new HashMap<>();
- intervalToDistribution.put(Intervals.of("2011-04-01/2011-04-02"), new StringSketch());
- this.firstMap.put("A", new DimensionDistributionReport("id", intervalToDistribution));
-
- this.secondMap = Collections.emptyMap();
-
+ this.failInPhase = succedsBeforeFailing;
}
@Override
@@ -263,141 +253,44 @@ public class RangePartitionTaskKillTest extends AbstractMultiPhaseParallelIndexi
Function<TaskToolbox, ParallelIndexTaskRunner<T, R>> runnerCreator
)
{
-
// Below are the conditions to determine phase:
- ParallelIndexTaskRunner<T, R> retVal;
- if (succeedsBeforeFailing == 0) {
- retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, firstMap, "PHASE-1");
- } else if (succeedsBeforeFailing == 1
- && numRuns == 1) {
- retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, secondMap, "PHASE-2");
- } else if (succeedsBeforeFailing == 2
- && numRuns == 2) {
- retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(false, secondMap, "PHASE-3");
+ final PartialDimensionDistributionParallelIndexTaskRunner retVal;
+ if (failInPhase == 0) {
+ retVal = createMockDistributionRunner(false, "PHASE-1");
+ } else if (failInPhase == 1 && currentPhase == 1) {
+ retVal = createMockDistributionRunner(false, "PHASE-2");
+ } else if (failInPhase == 2 && currentPhase == 2) {
+ retVal = createMockDistributionRunner(false, "PHASE-3");
} else {
- numRuns++;
- Map<String, DimensionDistributionReport> map;
- if (numRuns < 2) {
- map = firstMap;
- } else {
- map = secondMap;
- }
- retVal = (ParallelIndexTaskRunner<T, R>) new TestRunner(true, map, "SUCCESFUL-PHASE");
+ currentPhase++;
+ retVal = createMockDistributionRunner(true, "SUCCESFUL-PHASE");
}
- return retVal;
- }
- }
-
- static class TestRunner
- implements ParallelIndexTaskRunner<PartialDimensionDistributionTask, DimensionDistributionReport>
- {
- private final boolean succeeds;
- private final String phase;
-
- private final Map<String, DimensionDistributionReport> distributionMap;
-
- TestRunner(boolean succeeds, Map<String, DimensionDistributionReport> distributionMap, String phase)
- {
- this.succeeds = succeeds;
- this.distributionMap = distributionMap;
- this.phase = phase;
+ return (ParallelIndexTaskRunner<T, R>) retVal;
}
- @Override
- public String getName()
+ private PartialDimensionDistributionParallelIndexTaskRunner createMockDistributionRunner(
+ boolean succeeds,
+ String phase
+ )
{
- if (succeeds) {
- return StringUtils.format(phase);
- } else {
- return StringUtils.format(phase);
+ try {
+ final PartialDimensionDistributionParallelIndexTaskRunner runner = Mockito.mock(
+ PartialDimensionDistributionParallelIndexTaskRunner.class
+ );
+ Mockito.when(runner.getName()).thenReturn(phase);
+ Mockito.when(runner.run()).thenReturn(succeeds ? TaskState.SUCCESS : TaskState.FAILED);
+ Mockito.when(runner.getStopReason()).thenReturn(null);
+ Mockito.when(runner.getProgress()).thenReturn(null);
+ Mockito.when(runner.getIntervalToPartitionBoundaries(ArgumentMatchers.any()))
+ .thenReturn(INTERVAL_TO_PARTITIONS);
+
+ return runner;
}
- }
-
- @Override
- public TaskState run()
- {
- if (succeeds) {
- return TaskState.SUCCESS;
+ catch (Exception e) {
+ throw new ISE(e, "Error while mocking distribution phase runner");
}
- return TaskState.FAILED;
- }
-
- @Override
- public void stopGracefully(String stopReason)
- {
-
- }
-
- @Override
- public String getStopReason()
- {
- return null;
}
-
- @Override
- public void collectReport(DimensionDistributionReport report)
- {
-
- }
-
- @Override
- public Map<String, DimensionDistributionReport> getReports()
- {
- return distributionMap;
- }
-
- @Override
- public ParallelIndexingPhaseProgress getProgress()
- {
- return null;
- }
-
- @Override
- public Set<String> getRunningTaskIds()
- {
- return null;
- }
-
- @Override
- public List<SubTaskSpec<PartialDimensionDistributionTask>> getSubTaskSpecs()
- {
- return null;
- }
-
- @Override
- public List<SubTaskSpec<PartialDimensionDistributionTask>> getRunningSubTaskSpecs()
- {
- return null;
- }
-
- @Override
- public List<SubTaskSpec<PartialDimensionDistributionTask>> getCompleteSubTaskSpecs()
- {
- return null;
- }
-
- @Nullable
- @Override
- public SubTaskSpec<PartialDimensionDistributionTask> getSubTaskSpec(String subTaskSpecId)
- {
- return null;
- }
-
- @Nullable
- @Override
- public SubTaskSpecStatus getSubTaskState(String subTaskSpecId)
- {
- return null;
- }
-
- @Nullable
- @Override
- public TaskHistory<PartialDimensionDistributionTask> getCompleteSubTaskSpecAttemptHistory(String subTaskSpecId)
- {
- return null;
- }
-
}
protected ParallelIndexSupervisorTask newTask(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org