You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by kf...@apache.org on 2024/10/10 03:14:15 UTC
(druid) branch master updated: Fix duplicate compaction task launched by OverlordCompactionScheduler (#17287)
This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 3f797c52d0f Fix duplicate compaction task launched by OverlordCompactionScheduler (#17287)
3f797c52d0f is described below
commit 3f797c52d0f3910bb52724e2d2e748e2c2acc2f1
Author: Kashif Faraz <ka...@gmail.com>
AuthorDate: Thu Oct 10 08:44:09 2024 +0530
Fix duplicate compaction task launched by OverlordCompactionScheduler (#17287)
Description
-----------
The `OverlordCompactionScheduler` may sometimes launch a duplicate compaction
task for an interval that has just been compacted.
This may happen as follows:
- Scheduler launches a compaction task for an uncompacted interval.
- While the compaction task is running, the `CompactionStatusTracker` does not consider
this interval as compactible and returns the `CompactionStatus` as `SKIPPED` for it.
- As soon as the compaction task finishes, the `CompactionStatusTracker` starts considering
the interval eligible for compaction again.
- This interval remains eligible for compaction until the newly published segments are polled
from the database.
- Once the new segments have been polled, the `CompactionStatus` of the interval changes
to `COMPLETE`.
Change
--------
- Keep track of the `snapshotTime` in `DataSourcesSnapshot`. This time represents the start of the poll.
- Use the `snapshotTime` to determine if a poll has happened after a compaction task completed.
- If not, then skip the interval to avoid launching duplicate tasks.
- For tests, use a future `snapshotTime` to ensure that compaction is always triggered.
---
.../benchmark/DataSourcesSnapshotBenchmark.java | 11 ++--
.../NewestSegmentFirstPolicyBenchmark.java | 3 +-
.../compact/OverlordCompactionScheduler.java | 11 ++--
.../apache/druid/client/DataSourcesSnapshot.java | 61 ++++++++++++++--------
.../druid/metadata/SqlSegmentsMetadataManager.java | 16 +++---
.../server/compaction/CompactionRunSimulator.java | 6 +--
.../server/compaction/CompactionStatusTracker.java | 18 +++++++
.../druid/server/coordinator/DruidCoordinator.java | 21 ++++++--
.../coordinator/DruidCoordinatorRuntimeParams.java | 3 +-
.../server/coordinator/duty/CompactSegments.java | 9 ++--
.../compaction/CompactionRunSimulatorTest.java | 3 +-
.../compaction/CompactionStatusTrackerTest.java | 35 +++++++++++++
.../server/coordinator/DruidCoordinatorTest.java | 13 ++---
.../coordinator/duty/CompactSegmentsTest.java | 33 +++++-------
.../simulate/TestSegmentsMetadataManager.java | 3 +-
15 files changed, 153 insertions(+), 93 deletions(-)
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java
index ab6fff0186e..23067aa8318 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/DataSourcesSnapshotBenchmark.java
@@ -41,9 +41,7 @@ import org.openjdk.jmh.infra.Blackhole;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.TimeUnit;
@State(Scope.Benchmark)
@@ -54,7 +52,7 @@ import java.util.concurrent.TimeUnit;
@Measurement(iterations = 50)
public class DataSourcesSnapshotBenchmark
{
- private static Interval TEST_SEGMENT_INTERVAL = Intervals.of("2012-03-15T00:00:00.000/2012-03-16T00:00:00.000");
+ private static final Interval TEST_SEGMENT_INTERVAL = Intervals.of("2012-03-15T00:00:00.000/2012-03-16T00:00:00.000");
@Param({"500", "1000"})
private int numDataSources;
@@ -69,11 +67,10 @@ public class DataSourcesSnapshotBenchmark
{
long start = System.currentTimeMillis();
- Map<String, ImmutableDruidDataSource> dataSources = new HashMap<>();
+ final List<DataSegment> segments = new ArrayList<>();
for (int i = 0; i < numDataSources; i++) {
String dataSource = StringUtils.format("ds-%d", i);
- List<DataSegment> segments = new ArrayList<>();
for (int j = 0; j < numSegmentPerDataSource; j++) {
segments.add(
@@ -90,11 +87,9 @@ public class DataSourcesSnapshotBenchmark
)
);
}
-
- dataSources.put(dataSource, new ImmutableDruidDataSource(dataSource, Collections.emptyMap(), segments));
}
- snapshot = new DataSourcesSnapshot(dataSources);
+ snapshot = DataSourcesSnapshot.fromUsedSegments(segments);
System.out.println("Setup Time " + (System.currentTimeMillis() - start) + " ms");
}
diff --git a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
index fb938dca4cc..78c166e266b 100644
--- a/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/server/coordinator/NewestSegmentFirstPolicyBenchmark.java
@@ -20,7 +20,6 @@
package org.apache.druid.server.coordinator;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.DateTimes;
@@ -128,7 +127,7 @@ public class NewestSegmentFirstPolicyBenchmark
}
}
}
- dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of()).getUsedSegmentsTimelinesPerDataSource();
+ dataSources = DataSourcesSnapshot.fromUsedSegments(segments).getUsedSegmentsTimelinesPerDataSource();
}
@Benchmark
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
index 9e2668c8109..020c0a0ac22 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.inject.Inject;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.indexing.ClientCompactionRunnerInfo;
import org.apache.druid.indexer.TaskLocation;
import org.apache.druid.indexer.TaskStatus;
@@ -52,7 +53,6 @@ import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
import org.apache.druid.server.coordinator.stats.CoordinatorStat;
import org.apache.druid.server.coordinator.stats.Dimension;
import org.apache.druid.server.coordinator.stats.Stats;
-import org.apache.druid.timeline.SegmentTimeline;
import org.joda.time.Duration;
import java.util.ArrayList;
@@ -272,7 +272,7 @@ public class OverlordCompactionScheduler implements CompactionScheduler
private synchronized void runCompactionDuty()
{
final CoordinatorRunStats stats = new CoordinatorRunStats();
- duty.run(getLatestConfig(), getCurrentDatasourceTimelines(), supervisorConfig.getEngine(), stats);
+ duty.run(getLatestConfig(), getDatasourceSnapshot(), supervisorConfig.getEngine(), stats);
// Emit stats only if emission period has elapsed
if (!sinceStatsEmitted.isRunning() || sinceStatsEmitted.hasElapsed(METRIC_EMISSION_PERIOD)) {
@@ -309,7 +309,7 @@ public class OverlordCompactionScheduler implements CompactionScheduler
if (isRunning()) {
return new CompactionRunSimulator(statusTracker, overlordClient).simulateRunWithConfig(
getLatestConfig().withClusterConfig(updateRequest),
- getCurrentDatasourceTimelines(),
+ getDatasourceSnapshot(),
supervisorConfig.getEngine()
);
} else {
@@ -334,10 +334,9 @@ public class OverlordCompactionScheduler implements CompactionScheduler
.withDatasourceConfigs(new ArrayList<>(activeDatasourceConfigs.values()));
}
- private Map<String, SegmentTimeline> getCurrentDatasourceTimelines()
+ private DataSourcesSnapshot getDatasourceSnapshot()
{
- return segmentManager.getSnapshotOfDataSourcesWithAllUsedSegments()
- .getUsedSegmentsTimelinesPerDataSource();
+ return segmentManager.getSnapshotOfDataSourcesWithAllUsedSegments();
}
private void scheduleOnExecutor(Runnable runnable)
diff --git a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
index 5f58117a4ec..27f16fcddb1 100644
--- a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
+++ b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
@@ -21,11 +21,13 @@ package org.apache.druid.client;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
+import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.SegmentTimeline;
import org.apache.druid.timeline.VersionedIntervalTimeline;
import org.apache.druid.utils.CollectionUtils;
+import org.joda.time.DateTime;
import javax.annotation.Nullable;
import java.util.ArrayList;
@@ -40,43 +42,56 @@ import java.util.Map;
*/
public class DataSourcesSnapshot
{
- public static DataSourcesSnapshot fromUsedSegments(
- Iterable<DataSegment> segments,
- ImmutableMap<String, String> dataSourceProperties
- )
+ public static DataSourcesSnapshot fromUsedSegments(Iterable<DataSegment> segments)
{
+ return fromUsedSegments(segments, DateTimes.nowUtc());
+ }
+
+ /**
+ * Creates a snapshot of all "used" segments that existed in the database at
+ * the {@code snapshotTime}.
+ */
+ public static DataSourcesSnapshot fromUsedSegments(Iterable<DataSegment> segments, DateTime snapshotTime)
+ {
+ final Map<String, String> dataSourceProperties = ImmutableMap.of("created", DateTimes.nowUtc().toString());
Map<String, DruidDataSource> dataSources = new HashMap<>();
- segments.forEach(segment -> {
- dataSources
- .computeIfAbsent(segment.getDataSource(), dsName -> new DruidDataSource(dsName, dataSourceProperties))
- .addSegmentIfAbsent(segment);
- });
- return new DataSourcesSnapshot(CollectionUtils.mapValues(dataSources, DruidDataSource::toImmutableDruidDataSource));
+ segments.forEach(
+ segment -> dataSources
+ .computeIfAbsent(segment.getDataSource(), dsName -> new DruidDataSource(dsName, dataSourceProperties))
+ .addSegmentIfAbsent(segment)
+ );
+ return new DataSourcesSnapshot(
+ snapshotTime,
+ CollectionUtils.mapValues(dataSources, DruidDataSource::toImmutableDruidDataSource)
+ );
}
+ private final DateTime snapshotTime;
private final Map<String, ImmutableDruidDataSource> dataSourcesWithAllUsedSegments;
private final Map<String, SegmentTimeline> usedSegmentsTimelinesPerDataSource;
private final ImmutableSet<DataSegment> overshadowedSegments;
- public DataSourcesSnapshot(Map<String, ImmutableDruidDataSource> dataSourcesWithAllUsedSegments)
+ private DataSourcesSnapshot(
+ DateTime snapshotTime,
+ Map<String, ImmutableDruidDataSource> dataSourcesWithAllUsedSegments
+ )
{
- this(
+ this.snapshotTime = snapshotTime;
+ this.dataSourcesWithAllUsedSegments = dataSourcesWithAllUsedSegments;
+ this.usedSegmentsTimelinesPerDataSource = CollectionUtils.mapValues(
dataSourcesWithAllUsedSegments,
- CollectionUtils.mapValues(
- dataSourcesWithAllUsedSegments,
- dataSource -> SegmentTimeline.forSegments(dataSource.getSegments())
- )
+ dataSource -> SegmentTimeline.forSegments(dataSource.getSegments())
);
+ this.overshadowedSegments = ImmutableSet.copyOf(determineOvershadowedSegments());
}
- private DataSourcesSnapshot(
- Map<String, ImmutableDruidDataSource> dataSourcesWithAllUsedSegments,
- Map<String, SegmentTimeline> usedSegmentsTimelinesPerDataSource
- )
+ /**
+ * Time when this snapshot was taken. Since polling segments from the database
+ * may be a slow operation, this represents the poll start time.
+ */
+ public DateTime getSnapshotTime()
{
- this.dataSourcesWithAllUsedSegments = dataSourcesWithAllUsedSegments;
- this.usedSegmentsTimelinesPerDataSource = usedSegmentsTimelinesPerDataSource;
- this.overshadowedSegments = ImmutableSet.copyOf(determineOvershadowedSegments());
+ return snapshotTime;
}
public Collection<ImmutableDruidDataSource> getDataSourcesWithAllUsedSegments()
diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
index 12327785b92..053dc10f41c 100644
--- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
@@ -1035,6 +1035,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
private void doPollSegments()
{
+ final DateTime startTime = DateTimes.nowUtc();
final Stopwatch stopwatch = Stopwatch.createStarted();
// Some databases such as PostgreSQL require auto-commit turned off
@@ -1071,11 +1072,12 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
segments.size(), stopwatch.millisElapsed()
);
- createDatasourcesSnapshot(segments);
+ createDatasourcesSnapshot(startTime, segments);
}
private void doPollSegmentAndSchema()
{
+ final DateTime startTime = DateTimes.nowUtc();
final Stopwatch stopwatch = Stopwatch.createStarted();
ImmutableMap.Builder<SegmentId, SegmentMetadata> segmentMetadataBuilder = new ImmutableMap.Builder<>();
@@ -1174,7 +1176,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
segments.size(), schemaMap.size(), stopwatch.millisElapsed()
);
- createDatasourcesSnapshot(segments);
+ createDatasourcesSnapshot(startTime, segments);
}
private void emitMetric(String metricName, long value)
@@ -1182,7 +1184,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
serviceEmitter.emit(new ServiceMetricEvent.Builder().setMetric(metricName, value));
}
- private void createDatasourcesSnapshot(List<DataSegment> segments)
+ private void createDatasourcesSnapshot(DateTime snapshotTime, List<DataSegment> segments)
{
final Stopwatch stopwatch = Stopwatch.createStarted();
// dataSourcesSnapshot is updated only here and the DataSourcesSnapshot object is immutable. If data sources or
@@ -1193,11 +1195,10 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
// segment mark calls in rapid succession. So the snapshot update is not done outside of database poll at this time.
// Updates outside of database polls were primarily for the user experience, so users would immediately see the
// effect of a segment mark call reflected in MetadataResource API calls.
- ImmutableMap<String, String> dataSourceProperties = createDefaultDataSourceProperties();
dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(
Iterables.filter(segments, Objects::nonNull), // Filter corrupted entries (see above in this method).
- dataSourceProperties
+ snapshotTime
);
emitMetric("segment/buildSnapshot/time", stopwatch.millisElapsed());
log.debug(
@@ -1206,11 +1207,6 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
);
}
- private static ImmutableMap<String, String> createDefaultDataSourceProperties()
- {
- return ImmutableMap.of("created", DateTimes.nowUtc().toString());
- }
-
/**
* For the garbage collector in Java, it's better to keep new objects short-living, but once they are old enough
* (i. e. promoted to old generation), try to keep them alive. In {@link #poll()}, we fetch and deserialize all
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
index e3b1c66fba8..db0a5f8c98c 100644
--- a/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionRunSimulator.java
@@ -21,6 +21,7 @@ package org.apache.druid.server.compaction;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
import org.apache.druid.client.indexing.ClientCompactionTaskQueryTuningConfig;
import org.apache.druid.client.indexing.IndexingTotalWorkerCapacityInfo;
@@ -41,7 +42,6 @@ import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.server.coordinator.DruidCompactionConfig;
import org.apache.druid.server.coordinator.duty.CompactSegments;
import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
-import org.apache.druid.timeline.SegmentTimeline;
import org.joda.time.Interval;
import javax.annotation.Nullable;
@@ -76,7 +76,7 @@ public class CompactionRunSimulator
*/
public CompactionSimulateResult simulateRunWithConfig(
DruidCompactionConfig compactionConfig,
- Map<String, SegmentTimeline> datasourceTimelines,
+ DataSourcesSnapshot dataSourcesSnapshot,
CompactionEngine defaultEngine
)
{
@@ -146,7 +146,7 @@ public class CompactionRunSimulator
final CoordinatorRunStats stats = new CoordinatorRunStats();
new CompactSegments(simulationStatusTracker, readOnlyOverlordClient).run(
configWithUnlimitedTaskSlots,
- datasourceTimelines,
+ dataSourcesSnapshot,
defaultEngine,
stats
);
diff --git a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
index ab7ddbbb91a..cbf5f25f9d7 100644
--- a/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
+++ b/server/src/main/java/org/apache/druid/server/compaction/CompactionStatusTracker.java
@@ -34,6 +34,7 @@ import org.joda.time.Interval;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicReference;
/**
* Tracks status of recently submitted compaction tasks. Can be used by a segment
@@ -50,6 +51,8 @@ public class CompactionStatusTracker
private final ConcurrentHashMap<String, CompactionCandidate> submittedTaskIdToSegments
= new ConcurrentHashMap<>();
+ private final AtomicReference<DateTime> segmentSnapshotTime = new AtomicReference<>();
+
@Inject
public CompactionStatusTracker(ObjectMapper objectMapper)
{
@@ -109,6 +112,16 @@ public class CompactionStatusTracker
return CompactionStatus.skipped("Task for interval is already running");
}
+ // Skip intervals that have been recently compacted if segment timeline is not updated yet
+ final DateTime snapshotTime = segmentSnapshotTime.get();
+ if (lastTaskStatus != null
+ && lastTaskStatus.getState() == TaskState.SUCCESS
+ && snapshotTime != null && snapshotTime.isBefore(lastTaskStatus.getUpdatedTime())) {
+ return CompactionStatus.skipped(
+ "Segment timeline not updated since last compaction task succeeded"
+ );
+ }
+
// Skip intervals that have been filtered out by the policy
if (!searchPolicy.isEligibleForCompaction(candidate, compactionStatus, lastTaskStatus)) {
return CompactionStatus.skipped("Rejected by search policy");
@@ -125,6 +138,11 @@ public class CompactionStatusTracker
// Nothing to do, used by simulator
}
+ public void onSegmentTimelineUpdated(DateTime snapshotTime)
+ {
+ this.segmentSnapshotTime.set(snapshotTime);
+ }
+
public void onCompactionConfigUpdated(DruidCompactionConfig compactionConfig)
{
final Set<String> compactionEnabledDatasources = new HashSet<>();
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
index 4c16ba1107c..f8c29cd7984 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
@@ -27,6 +27,7 @@ import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntMaps;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2LongMap;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.DruidDataSource;
import org.apache.druid.client.DruidServer;
import org.apache.druid.client.ImmutableDruidDataSource;
@@ -37,6 +38,7 @@ import org.apache.druid.discovery.DruidLeaderSelector;
import org.apache.druid.guice.ManageLifecycle;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.indexer.CompactionEngine;
+import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
@@ -333,8 +335,7 @@ public class DruidCoordinator
return new CompactionRunSimulator(compactionStatusTracker, overlordClient).simulateRunWithConfig(
metadataManager.configs().getCurrentCompactionConfig().withClusterConfig(updateRequest),
metadataManager.segments()
- .getSnapshotOfDataSourcesWithAllUsedSegments()
- .getUsedSegmentsTimelinesPerDataSource(),
+ .getSnapshotOfDataSourcesWithAllUsedSegments(),
CompactionEngine.NATIVE
);
}
@@ -669,9 +670,23 @@ public class DruidCoordinator
}
if (metadataManager.isStarted() && serverInventoryView.isStarted()) {
+ final DataSourcesSnapshot dataSourcesSnapshot;
+ if (dutyGroup.getName().equals(COMPACT_SEGMENTS_DUTIES_DUTY_GROUP)) {
+ // If this is a compact segments duty group triggered by IT,
+ // use a future snapshotTime to ensure that compaction always runs
+ dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(
+ metadataManager.segments()
+ .getSnapshotOfDataSourcesWithAllUsedSegments()
+ .iterateAllUsedSegmentsInSnapshot(),
+ DateTimes.nowUtc().plusMinutes(60)
+ );
+ } else {
+ dataSourcesSnapshot = metadataManager.segments().getSnapshotOfDataSourcesWithAllUsedSegments();
+ }
+
final DruidCoordinatorRuntimeParams params = DruidCoordinatorRuntimeParams
.builder()
- .withDataSourcesSnapshot(metadataManager.segments().getSnapshotOfDataSourcesWithAllUsedSegments())
+ .withDataSourcesSnapshot(dataSourcesSnapshot)
.withDynamicConfigs(metadataManager.configs().getCurrentDynamicConfig())
.withCompactionConfig(metadataManager.configs().getCurrentCompactionConfig())
.build();
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
index ebef8754632..576b2155ac7 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
@@ -20,7 +20,6 @@
package org.apache.druid.server.coordinator;
import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.server.coordinator.balancer.BalancerStrategy;
import org.apache.druid.server.coordinator.loading.SegmentHolder;
@@ -331,7 +330,7 @@ public class DruidCoordinatorRuntimeParams
public Builder withUsedSegments(Collection<DataSegment> usedSegments)
{
this.usedSegmentsNewestFirst = createUsedSegmentsSet(usedSegments);
- this.dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments, ImmutableMap.of());
+ this.dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments);
return this;
}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
index 035286692bf..ca80c7efe30 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/CompactSegments.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonCreator;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.indexing.ClientCompactionIOConfig;
import org.apache.druid.client.indexing.ClientCompactionIntervalSpec;
import org.apache.druid.client.indexing.ClientCompactionRunnerInfo;
@@ -61,7 +62,6 @@ import org.apache.druid.server.coordinator.stats.Dimension;
import org.apache.druid.server.coordinator.stats.RowKey;
import org.apache.druid.server.coordinator.stats.Stats;
import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.SegmentTimeline;
import org.joda.time.Interval;
import javax.annotation.Nullable;
@@ -129,7 +129,7 @@ public class CompactSegments implements CoordinatorCustomDuty
} else {
run(
params.getCompactionConfig(),
- params.getUsedSegmentsTimelinesPerDataSource(),
+ params.getDataSourcesSnapshot(),
CompactionEngine.NATIVE,
params.getCoordinatorStats()
);
@@ -139,7 +139,7 @@ public class CompactSegments implements CoordinatorCustomDuty
public void run(
DruidCompactionConfig dynamicConfig,
- Map<String, SegmentTimeline> dataSources,
+ DataSourcesSnapshot dataSources,
CompactionEngine defaultEngine,
CoordinatorRunStats stats
)
@@ -150,6 +150,7 @@ public class CompactSegments implements CoordinatorCustomDuty
return;
}
+ statusTracker.onSegmentTimelineUpdated(dataSources.getSnapshotTime());
statusTracker.onCompactionConfigUpdated(dynamicConfig);
List<DataSourceCompactionConfig> compactionConfigList = dynamicConfig.getCompactionConfigs();
if (compactionConfigList == null || compactionConfigList.isEmpty()) {
@@ -225,7 +226,7 @@ public class CompactSegments implements CoordinatorCustomDuty
final CompactionSegmentIterator iterator = new PriorityBasedCompactionSegmentIterator(
policy,
compactionConfigs,
- dataSources,
+ dataSources.getUsedSegmentsTimelinesPerDataSource(),
intervalsToSkipCompaction,
statusTracker
);
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
index aacf4216de7..690a4bcfc7c 100644
--- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
@@ -76,8 +76,7 @@ public class CompactionRunSimulatorTest
DruidCompactionConfig.empty().withDatasourceConfig(
DataSourceCompactionConfig.builder().forDataSource("wiki").build()
),
- segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
- .getUsedSegmentsTimelinesPerDataSource(),
+ segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments(),
CompactionEngine.NATIVE
);
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
index f64c0865d6d..8cd20bfdb88 100644
--- a/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionStatusTrackerTest.java
@@ -24,8 +24,10 @@ import org.apache.druid.client.indexing.ClientCompactionTaskQuery;
import org.apache.druid.indexer.TaskState;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.segment.TestDataSource;
import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.timeline.DataSegment;
import org.junit.Assert;
import org.junit.Before;
@@ -104,6 +106,39 @@ public class CompactionStatusTrackerTest
Assert.assertEquals(2, status.getNumConsecutiveFailures());
}
+ @Test
+ public void testComputeCompactionStatusForSuccessfulTask()
+ {
+ final DataSourceCompactionConfig compactionConfig
+ = DataSourceCompactionConfig.builder().forDataSource(TestDataSource.WIKI).build();
+ final NewestSegmentFirstPolicy policy = new NewestSegmentFirstPolicy(null);
+ final CompactionCandidate candidateSegments
+ = CompactionCandidate.from(Collections.singletonList(WIKI_SEGMENT));
+
+ // Verify that interval is originally eligible for compaction
+ CompactionStatus status
+ = statusTracker.computeCompactionStatus(candidateSegments, compactionConfig, policy);
+ Assert.assertEquals(CompactionStatus.State.PENDING, status.getState());
+ Assert.assertEquals("not compacted yet", status.getReason());
+
+ // Verify that interval is skipped for compaction after task has finished
+ statusTracker.onSegmentTimelineUpdated(DateTimes.nowUtc().minusMinutes(1));
+ statusTracker.onTaskSubmitted(createCompactionTask("task1"), candidateSegments);
+ statusTracker.onTaskFinished("task1", TaskStatus.success("task1"));
+
+ status = statusTracker.computeCompactionStatus(candidateSegments, compactionConfig, policy);
+ Assert.assertEquals(CompactionStatus.State.SKIPPED, status.getState());
+ Assert.assertEquals(
+ "Segment timeline not updated since last compaction task succeeded",
+ status.getReason()
+ );
+
+ // Verify that interval becomes eligible again after timeline has been updated
+ statusTracker.onSegmentTimelineUpdated(DateTimes.nowUtc());
+ status = statusTracker.computeCompactionStatus(candidateSegments, compactionConfig, policy);
+ Assert.assertEquals(CompactionStatus.State.PENDING, status.getState());
+ }
+
private ClientCompactionTaskQuery createCompactionTask(
String taskId
)
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
index 0e8ff749dbf..512efd0316e 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
@@ -777,7 +777,6 @@ public class DruidCoordinatorTest extends CuratorTestBase
)
).andReturn(new AtomicReference<>(DruidCompactionConfig.empty())).anyTimes();
EasyMock.replay(configManager);
- DruidDataSource dataSource = new DruidDataSource("dataSource1", Collections.emptyMap());
DataSegment dataSegment = new DataSegment(
"dataSource1",
Intervals.of("2010-01-01/P1D"),
@@ -789,9 +788,9 @@ public class DruidCoordinatorTest extends CuratorTestBase
0x9,
0
);
- dataSource.addSegment(dataSegment);
- DataSourcesSnapshot dataSourcesSnapshot =
- new DataSourcesSnapshot(ImmutableMap.of(dataSource.getName(), dataSource.toImmutableDruidDataSource()));
+ DataSourcesSnapshot dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(
+ Collections.singleton(dataSegment)
+ );
EasyMock
.expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
.andReturn(dataSourcesSnapshot)
@@ -969,9 +968,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
@Test
public void testSimulateRunWithEmptyDatasourceCompactionConfigs()
{
- DruidDataSource dataSource = new DruidDataSource("dataSource", Collections.emptyMap());
- DataSourcesSnapshot dataSourcesSnapshot =
- new DataSourcesSnapshot(ImmutableMap.of(dataSource.getName(), dataSource.toImmutableDruidDataSource()));
+ DataSourcesSnapshot dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(Collections.emptyList());
EasyMock
.expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
.andReturn(dataSourcesSnapshot)
@@ -1022,7 +1019,7 @@ public class DruidCoordinatorTest extends CuratorTestBase
.andReturn(Collections.singleton(dataSource.toImmutableDruidDataSource()))
.anyTimes();
DataSourcesSnapshot dataSourcesSnapshot =
- new DataSourcesSnapshot(ImmutableMap.of(dataSource.getName(), dataSource.toImmutableDruidDataSource()));
+ DataSourcesSnapshot.fromUsedSegments(dataSource.getSegments());
EasyMock
.expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
.andReturn(dataSourcesSnapshot)
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
index 4009081474b..31789dd540c 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/CompactSegmentsTest.java
@@ -176,6 +176,7 @@ public class CompactSegmentsTest
private final BiFunction<Integer, Integer, ShardSpec> shardSpecFactory;
private final CompactionEngine engine;
+ private final List<DataSegment> allSegments = new ArrayList<>();
private DataSourcesSnapshot dataSources;
private CompactionStatusTracker statusTracker;
Map<String, List<DataSegment>> datasourceToSegments = new HashMap<>();
@@ -194,7 +195,7 @@ public class CompactSegmentsTest
@Before
public void setup()
{
- List<DataSegment> allSegments = new ArrayList<>();
+ allSegments.clear();
for (int i = 0; i < 3; i++) {
final String dataSource = DATA_SOURCE_PREFIX + i;
for (int j : new int[]{0, 1, 2, 3, 7, 8}) {
@@ -209,7 +210,7 @@ public class CompactSegmentsTest
}
}
}
- dataSources = DataSourcesSnapshot.fromUsedSegments(allSegments, ImmutableMap.of());
+ dataSources = DataSourcesSnapshot.fromUsedSegments(allSegments);
statusTracker = new CompactionStatusTracker(JSON_MAPPER);
}
@@ -435,7 +436,7 @@ public class CompactSegmentsTest
}
}
- dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
+ dataSources = DataSourcesSnapshot.fromUsedSegments(segments);
final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
@@ -589,7 +590,7 @@ public class CompactSegmentsTest
}
}
- dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
+ dataSources = DataSourcesSnapshot.fromUsedSegments(segments);
final TestOverlordClient overlordClient = new TestOverlordClient(JSON_MAPPER);
final CompactSegments compactSegments = new CompactSegments(statusTracker, overlordClient);
@@ -1497,7 +1498,7 @@ public class CompactSegmentsTest
10L
)
);
- dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
+ dataSources = DataSourcesSnapshot.fromUsedSegments(segments);
final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
@@ -1584,7 +1585,7 @@ public class CompactSegmentsTest
10L
)
);
- dataSources = DataSourcesSnapshot.fromUsedSegments(segments, ImmutableMap.of());
+ dataSources = DataSourcesSnapshot.fromUsedSegments(segments);
final OverlordClient mockClient = Mockito.mock(OverlordClient.class);
final ArgumentCaptor<Object> payloadCaptor = setUpMockClient(mockClient);
@@ -1988,22 +1989,14 @@ public class CompactSegmentsTest
private void addMoreData(String dataSource, int day)
{
- final SegmentTimeline timeline
- = dataSources.getUsedSegmentsTimelinesPerDataSource().get(dataSource);
for (int i = 0; i < 2; i++) {
- DataSegment newSegment = createSegment(dataSource, day, true, i);
- timeline.add(
- newSegment.getInterval(),
- newSegment.getVersion(),
- newSegment.getShardSpec().createChunk(newSegment)
- );
- newSegment = createSegment(dataSource, day, false, i);
- timeline.add(
- newSegment.getInterval(),
- newSegment.getVersion(),
- newSegment.getShardSpec().createChunk(newSegment)
- );
+ allSegments.add(createSegment(dataSource, day, true, i));
+ allSegments.add(createSegment(dataSource, day, false, i));
}
+
+ // Recreate the DataSourcesSnapshot with a future snapshotTime so that the
+ // statusTracker considers the intervals with new data eligible for compaction again
+ dataSources = DataSourcesSnapshot.fromUsedSegments(allSegments, DateTimes.nowUtc().plusMinutes(10));
}
private List<DataSourceCompactionConfig> createCompactionConfigs()
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
index 934f61dbe07..ad8294786e6 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
@@ -20,7 +20,6 @@
package org.apache.druid.server.coordinator.simulate;
import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableMap;
import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.metadata.SegmentsMetadataManager;
@@ -171,7 +170,7 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
public DataSourcesSnapshot getSnapshotOfDataSourcesWithAllUsedSegments()
{
if (snapshot == null) {
- snapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments.values(), ImmutableMap.of());
+ snapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments.values());
}
return snapshot;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org