You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by kf...@apache.org on 2025/04/29 05:52:01 UTC
(druid) branch master updated: Allow use of SegmentMetadataCache on the Coordinator (#17935)
This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new f71263f762f Allow use of SegmentMetadataCache on the Coordinator (#17935)
f71263f762f is described below
commit f71263f762f2ab22664a299296bfe9ec76fb5fb3
Author: Kashif Faraz <ka...@gmail.com>
AuthorDate: Tue Apr 29 11:21:56 2025 +0530
Allow use of SegmentMetadataCache on the Coordinator (#17935)
Change summary
-----------------
- Repurpose `SegmentsMetadataManager` to manage only segment polling and caching
- Move all segment metadata read or write APIs to `IndexerMetadataStorageCoordinator`
- Move implementation of above methods to `SqlSegmentsMetadataQuery`
- Simplify `SegmentsMetadataManager` interface by removing all redundant methods such as
`getImmutableDatasources` and `getAllUsedSegments`, which were anyway just delegated to the snapshot
- Add method `getDatasourcesSnapshot()` to `SegmentMetadataCache`
- Add `SqlSegmentsMetadataManagerV2` which is a wrapper over `SegmentMetadataCache` and
`SqlSegmentsMetadataManager` and can use either one to get the `DataSourcesSnapshot`
- Use config `druid.manager.segments.useIncrementalCache` on Coordinator to enable building
snapshot in `HeapMemorySegmentMetadataCache`
REST API Changes
-------------------
- APIs `markUsed` and `markUnused` on the Coordinator do not update the metadata store anymore
- These APIs redirect to the corresponding Overlord API
- If the request to the Overlord returns 404 (due to the Overlord being on an older version), the request fails
rather than falling back to updating the metadata store directly
---
docs/configuration/index.md | 5 +-
.../MaterializedViewSupervisor.java | 8 +-
.../MaterializedViewSupervisorSpec.java | 7 -
.../MaterializedViewSupervisorSpecTest.java | 6 -
.../MaterializedViewSupervisorTest.java | 7 +-
.../materializedview/DatasourceOptimizerTest.java | 2 +
.../mysql/MySQLMetadataStorageModuleTest.java | 10 +
.../compact/OverlordCompactionScheduler.java | 2 +-
.../druid/indexing/overlord/DruidOverlord.java | 15 +-
.../overlord/http/OverlordDataSourcesResource.java | 37 +-
.../actions/RetrieveSegmentsActionsTest.java | 2 +-
.../indexing/common/actions/TaskActionTestKit.java | 32 +-
.../druid/indexing/common/task/IndexTaskTest.java | 4 +-
.../indexing/common/task/IngestionTestBase.java | 15 +-
.../common/task/KillUnusedSegmentsTaskTest.java | 75 +-
.../AbstractParallelIndexSupervisorTaskTest.java | 26 +-
.../overlord/TaskLockBoxConcurrencyTest.java | 3 +
.../druid/indexing/overlord/TaskLockboxTest.java | 3 +
.../indexing/overlord/TaskQueueScaleTest.java | 3 +
.../http/OverlordDataSourcesResourceTest.java | 42 +-
.../SeekableStreamIndexTaskTestBase.java | 2 +
.../TestIndexerMetadataStorageCoordinator.java | 69 +-
.../Common/environment-configs/coordinator.env | 1 +
.../Common/environment-configs/overlord.env | 4 +-
...ose.cds-coordinator-metadata-query-disabled.yml | 2 +
...er-compose.cds-task-schema-publish-disabled.yml | 2 +
...ocker-compose.centralized-datasource-schema.yml | 2 +
.../docker/environment-configs/coordinator | 1 +
.../docker/environment-configs/overlord | 4 +-
.../apache/druid/java/util/common/StringUtils.java | 10 +
.../apache/druid/common/utils/StringUtilsTest.java | 6 +
.../apache/druid/client/DataSourcesSnapshot.java | 46 +-
.../druid/guice/SQLMetadataStorageDruidModule.java | 10 +-
.../IndexerMetadataStorageCoordinator.java | 91 ++
.../IndexerSQLMetadataStorageCoordinator.java | 139 +++
.../druid/metadata/SQLMetadataRuleManager.java | 2 +-
.../druid/metadata/SegmentsMetadataManager.java | 184 +--
.../metadata/SegmentsMetadataManagerConfig.java | 13 +-
.../druid/metadata/SqlSegmentsMetadataManager.java | 504 +-------
.../SqlSegmentsMetadataManagerProvider.java | 16 +-
.../druid/metadata/SqlSegmentsMetadataQuery.java | 290 ++++-
.../segment/SqlSegmentMetadataTransaction.java | 4 +-
.../SqlSegmentMetadataTransactionFactory.java | 30 +-
.../segment/SqlSegmentsMetadataManagerV2.java | 185 +++
.../cache/HeapMemorySegmentMetadataCache.java | 93 +-
.../druid/metadata/segment/cache/Metric.java | 5 +
.../segment/cache/NoopSegmentMetadataCache.java | 14 +
.../segment/cache/SegmentMetadataCache.java | 14 +-
.../metadata/CoordinatorSegmentMetadataCache.java | 20 +-
.../CoordinatorOverlordServiceConfig.java | 3 +
.../druid/server/coordinator/DruidCoordinator.java | 32 +-
.../druid/server/coordinator/MetadataManager.java | 31 +-
.../coordinator/duty/KillCompactionConfig.java | 10 +-
.../coordinator/duty/KillUnusedSegments.java | 12 +-
.../druid/server/http/DataSourcesResource.java | 203 +---
.../apache/druid/server/http/MetadataResource.java | 23 +-
...rSQLMetadataStorageCoordinatorMarkUsedTest.java | 711 ++++++++++++
.../IndexerSQLMetadataStorageCoordinatorTest.java | 5 +-
...ataStorageCoordinatorSchemaPersistenceTest.java | 2 +
...dexerSqlMetadataStorageCoordinatorTestBase.java | 11 +-
.../druid/metadata/SQLMetadataRuleManagerTest.java | 2 +-
.../SqlSegmentsMetadataManagerProviderTest.java | 5 +-
.../SqlSegmentsMetadataManagerSchemaPollTest.java | 34 +-
.../metadata/SqlSegmentsMetadataManagerTest.java | 1228 +++-----------------
.../SqlSegmentsMetadataManagerTestBase.java | 119 +-
.../metadata/SqlSegmentsMetadataQueryTest.java | 317 +++++
.../segment/SqlSegmentsMetadataManagerV2Test.java | 212 ++++
.../cache/HeapMemorySegmentMetadataCacheTest.java | 4 +-
...CoordinatorSegmentDataCacheConcurrencyTest.java | 15 +-
.../CoordinatorSegmentMetadataCacheTest.java | 155 +--
.../compaction/CompactionRunSimulatorTest.java | 2 +-
.../server/coordinator/DruidCoordinatorTest.java | 28 +-
.../coordinator/duty/KillCompactionConfigTest.java | 28 +-
.../coordinator/duty/KillUnusedSegmentsTest.java | 46 +-
.../duty/MarkEternityTombstonesAsUnusedTest.java | 28 +-
.../duty/MarkOvershadowedSegmentsAsUnusedTest.java | 8 +-
.../simulate/CoordinatorSimulationBuilder.java | 11 +-
.../simulate/TestSegmentsMetadataManager.java | 92 +-
.../druid/server/http/DataSourcesResourceTest.java | 85 +-
.../druid/server/http/MetadataResourceTest.java | 9 +-
.../java/org/apache/druid/cli/CliOverlordTest.java | 6 +
.../sql/calcite/schema/MetadataSegmentView.java | 10 +-
82 files changed, 2991 insertions(+), 2563 deletions(-)
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index 328ab350825..c1f78012347 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -925,6 +925,7 @@ These Coordinator static configurations can be defined in the `coordinator/runti
|--------|-----------|-------|
|`druid.manager.config.pollDuration`|How often the manager polls the config table for updates.|`PT1M`|
|`druid.manager.segments.pollDuration`|The duration between polls the Coordinator does for updates to the set of active segments. Generally defines the amount of lag time it can take for the Coordinator to notice new segments.|`PT1M`|
+|`druid.manager.segments.useIncrementalCache`|(Experimental) Denotes the usage mode of the segment metadata incremental cache. This cache provides a performance improvement over the polling mechanism currently employed by the Coordinator as it retrieves payloads of only updated segments. Possible cache modes are: (a) `never`: Incremental cache is disabled. (b) `always`: Incremental cache is enabled. Service start-up will be blocked until cache has synced with the metadata store at least [...]
|`druid.manager.rules.pollDuration`|The duration between polls the Coordinator does for updates to the set of active rules. Generally defines the amount of lag time it can take for the Coordinator to notice rules.|`PT1M`|
|`druid.manager.rules.defaultRule`|The default rule for the cluster|`_default`|
|`druid.manager.rules.alertThreshold`|The duration after a failed poll upon which an alert should be emitted.|`PT10M`|
@@ -1176,8 +1177,8 @@ The following properties pertain to segment metadata caching on the Overlord tha
|Property|Description|Default|
|--------|-----------|-------|
-|`druid.manager.segments.useCache`|Denotes the usage mode of the segment metadata cache. Possible modes are: (a) `never`: Cache is disabled. (b) `always`: Reads are always done from the cache. Service start-up will be blocked until cache has synced with the metadata store at least once. Transactions will block until cache has synced with the metadata store at least once after becoming leader. (c) `ifSynced`: Reads are done from the cache only if it has already synced with the metadata st [...]
-|`druid.manager.segments.pollDuration`|Duration (in ISO 8601 format) between successive syncs of the cache with the metadata store. This property is used only when `druid.manager.segments.useCache` is set to `always` or `ifSynced`.|`PT1M` (1 minute)|
+|`druid.manager.segments.useIncrementalCache`|Denotes the usage mode of the segment metadata incremental cache. Possible modes are: (a) `never`: Cache is disabled. (b) `always`: Reads are always done from the cache. Service start-up will be blocked until cache has synced with the metadata store at least once. Transactions will block until cache has synced with the metadata store at least once after becoming leader. (c) `ifSynced`: Reads are done from the cache only if it has already sync [...]
+|`druid.manager.segments.pollDuration`|Duration (in ISO 8601 format) between successive syncs of the cache with the metadata store. This property is used only when `druid.manager.segments.useIncrementalCache` is set to `always` or `ifSynced`.|`PT1M` (1 minute)|
#### Overlord dynamic configuration
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
index 0beb384354e..5622556b389 100644
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
+++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
@@ -49,7 +49,6 @@ import org.apache.druid.java.util.common.concurrent.Execs;
import org.apache.druid.java.util.common.guava.Comparators;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.timeline.DataSegment;
import org.joda.time.Duration;
import org.joda.time.Interval;
@@ -77,7 +76,6 @@ public class MaterializedViewSupervisor implements Supervisor
private final MetadataSupervisorManager metadataSupervisorManager;
private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private final SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
private final MaterializedViewSupervisorSpec spec;
private final TaskMaster taskMaster;
private final TaskStorage taskStorage;
@@ -104,7 +102,6 @@ public class MaterializedViewSupervisor implements Supervisor
TaskMaster taskMaster,
TaskStorage taskStorage,
MetadataSupervisorManager metadataSupervisorManager,
- SqlSegmentsMetadataManager sqlSegmentsMetadataManager,
IndexerMetadataStorageCoordinator metadataStorageCoordinator,
MaterializedViewTaskConfig config,
MaterializedViewSupervisorSpec spec
@@ -113,7 +110,6 @@ public class MaterializedViewSupervisor implements Supervisor
this.taskMaster = taskMaster;
this.taskStorage = taskStorage;
this.metadataStorageCoordinator = metadataStorageCoordinator;
- this.sqlSegmentsMetadataManager = sqlSegmentsMetadataManager;
this.metadataSupervisorManager = metadataSupervisorManager;
this.config = config;
this.spec = spec;
@@ -391,7 +387,7 @@ public class MaterializedViewSupervisor implements Supervisor
// drop derivative segments which interval equals the interval in toDeleteBaseSegments
for (Interval interval : toDropInterval.keySet()) {
for (DataSegment segment : derivativeSegments.get(interval)) {
- sqlSegmentsMetadataManager.markSegmentAsUnused(segment.getId());
+ metadataStorageCoordinator.markSegmentAsUnused(segment.getId());
}
}
// data of the latest interval will be built firstly.
@@ -500,7 +496,7 @@ public class MaterializedViewSupervisor implements Supervisor
{
log.info("Clear all metadata of dataSource %s", dataSource);
metadataStorageCoordinator.deletePendingSegments(dataSource);
- sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(dataSource);
+ metadataStorageCoordinator.markAllSegmentsAsUnused(dataSource);
metadataStorageCoordinator.deleteDataSourceMetadata(dataSource);
}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
index b7b3249a8b9..a757a065832 100644
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
+++ b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
@@ -44,7 +44,6 @@ import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.segment.indexing.DataSchema;
import org.apache.druid.segment.realtime.ChatHandlerProvider;
@@ -76,7 +75,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
private final ObjectMapper objectMapper;
private final MetadataSupervisorManager metadataSupervisorManager;
private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private final SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
private final TaskMaster taskMaster;
private final TaskStorage taskStorage;
private final MaterializedViewTaskConfig config;
@@ -100,7 +98,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
@JacksonInject TaskMaster taskMaster,
@JacksonInject TaskStorage taskStorage,
@JacksonInject MetadataSupervisorManager metadataSupervisorManager,
- @JacksonInject SqlSegmentsMetadataManager sqlSegmentsMetadataManager,
@JacksonInject IndexerMetadataStorageCoordinator metadataStorageCoordinator,
@JacksonInject MaterializedViewTaskConfig config,
@JacksonInject AuthorizerMapper authorizerMapper,
@@ -142,7 +139,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
this.taskMaster = taskMaster;
this.taskStorage = taskStorage;
this.metadataSupervisorManager = metadataSupervisorManager;
- this.sqlSegmentsMetadataManager = sqlSegmentsMetadataManager;
this.metadataStorageCoordinator = metadataStorageCoordinator;
this.authorizerMapper = authorizerMapper;
this.chatHandlerProvider = chatHandlerProvider;
@@ -353,7 +349,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
taskMaster,
taskStorage,
metadataSupervisorManager,
- sqlSegmentsMetadataManager,
metadataStorageCoordinator,
config,
this
@@ -384,7 +379,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
taskMaster,
taskStorage,
metadataSupervisorManager,
- sqlSegmentsMetadataManager,
metadataStorageCoordinator,
config,
authorizerMapper,
@@ -411,7 +405,6 @@ public class MaterializedViewSupervisorSpec implements SupervisorSpec
taskMaster,
taskStorage,
metadataSupervisorManager,
- sqlSegmentsMetadataManager,
metadataStorageCoordinator,
config,
authorizerMapper,
diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java
index c7096675ec9..d690e007640 100644
--- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java
+++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java
@@ -34,7 +34,6 @@ import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfi
import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -71,7 +70,6 @@ public class MaterializedViewSupervisorSpecTest
.addValue(ExprMacroTable.class.getName(), LookupEnabledTestExprMacroTable.INSTANCE)
.addValue(ObjectMapper.class, objectMapper)
.addValue(MetadataSupervisorManager.class, null)
- .addValue(SqlSegmentsMetadataManager.class, null)
.addValue(IndexerMetadataStorageCoordinator.class, null)
.addValue(MaterializedViewTaskConfig.class, new MaterializedViewTaskConfig())
.addValue(AuthorizerMapper.class, EasyMock.createMock(AuthorizerMapper.class))
@@ -141,7 +139,6 @@ public class MaterializedViewSupervisorSpecTest
null,
null,
null,
- null,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
new NoopChatHandlerProvider(),
@@ -190,7 +187,6 @@ public class MaterializedViewSupervisorSpecTest
null,
null,
null,
- null,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
new NoopChatHandlerProvider(),
@@ -297,7 +293,6 @@ public class MaterializedViewSupervisorSpecTest
null,
null,
null,
- null,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
new NoopChatHandlerProvider(),
@@ -342,7 +337,6 @@ public class MaterializedViewSupervisorSpecTest
null,
null,
null,
- null,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
new NoopChatHandlerProvider(),
diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java
index 444eb7cb61b..62b3f4af3d7 100644
--- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java
+++ b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java
@@ -28,6 +28,7 @@ import com.google.common.collect.Sets;
import junit.framework.AssertionFailedError;
import org.apache.druid.data.input.impl.DimensionsSpec;
import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.error.EntryAlreadyExists;
import org.apache.druid.indexer.HadoopIOConfig;
import org.apache.druid.indexer.HadoopIngestionSpec;
@@ -44,7 +45,6 @@ import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
@@ -86,7 +86,6 @@ public class MaterializedViewSupervisorTest
private TaskMaster taskMaster;
private IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator;
private MetadataSupervisorManager metadataSupervisorManager;
- private SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
private TaskQueue taskQueue;
private MaterializedViewSupervisor supervisor;
private String derivativeDatasourceName;
@@ -114,6 +113,7 @@ public class MaterializedViewSupervisorTest
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
@@ -124,7 +124,6 @@ public class MaterializedViewSupervisorTest
CentralizedDatasourceSchemaConfig.create()
);
metadataSupervisorManager = EasyMock.createMock(MetadataSupervisorManager.class);
- sqlSegmentsMetadataManager = EasyMock.createMock(SqlSegmentsMetadataManager.class);
taskQueue = EasyMock.createMock(TaskQueue.class);
objectMapper.registerSubtypes(new NamedType(HashBasedNumberedShardSpec.class, "hashed"));
@@ -143,7 +142,6 @@ public class MaterializedViewSupervisorTest
taskMaster,
taskStorage,
metadataSupervisorManager,
- sqlSegmentsMetadataManager,
indexerMetadataStorageCoordinator,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
@@ -325,7 +323,6 @@ public class MaterializedViewSupervisorTest
taskMaster,
taskStorage,
metadataSupervisorManager,
- sqlSegmentsMetadataManager,
indexerMetadataStorageCoordinator,
new MaterializedViewTaskConfig(),
EasyMock.createMock(AuthorizerMapper.class),
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java
index 1ec98359db7..755b0efddac 100644
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java
+++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java
@@ -37,6 +37,7 @@ import org.apache.druid.client.DruidServer;
import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy;
import org.apache.druid.client.selector.RandomServerSelectorStrategy;
import org.apache.druid.curator.CuratorTestBase;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexing.materializedview.DerivativeDataSourceMetadata;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.Intervals;
@@ -122,6 +123,7 @@ public class DatasourceOptimizerTest extends CuratorTestBase
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
diff --git a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java
index cbc3e3ff65e..c1d008bfbb1 100644
--- a/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java
+++ b/extensions-core/mysql-metadata-storage/src/test/java/org/apache/druid/metadata/storage/mysql/MySQLMetadataStorageModuleTest.java
@@ -28,12 +28,14 @@ import com.google.inject.Module;
import com.google.inject.Provides;
import org.apache.druid.client.indexing.IndexingService;
import org.apache.druid.discovery.DruidLeaderSelector;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.guice.GuiceInjectors;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.JsonConfigurator;
import org.apache.druid.guice.LifecycleModule;
import org.apache.druid.guice.MetadataConfigModule;
import org.apache.druid.guice.annotations.Json;
+import org.apache.druid.guice.annotations.Self;
import org.apache.druid.guice.security.EscalatorModule;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory;
import org.apache.druid.java.util.emitter.core.NoopEmitter;
@@ -42,6 +44,7 @@ import org.junit.Assert;
import org.junit.Test;
import java.util.Properties;
+import java.util.Set;
public class MySQLMetadataStorageModuleTest
{
@@ -147,6 +150,13 @@ public class MySQLMetadataStorageModuleTest
// Required for HeapMemorySegmentMetadataCache
return null;
}
+
+ @Self
+ @Provides
+ public Set<NodeRole> getNodeRoles()
+ {
+ return Set.of(NodeRole.OVERLORD);
+ }
}
)
);
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
index 9db8a73d381..95ea3d11693 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/compact/OverlordCompactionScheduler.java
@@ -389,7 +389,7 @@ public class OverlordCompactionScheduler implements CompactionScheduler
private DataSourcesSnapshot getDatasourceSnapshot()
{
- return segmentManager.getSnapshotOfDataSourcesWithAllUsedSegments();
+ return segmentManager.getRecentDataSourcesSnapshot();
}
private void scheduleOnExecutor(Runnable runnable, long delaySeconds)
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
index aa7a38a1f9e..36c8f36696b 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/DruidOverlord.java
@@ -60,6 +60,7 @@ public class DruidOverlord
private final DruidLeaderSelector overlordLeaderSelector;
private final DruidLeaderSelector.Listener leadershipListener;
private final SegmentMetadataCache segmentMetadataCache;
+ private final CoordinatorOverlordServiceConfig coordinatorOverlordServiceConfig;
private final ReentrantLock giant = new ReentrantLock(true);
@@ -100,6 +101,7 @@ public class DruidOverlord
{
this.overlordLeaderSelector = overlordLeaderSelector;
this.segmentMetadataCache = segmentMetadataCache;
+ this.coordinatorOverlordServiceConfig = coordinatorOverlordServiceConfig;
final DruidNode node = coordinatorOverlordServiceConfig.getOverlordService() == null ? selfNode :
selfNode.withService(coordinatorOverlordServiceConfig.getOverlordService());
@@ -227,7 +229,9 @@ public class DruidOverlord
giant.lock();
try {
- segmentMetadataCache.start();
+ if (isStandalone()) {
+ segmentMetadataCache.start();
+ }
overlordLeaderSelector.registerListener(leadershipListener);
}
finally {
@@ -247,7 +251,9 @@ public class DruidOverlord
try {
gracefulStopLeaderLifecycle();
overlordLeaderSelector.unregisterListener();
- segmentMetadataCache.stop();
+ if (isStandalone()) {
+ segmentMetadataCache.stop();
+ }
}
finally {
giant.unlock();
@@ -292,4 +298,9 @@ public class DruidOverlord
}
}
+ private boolean isStandalone()
+ {
+ return !coordinatorOverlordServiceConfig.isEnabled();
+ }
+
}
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResource.java b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResource.java
index 94e4ef274a8..1f711167810 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResource.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResource.java
@@ -31,7 +31,6 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.indexing.overlord.TaskMaster;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.rpc.indexing.SegmentUpdateResponse;
import org.apache.druid.server.http.SegmentsToUpdateFilter;
import org.apache.druid.server.http.ServletResourceUtils;
@@ -65,7 +64,6 @@ public class OverlordDataSourcesResource
{
private static final Logger log = new Logger(OverlordDataSourcesResource.class);
- private final SegmentsMetadataManager segmentsMetadataManager;
private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
private final TaskMaster taskMaster;
private final AuditManager auditManager;
@@ -73,14 +71,12 @@ public class OverlordDataSourcesResource
@Inject
public OverlordDataSourcesResource(
TaskMaster taskMaster,
- SegmentsMetadataManager segmentsMetadataManager,
IndexerMetadataStorageCoordinator metadataStorageCoordinator,
AuditManager auditManager
)
{
this.taskMaster = taskMaster;
this.auditManager = auditManager;
- this.segmentsMetadataManager = segmentsMetadataManager;
this.metadataStorageCoordinator = metadataStorageCoordinator;
}
@@ -98,8 +94,8 @@ public class OverlordDataSourcesResource
@Context HttpServletRequest req
)
{
- SegmentUpdateOperation operation = () -> segmentsMetadataManager
- .markAsUsedAllNonOvershadowedSegmentsInDataSource(dataSourceName);
+ SegmentUpdateOperation operation = () -> metadataStorageCoordinator
+ .markAllNonOvershadowedSegmentsAsUsed(dataSourceName);
return performSegmentUpdate(dataSourceName, operation);
}
@@ -143,14 +139,14 @@ public class OverlordDataSourcesResource
final Interval interval = payload.getInterval();
final List<String> versions = payload.getVersions();
if (interval != null) {
- return segmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(dataSourceName, interval, versions);
+ return metadataStorageCoordinator.markNonOvershadowedSegmentsAsUsed(dataSourceName, interval, versions);
} else {
final Set<String> segmentIds = payload.getSegmentIds();
if (segmentIds == null || segmentIds.isEmpty()) {
return 0;
}
- return segmentsMetadataManager.markAsUsedNonOvershadowedSegments(
+ return metadataStorageCoordinator.markNonOvershadowedSegmentsAsUsed(
dataSourceName,
IdUtils.getValidSegmentIds(dataSourceName, segmentIds)
);
@@ -213,11 +209,22 @@ public class OverlordDataSourcesResource
@ResourceFilters(DatasourceResourceFilter.class)
public Response markSegmentAsUsed(
@PathParam("dataSourceName") String dataSourceName,
- @PathParam("segmentId") String segmentId
+ @PathParam("segmentId") String serializedSegmentId
)
{
+ final SegmentId segmentId = SegmentId.tryParse(dataSourceName, serializedSegmentId);
+ if (segmentId == null) {
+ return Response.status(Response.Status.BAD_REQUEST).entity(
+ StringUtils.format(
+ "Could not parse Segment ID[%s] for DataSource[%s]",
+ StringUtils.escapeHtml(serializedSegmentId),
+ StringUtils.escapeHtml(dataSourceName)
+ )
+ ).build();
+ }
+
SegmentUpdateOperation operation =
- () -> segmentsMetadataManager.markSegmentAsUsed(segmentId) ? 1 : 0;
+ () -> metadataStorageCoordinator.markSegmentAsUsed(segmentId) ? 1 : 0;
return performSegmentUpdate(dataSourceName, operation);
}
@@ -226,13 +233,17 @@ public class OverlordDataSourcesResource
@ResourceFilters(DatasourceResourceFilter.class)
public Response markSegmentAsUnused(
@PathParam("dataSourceName") String dataSourceName,
- @PathParam("segmentId") String segmentIdString
+ @PathParam("segmentId") String serializedSegmentId
)
{
- final SegmentId segmentId = SegmentId.tryParse(dataSourceName, segmentIdString);
+ final SegmentId segmentId = SegmentId.tryParse(dataSourceName, serializedSegmentId);
if (segmentId == null) {
return Response.status(Response.Status.BAD_REQUEST).entity(
- StringUtils.format("Could not parse Segment ID[%s] for DataSource[%s]", segmentIdString, dataSourceName)
+ StringUtils.format(
+ "Could not parse Segment ID[%s] for DataSource[%s]",
+ StringUtils.escapeHtml(serializedSegmentId),
+ StringUtils.escapeHtml(dataSourceName)
+ )
).build();
}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsActionsTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsActionsTest.java
index 8ef97352921..ef27e4a600b 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsActionsTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/RetrieveSegmentsActionsTest.java
@@ -78,7 +78,7 @@ public class RetrieveSegmentsActionsTest
expectedUsedSegments.forEach(s -> actionTestKit.getTaskLockbox().unlock(task, s.getInterval()));
- expectedUnusedSegments.forEach(s -> actionTestKit.getSegmentsMetadataManager().markSegmentAsUnused(s.getId()));
+ expectedUnusedSegments.forEach(s -> actionTestKit.getMetadataStorageCoordinator().markSegmentAsUnused(s.getId()));
}
private static DataSegment createSegment(Interval interval, String version)
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java
index ed30db142a9..f5a28d5d67c 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/actions/TaskActionTestKit.java
@@ -21,6 +21,7 @@ package org.apache.druid.indexing.common.actions;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Suppliers;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexing.common.TestUtils;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
import org.apache.druid.indexing.overlord.HeapMemoryTaskStorage;
@@ -33,15 +34,12 @@ import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
import org.apache.druid.metadata.MetadataStorageConnectorConfig;
import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache;
import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
-import org.apache.druid.segment.metadata.SegmentSchemaCache;
import org.apache.druid.segment.metadata.SegmentSchemaManager;
import org.apache.druid.server.coordinator.simulate.BlockingExecutorService;
import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector;
@@ -51,6 +49,8 @@ import org.easymock.EasyMock;
import org.joda.time.Period;
import org.junit.rules.ExternalResource;
+import java.util.Set;
+
public class TaskActionTestKit extends ExternalResource
{
private final MetadataStorageTablesConfig metadataStorageTablesConfig = MetadataStorageTablesConfig.fromBase("druid");
@@ -59,10 +59,7 @@ public class TaskActionTestKit extends ExternalResource
private TaskLockbox taskLockbox;
private TestDerbyConnector testDerbyConnector;
private IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private SegmentsMetadataManager segmentsMetadataManager;
private TaskActionToolbox taskActionToolbox;
- private SegmentSchemaManager segmentSchemaManager;
- private SegmentSchemaCache segmentSchemaCache;
private SegmentMetadataCache segmentMetadataCache;
private BlockingExecutorService metadataCachePollExec;
@@ -79,11 +76,6 @@ public class TaskActionTestKit extends ExternalResource
return metadataStorageCoordinator;
}
- public SegmentsMetadataManager getSegmentsMetadataManager()
- {
- return segmentsMetadataManager;
- }
-
public TaskActionToolbox getTaskActionToolbox()
{
return taskActionToolbox;
@@ -113,7 +105,11 @@ public class TaskActionTestKit extends ExternalResource
Suppliers.ofInstance(metadataStorageTablesConfig)
);
final ObjectMapper objectMapper = new TestUtils().getTestObjectMapper();
- segmentSchemaManager = new SegmentSchemaManager(metadataStorageTablesConfig, objectMapper, testDerbyConnector);
+ final SegmentSchemaManager segmentSchemaManager = new SegmentSchemaManager(
+ metadataStorageTablesConfig,
+ objectMapper,
+ testDerbyConnector
+ );
final SqlSegmentMetadataTransactionFactory transactionFactory = setupTransactionFactory(objectMapper);
metadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(
@@ -125,16 +121,6 @@ public class TaskActionTestKit extends ExternalResource
CentralizedDatasourceSchemaConfig.create()
);
taskLockbox = new TaskLockbox(taskStorage, metadataStorageCoordinator);
- segmentSchemaCache = new SegmentSchemaCache(NoopServiceEmitter.instance());
- segmentsMetadataManager = new SqlSegmentsMetadataManager(
- objectMapper,
- Suppliers.ofInstance(new SegmentsMetadataManagerConfig(null, null)),
- Suppliers.ofInstance(metadataStorageTablesConfig),
- testDerbyConnector,
- segmentSchemaCache,
- CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
- );
final TaskLockConfig taskLockConfig = new TaskLockConfig()
{
@Override
@@ -203,6 +189,7 @@ public class TaskActionTestKit extends ExternalResource
metadataStorageTablesConfig,
testDerbyConnector,
leaderSelector,
+ Set.of(NodeRole.OVERLORD),
segmentMetadataCache,
NoopServiceEmitter.instance()
)
@@ -223,7 +210,6 @@ public class TaskActionTestKit extends ExternalResource
taskLockbox = null;
testDerbyConnector = null;
metadataStorageCoordinator = null;
- segmentsMetadataManager = null;
taskActionToolbox = null;
segmentMetadataCache.stopBeingLeader();
segmentMetadataCache.stop();
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
index 53549d3c791..9cb6860ee37 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexTaskTest.java
@@ -2543,8 +2543,8 @@ public class IndexTaskTest extends IngestionTestBase
{
return Sets.newHashSet(
getSegmentsMetadataManager()
- .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(DATASOURCE, Intervals.ETERNITY, true)
- .get()
+ .forceUpdateDataSourcesSnapshot()
+ .getAllUsedNonOvershadowedSegments(DATASOURCE, Intervals.ETERNITY)
);
}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
index 6e7b30b4be2..69564ad6506 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/IngestionTestBase.java
@@ -34,6 +34,7 @@ import org.apache.druid.data.input.impl.JsonInputFormat;
import org.apache.druid.data.input.impl.ParseSpec;
import org.apache.druid.data.input.impl.RegexInputFormat;
import org.apache.druid.data.input.impl.RegexParseSpec;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexer.report.IngestionStatsAndErrors;
import org.apache.druid.indexer.report.IngestionStatsAndErrorsTaskReport;
@@ -70,9 +71,9 @@ import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
import org.apache.druid.metadata.SQLMetadataConnector;
import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
+import org.apache.druid.metadata.segment.SqlSegmentsMetadataManagerV2;
import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache;
import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
import org.apache.druid.segment.DataSegmentsWithSchemas;
@@ -176,14 +177,15 @@ public abstract class IngestionTestBase extends InitializedNullHandlingTest
CentralizedDatasourceSchemaConfig.create()
);
SegmentSchemaCache segmentSchemaCache = new SegmentSchemaCache(NoopServiceEmitter.instance());
- segmentsMetadataManager = new SqlSegmentsMetadataManager(
- objectMapper,
+ segmentsMetadataManager = new SqlSegmentsMetadataManagerV2(
+ segmentMetadataCache,
+ segmentSchemaCache,
+ derbyConnectorRule.getConnector(),
() -> new SegmentsMetadataManagerConfig(null, null),
derbyConnectorRule.metadataTablesConfigSupplier(),
- derbyConnectorRule.getConnector(),
- segmentSchemaCache,
CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
+ NoopServiceEmitter.instance(),
+ objectMapper
);
lockbox = new TaskLockbox(taskStorage, storageCoordinator);
segmentCacheManagerFactory = new SegmentCacheManagerFactory(TestIndex.INDEX_IO, getObjectMapper());
@@ -336,6 +338,7 @@ public abstract class IngestionTestBase extends InitializedNullHandlingTest
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnectorRule.getConnector(),
leaderSelector,
+ Set.of(NodeRole.OVERLORD),
segmentMetadataCache,
NoopServiceEmitter.instance()
);
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java
index 052c1ae0a20..d1eb4229aa9 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/KillUnusedSegmentsTaskTest.java
@@ -103,12 +103,12 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, announced);
Assert.assertTrue(
- getSegmentsMetadataManager().markSegmentAsUnused(
+ getMetadataStorageCoordinator().markSegmentAsUnused(
segment2.getId()
)
);
Assert.assertTrue(
- getSegmentsMetadataManager().markSegmentAsUnused(
+ getMetadataStorageCoordinator().markSegmentAsUnused(
segment3.getId()
)
);
@@ -233,8 +233,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
segment3.getId().toString()
);
insertUsedSegments(ImmutableSet.of(segment1, segment2, segment3), upgradeSegmentMapping);
- getSegmentsMetadataManager().markSegmentAsUnused(segment2.getId());
- getSegmentsMetadataManager().markSegmentAsUnused(segment3.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment2.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment3.getId());
final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder()
@@ -278,8 +278,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
segment3.getId().toString()
);
insertUsedSegments(ImmutableSet.of(segment1, segment2, segment3), upgradeSegmentMapping);
- getSegmentsMetadataManager().markSegmentAsUnused(segment1.getId());
- getSegmentsMetadataManager().markSegmentAsUnused(segment2.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment1.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment2.getId());
final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder()
@@ -323,9 +323,9 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
segment3.getId().toString()
);
insertUsedSegments(ImmutableSet.of(segment1, segment2, segment3), upgradeSegmentMapping);
- getSegmentsMetadataManager().markSegmentAsUnused(segment1.getId());
- getSegmentsMetadataManager().markSegmentAsUnused(segment2.getId());
- getSegmentsMetadataManager().markSegmentAsUnused(segment3.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment1.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment2.getId());
+ getMetadataStorageCoordinator().markSegmentAsUnused(segment3.getId());
final KillUnusedSegmentsTask task = new KillUnusedSegmentsTaskBuilder()
@@ -371,7 +371,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments, null));
Assert.assertEquals(
segments.size(),
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
segments.stream().map(DataSegment::getId).collect(Collectors.toSet())
)
);
@@ -419,7 +420,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments, null));
Assert.assertEquals(
segments.size(),
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
segments.stream().map(DataSegment::getId).collect(Collectors.toSet())
)
);
@@ -467,7 +469,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments, null));
Assert.assertEquals(
segments.size(),
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
segments.stream().map(DataSegment::getId).collect(Collectors.toSet())
)
);
@@ -516,7 +519,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments, null));
Assert.assertEquals(
segments.size(),
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
segments.stream().map(DataSegment::getId).collect(Collectors.toSet())
)
);
@@ -570,7 +574,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, getMetadataStorageCoordinator().commitSegments(segments, null));
Assert.assertEquals(
unusedSegments.size(),
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
unusedSegments.stream().map(DataSegment::getId).collect(Collectors.toSet())
)
);
@@ -638,7 +643,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(segments, announced);
Assert.assertEquals(
segments.size(),
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
Intervals.of("2018-01-01/2020-01-01"),
null
@@ -686,7 +691,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment1.getInterval(),
null
@@ -695,7 +700,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment4.getInterval(),
null
@@ -704,7 +709,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment3.getInterval(),
null
@@ -763,7 +768,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment1.getInterval(),
null
@@ -772,7 +777,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment4.getInterval(),
null
@@ -786,7 +791,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
// Now mark the third segment as unused
Assert.assertEquals(
1,
- getSegmentsMetadataManager().markAsUnusedSegmentsInInterval(
+ getMetadataStorageCoordinator().markSegmentsWithinIntervalAsUnused(
DATA_SOURCE,
segment3.getInterval(),
null
@@ -874,11 +879,9 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
2,
- getSegmentsMetadataManager().markSegmentsAsUnused(
- ImmutableSet.of(
- segment1.getId(),
- segment4.getId()
- )
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
+ ImmutableSet.of(segment1.getId(), segment4.getId())
)
);
@@ -888,11 +891,9 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
2,
- getSegmentsMetadataManager().markSegmentsAsUnused(
- ImmutableSet.of(
- segment2.getId(),
- segment3.getId()
- )
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
+ ImmutableSet.of(segment2.getId(), segment3.getId())
)
);
@@ -970,7 +971,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
3,
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
ImmutableSet.of(segment1.getId(), segment2.getId(), segment4.getId())
)
);
@@ -982,7 +984,8 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
Assert.assertEquals(
2,
- getSegmentsMetadataManager().markSegmentsAsUnused(
+ getMetadataStorageCoordinator().markSegmentsAsUnused(
+ DATA_SOURCE,
ImmutableSet.of(segment3.getId(), segment5.getId())
)
);
@@ -1057,7 +1060,7 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
for (DataSegment segment : segments) {
Assert.assertTrue(
- getSegmentsMetadataManager().markSegmentAsUnused(
+ getMetadataStorageCoordinator().markSegmentAsUnused(
segment.getId()
)
);
@@ -1485,12 +1488,10 @@ public class KillUnusedSegmentsTaskTest extends IngestionTestBase
private void insertUsedSegments(Set<DataSegment> segments, Map<String, String> upgradedFromSegmentIdMap)
{
- final String table = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
IndexerSqlMetadataStorageCoordinatorTestBase.insertUsedSegments(
segments,
upgradedFromSegmentIdMap,
- derbyConnectorRule.getConnector(),
- table,
+ derbyConnectorRule,
getObjectMapper()
);
}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
index 6c251ea584c..6a6d2bffbaf 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/common/task/batch/parallel/AbstractParallelIndexSupervisorTaskTest.java
@@ -30,7 +30,6 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.client.coordinator.CoordinatorClient;
import org.apache.druid.client.coordinator.NoopCoordinatorClient;
import org.apache.druid.client.indexing.TaskStatusResponse;
@@ -106,7 +105,6 @@ import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.server.security.AuthTestUtils;
import org.apache.druid.server.security.AuthorizerMapper;
import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.SegmentId;
import org.apache.druid.utils.CompressionUtils;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.joda.time.DateTime;
@@ -1025,26 +1023,22 @@ public class AbstractParallelIndexSupervisorTaskTest extends IngestionTestBase
@Override
public ListenableFuture<DataSegment> fetchSegment(String dataSource, String segmentId, boolean includeUnused)
{
- ImmutableDruidDataSource druidDataSource;
try {
- druidDataSource = exec.submit(
- () -> getSegmentsMetadataManager().getImmutableDataSourceWithUsedSegments(dataSource)
+ final DataSegment segment = exec.submit(
+ () -> includeUnused
+ ? getStorageCoordinator().retrieveSegmentForId(dataSource, segmentId)
+ : getStorageCoordinator().retrieveUsedSegmentForId(dataSource, segmentId)
).get();
- }
- catch (InterruptedException | ExecutionException e) {
- throw new RuntimeException(e);
- }
- if (druidDataSource == null) {
- throw new ISE("Unknown datasource[%s]", dataSource);
- }
- for (SegmentId possibleSegmentId : SegmentId.iteratePossibleParsingsWithDataSource(dataSource, segmentId)) {
- DataSegment segment = druidDataSource.getSegment(possibleSegmentId);
- if (segment != null) {
+ if (segment == null) {
+ throw new ISE("Can't find segment for id[%s]", segmentId);
+ } else {
return Futures.immediateFuture(segment);
}
}
- throw new ISE("Can't find segment for id[%s]", segmentId);
+ catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
}
}
}
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java
index a55117aad12..a6eb19f4783 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockBoxConcurrencyTest.java
@@ -22,6 +22,7 @@ package org.apache.druid.indexing.overlord;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import org.apache.druid.common.guava.SettableSupplier;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexing.common.TaskLockType;
import org.apache.druid.indexing.common.config.TaskStorageConfig;
@@ -49,6 +50,7 @@ import org.junit.Test;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -89,6 +91,7 @@ public class TaskLockBoxConcurrencyTest
derby.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java
index 48c5b5c4e94..8c2c78611ac 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskLockboxTest.java
@@ -29,6 +29,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexing.common.LockGranularity;
import org.apache.druid.indexing.common.SegmentLock;
@@ -137,6 +138,7 @@ public class TaskLockboxTest
tablesConfig,
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
@@ -479,6 +481,7 @@ public class TaskLockboxTest
derby.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java
index f54ea7fe586..1900752e8be 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/TaskQueueScaleTest.java
@@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
import com.google.errorprone.annotations.concurrent.GuardedBy;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexer.RunnerTaskState;
import org.apache.druid.indexer.TaskLocation;
import org.apache.druid.indexer.TaskStatus;
@@ -69,6 +70,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -111,6 +113,7 @@ public class TaskQueueScaleTest
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnectorRule.getConnector(),
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResourceTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResourceTest.java
index fa4a3f6c51b..53af56961d2 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResourceTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/overlord/http/OverlordDataSourcesResourceTest.java
@@ -20,7 +20,7 @@
package org.apache.druid.indexing.overlord.http;
import org.apache.druid.audit.AuditManager;
-import org.apache.druid.client.ImmutableDruidDataSource;
+import org.apache.druid.indexing.overlord.Segments;
import org.apache.druid.indexing.overlord.TaskMaster;
import org.apache.druid.indexing.test.TestIndexerMetadataStorageCoordinator;
import org.apache.druid.java.util.common.Intervals;
@@ -28,7 +28,6 @@ import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.rpc.indexing.SegmentUpdateResponse;
import org.apache.druid.segment.TestDataSource;
import org.apache.druid.server.coordinator.CreateDataSegments;
-import org.apache.druid.server.coordinator.simulate.TestSegmentsMetadataManager;
import org.apache.druid.server.http.SegmentsToUpdateFilter;
import org.apache.druid.server.security.AuthConfig;
import org.apache.druid.timeline.DataSegment;
@@ -55,7 +54,7 @@ public class OverlordDataSourcesResourceTest
.startingAt(WIKI_SEGMENTS_START)
.eachOfSizeInMb(500);
- private TestSegmentsMetadataManager segmentsMetadataManager;
+ private TestIndexerMetadataStorageCoordinator storageCoordinator;
private OverlordDataSourcesResource dataSourcesResource;
@@ -63,21 +62,17 @@ public class OverlordDataSourcesResourceTest
public void setup()
{
AuditManager auditManager = EasyMock.createStrictMock(AuditManager.class);
- final TestIndexerMetadataStorageCoordinator metadataStorageCoordinator
- = new TestIndexerMetadataStorageCoordinator();
- segmentsMetadataManager = metadataStorageCoordinator.getSegmentsMetadataManager();
+ storageCoordinator = new TestIndexerMetadataStorageCoordinator();
TaskMaster taskMaster = new TaskMaster(null, null);
dataSourcesResource = new OverlordDataSourcesResource(
taskMaster,
- segmentsMetadataManager,
- metadataStorageCoordinator,
+ storageCoordinator,
auditManager
);
taskMaster.becomeFullLeader();
- WIKI_SEGMENTS_10X1D.forEach(segmentsMetadataManager::addSegment);
- metadataStorageCoordinator.commitSegments(Set.copyOf(WIKI_SEGMENTS_10X1D), null);
+ storageCoordinator.commitSegments(Set.copyOf(WIKI_SEGMENTS_10X1D), null);
}
@Test
@@ -95,11 +90,11 @@ public class OverlordDataSourcesResourceTest
{
Response response = dataSourcesResource.markSegmentAsUnused(
TestDataSource.WIKI,
- "someSegment"
+ "some<script>Segment"
);
Assert.assertEquals(400, response.getStatus());
Assert.assertEquals(
- "Could not parse Segment ID[someSegment] for DataSource[wiki]",
+ "Could not parse Segment ID[some<script>Segment] for DataSource[wiki]",
response.getEntity()
);
}
@@ -194,6 +189,20 @@ public class OverlordDataSourcesResourceTest
verifyNumSegmentsUpdated(1, response);
}
+ @Test
+ public void testMarkSegmentAsUsed_withInvalidSegmentId()
+ {
+ Response response = dataSourcesResource.markSegmentAsUsed(
+ TestDataSource.WIKI,
+ "some<script>Segment"
+ );
+ Assert.assertEquals(400, response.getStatus());
+ Assert.assertEquals(
+ "Could not parse Segment ID[some<script>Segment] for DataSource[wiki]",
+ response.getEntity()
+ );
+ }
+
@Test
public void testMarkAllNonOvershadowedSegmentsAsUsed()
{
@@ -203,7 +212,7 @@ public class OverlordDataSourcesResourceTest
segment -> DataSegment.builder(segment).version(version2).build()
).collect(Collectors.toList());
- wikiSegmentsV2.forEach(segmentsMetadataManager::addSegment);
+ storageCoordinator.commitSegments(Set.copyOf(wikiSegmentsV2), null);
// Mark all segments as unused
Response response = dataSourcesResource.markAllSegmentsAsUnused(
@@ -219,11 +228,8 @@ public class OverlordDataSourcesResourceTest
);
verifyNumSegmentsUpdated(10, response);
- final ImmutableDruidDataSource dataSource = segmentsMetadataManager
- .getImmutableDataSourceWithUsedSegments(TestDataSource.WIKI);
- Assert.assertNotNull(dataSource);
-
- final Collection<DataSegment> usedSegments = dataSource.getSegments();
+ final Collection<DataSegment> usedSegments =
+ storageCoordinator.retrieveAllUsedSegments(TestDataSource.WIKI, Segments.INCLUDING_OVERSHADOWED);
Assert.assertEquals(10, usedSegments.size());
for (DataSegment segment : usedSegments) {
Assert.assertEquals(version2, segment.getVersion());
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java
index e0332708c3f..485b6699d1b 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/seekablestream/SeekableStreamIndexTaskTestBase.java
@@ -47,6 +47,7 @@ import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.discovery.DataNodeService;
import org.apache.druid.discovery.DruidNodeAnnouncer;
import org.apache.druid.discovery.LookupNodeService;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.error.DruidException;
import org.apache.druid.indexer.TaskStatus;
import org.apache.druid.indexer.granularity.UniformGranularitySpec;
@@ -598,6 +599,7 @@ public abstract class SeekableStreamIndexTaskTestBase extends EasyMockSupport
derby.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
),
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java
index c7c2bb5ad89..b67bd48856e 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/test/TestIndexerMetadataStorageCoordinator.java
@@ -30,9 +30,11 @@ import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.metadata.PendingSegmentRecord;
import org.apache.druid.metadata.ReplaceTaskLock;
+import org.apache.druid.metadata.SortOrder;
import org.apache.druid.segment.SegmentSchemaMapping;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
import org.apache.druid.server.coordinator.simulate.TestSegmentsMetadataManager;
+import org.apache.druid.server.http.DataSegmentPlus;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.SegmentTimeline;
@@ -54,11 +56,6 @@ public class TestIndexerMetadataStorageCoordinator implements IndexerMetadataSto
private int deleteSegmentsCount = 0;
- public TestSegmentsMetadataManager getSegmentsMetadataManager()
- {
- return segmentsMetadataManager;
- }
-
@Override
public DataSourceMetadata retrieveDataSourceMetadata(String dataSource)
{
@@ -84,11 +81,19 @@ public class TestIndexerMetadataStorageCoordinator implements IndexerMetadataSto
}
@Override
- public Set<DataSegment> retrieveAllUsedSegments(String dataSource, Segments visibility)
+ public Set<String> retrieveAllDatasourceNames()
{
return Set.of();
}
+ @Override
+ public Set<DataSegment> retrieveAllUsedSegments(String dataSource, Segments visibility)
+ {
+ return Set.copyOf(
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSource(dataSource).getSegments()
+ );
+ }
+
@Override
public List<Pair<DataSegment, String>> retrieveUsedSegmentsAndCreatedDates(String dataSource, List<Interval> intervals)
{
@@ -341,6 +346,58 @@ public class TestIndexerMetadataStorageCoordinator implements IndexerMetadataSto
return Map.of();
}
+ @Override
+ public List<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
+ String datasource,
+ @Nullable Interval interval,
+ @Nullable Integer limit,
+ @Nullable String lastSegmentId,
+ @Nullable SortOrder sortOrder
+ )
+ {
+ return List.of();
+ }
+
+ @Override
+ public List<Interval> getUnusedSegmentIntervals(
+ String dataSource,
+ @Nullable DateTime minStartTime,
+ DateTime maxEndTime,
+ int limit,
+ DateTime maxUsedStatusLastUpdatedTime
+ )
+ {
+ return List.of();
+ }
+
+ @Override
+ public int markAllNonOvershadowedSegmentsAsUsed(String dataSource)
+ {
+ return segmentsMetadataManager.markAsUsedAllNonOvershadowedSegmentsInDataSource(dataSource);
+ }
+
+ @Override
+ public int markNonOvershadowedSegmentsAsUsed(
+ String dataSource,
+ Interval interval,
+ @Nullable List<String> versions
+ )
+ {
+ return segmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(dataSource, interval, versions);
+ }
+
+ @Override
+ public int markNonOvershadowedSegmentsAsUsed(String dataSource, Set<SegmentId> segmentIds)
+ {
+ return segmentsMetadataManager.markAsUsedNonOvershadowedSegments(dataSource, segmentIds);
+ }
+
+ @Override
+ public boolean markSegmentAsUsed(SegmentId segmentId)
+ {
+ return segmentsMetadataManager.markSegmentAsUsed(segmentId.toString());
+ }
+
@Override
public SegmentTimeline getSegmentTimelineForAllocation(
String dataSource,
diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env
index 7a0b2ae3995..038ecd45d0d 100644
--- a/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env
+++ b/integration-tests-ex/cases/cluster/Common/environment-configs/coordinator.env
@@ -29,6 +29,7 @@ druid_manager_lookups_threadPoolSize=2
druid_manager_config_pollDuration=PT10S
druid_manager_rules_pollDuration=PT10S
druid_manager_segments_pollDuration=PT2S
+druid_manager_segments_useIncrementalCache=always
druid_auth_basic_common_cacheDirectory=/tmp/authCache/coordinator
druid_auth_unsecuredPaths=["/druid/coordinator/v1/loadqueue"]
druid_server_https_crlPath=/tls/revocations.crl
diff --git a/integration-tests-ex/cases/cluster/Common/environment-configs/overlord.env b/integration-tests-ex/cases/cluster/Common/environment-configs/overlord.env
index 5e8b6a809cc..ddeaaf570f0 100644
--- a/integration-tests-ex/cases/cluster/Common/environment-configs/overlord.env
+++ b/integration-tests-ex/cases/cluster/Common/environment-configs/overlord.env
@@ -29,5 +29,5 @@ druid_indexer_runner_type=remote
druid_auth_basic_common_cacheDirectory=/tmp/authCache/overlord
druid_server_https_crlPath=/tls/revocations.crl
-druid_manager_segments_pollDuration=PT5S
-druid_manager_segments_useCache=always
+druid_manager_segments_pollDuration=PT2S
+druid_manager_segments_useIncrementalCache=always
diff --git a/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml b/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml
index 090a746750f..60cff61472d 100644
--- a/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml
+++ b/integration-tests/docker/docker-compose.cds-coordinator-metadata-query-disabled.yml
@@ -40,6 +40,7 @@ services:
- druid_centralizedDatasourceSchema_backFillPeriod=15000
- druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S
- druid_coordinator_segmentMetadata_disableSegmentMetadataQueries=true
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-overlord
- druid-metadata-storage
@@ -52,6 +53,7 @@ services:
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_centralizedDatasourceSchema_enabled=true
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-metadata-storage
- druid-zookeeper-kafka
diff --git a/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml b/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml
index 190f4eaf8bc..f7923f368d8 100644
--- a/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml
+++ b/integration-tests/docker/docker-compose.cds-task-schema-publish-disabled.yml
@@ -40,6 +40,7 @@ services:
- druid_centralizedDatasourceSchema_backFillEnabled=true
- druid_centralizedDatasourceSchema_backFillPeriod=15000
- druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-overlord
- druid-metadata-storage
@@ -53,6 +54,7 @@ services:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_centralizedDatasourceSchema_enabled=true
- druid_centralizedDatasourceSchema_taskSchemaPublishDisabled=true
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-metadata-storage
- druid-zookeeper-kafka
diff --git a/integration-tests/docker/docker-compose.centralized-datasource-schema.yml b/integration-tests/docker/docker-compose.centralized-datasource-schema.yml
index e89e49bc132..8674a562012 100644
--- a/integration-tests/docker/docker-compose.centralized-datasource-schema.yml
+++ b/integration-tests/docker/docker-compose.centralized-datasource-schema.yml
@@ -39,6 +39,7 @@ services:
- druid_centralizedDatasourceSchema_backFillEnabled=true
- druid_centralizedDatasourceSchema_backFillPeriod=15000
- druid_coordinator_segmentMetadata_metadataRefreshPeriod=PT15S
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-overlord
- druid-metadata-storage
@@ -51,6 +52,7 @@ services:
environment:
- DRUID_INTEGRATION_TEST_GROUP=${DRUID_INTEGRATION_TEST_GROUP}
- druid_centralizedDatasourceSchema_enabled=true
+ - druid_manager_segments_useIncrementalCache=never
depends_on:
- druid-metadata-storage
- druid-zookeeper-kafka
diff --git a/integration-tests/docker/environment-configs/coordinator b/integration-tests/docker/environment-configs/coordinator
index c9226e1da69..28098c6d14d 100644
--- a/integration-tests/docker/environment-configs/coordinator
+++ b/integration-tests/docker/environment-configs/coordinator
@@ -36,6 +36,7 @@ druid_manager_lookups_threadPoolSize=2
druid_manager_config_pollDuration=PT10S
druid_manager_rules_pollDuration=PT10S
druid_manager_segments_pollDuration=PT2S
+druid_manager_segments_useIncrementalCache=always
druid_auth_basic_common_cacheDirectory=/tmp/authCache/coordinator
druid_auth_unsecuredPaths=["/druid/coordinator/v1/loadqueue"]
druid_server_https_crlPath=/tls/revocations.crl
diff --git a/integration-tests/docker/environment-configs/overlord b/integration-tests/docker/environment-configs/overlord
index df351c73d27..0c06cb757a6 100644
--- a/integration-tests/docker/environment-configs/overlord
+++ b/integration-tests/docker/environment-configs/overlord
@@ -33,5 +33,5 @@ druid_indexer_storage_type=metadata
druid_indexer_runner_type=remote
druid_auth_basic_common_cacheDirectory=/tmp/authCache/overlord
druid_server_https_crlPath=/tls/revocations.crl
-druid_manager_segments_pollDuration=PT5S
-druid_manager_segments_useCache=always
+druid_manager_segments_pollDuration=PT2S
+druid_manager_segments_useIncrementalCache=always
diff --git a/processing/src/main/java/org/apache/druid/java/util/common/StringUtils.java b/processing/src/main/java/org/apache/druid/java/util/common/StringUtils.java
index 4c3f334636f..416754576b3 100644
--- a/processing/src/main/java/org/apache/druid/java/util/common/StringUtils.java
+++ b/processing/src/main/java/org/apache/druid/java/util/common/StringUtils.java
@@ -22,6 +22,7 @@ package org.apache.druid.java.util.common;
import com.google.common.base.Strings;
import io.netty.util.SuppressForbidden;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.text.StringEscapeUtils;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -771,4 +772,13 @@ public class StringUtils
{
return str == null ? null : StringUtils.replace(str, "'", "''");
}
+
+ /**
+ * Uses {@link StringEscapeUtils#escapeHtml4} to escape the HTML entities in
+ * the given string. This method can be used to sanitize a string for XSS.
+ */
+ public static String escapeHtml(String value)
+ {
+ return StringEscapeUtils.escapeHtml4(value);
+ }
}
diff --git a/processing/src/test/java/org/apache/druid/common/utils/StringUtilsTest.java b/processing/src/test/java/org/apache/druid/common/utils/StringUtilsTest.java
index 787d5daa79d..8dd726c230c 100644
--- a/processing/src/test/java/org/apache/druid/common/utils/StringUtilsTest.java
+++ b/processing/src/test/java/org/apache/druid/common/utils/StringUtilsTest.java
@@ -87,4 +87,10 @@ public class StringUtilsTest
Assert.assertEquals(ByteBuffer.wrap(StringUtils.toUtf8("foo")), StringUtils.toUtf8ByteBuffer("foo"));
Assert.assertEquals(ByteBuffer.wrap(StringUtils.toUtf8("🙂")), StringUtils.toUtf8ByteBuffer("🙂"));
}
+
+ @Test
+ public void testEscapeHtml()
+ {
+ Assert.assertEquals("invalid<script>SegmentId", StringUtils.escapeHtml("invalid<script>SegmentId"));
+ }
}
diff --git a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
index 27f16fcddb1..574e1ec76ac 100644
--- a/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
+++ b/server/src/main/java/org/apache/druid/client/DataSourcesSnapshot.java
@@ -22,12 +22,14 @@ package org.apache.druid.client;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
+import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.Partitions;
import org.apache.druid.timeline.SegmentTimeline;
import org.apache.druid.timeline.VersionedIntervalTimeline;
import org.apache.druid.utils.CollectionUtils;
import org.joda.time.DateTime;
+import org.joda.time.Interval;
import javax.annotation.Nullable;
import java.util.ArrayList;
@@ -35,10 +37,11 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* An immutable snapshot of metadata information about used segments and overshadowed segments, coming from
- * {@link SqlSegmentsMetadataManager}.
+ * {@link SegmentsMetadataManager}.
*/
public class DataSourcesSnapshot
{
@@ -47,6 +50,28 @@ public class DataSourcesSnapshot
return fromUsedSegments(segments, DateTimes.nowUtc());
}
+ /**
+ * Creates a snapshot of all "used" segments that existed in the database at
+ * the {@code snapshotTime}.
+ */
+ public static DataSourcesSnapshot fromUsedSegments(
+ Map<String, Set<DataSegment>> datasourceToUsedSegments,
+ DateTime snapshotTime
+ )
+ {
+ final Map<String, String> properties = Map.of("created", snapshotTime.toString());
+
+ final Map<String, ImmutableDruidDataSource> dataSources = new HashMap<>();
+ datasourceToUsedSegments.forEach(
+ (dataSource, segments) -> dataSources.put(
+ dataSource,
+ new ImmutableDruidDataSource(dataSource, properties, segments)
+ )
+ );
+
+ return new DataSourcesSnapshot(snapshotTime, dataSources);
+ }
+
/**
* Creates a snapshot of all "used" segments that existed in the database at
* the {@code snapshotTime}.
@@ -120,6 +145,23 @@ public class DataSourcesSnapshot
return overshadowedSegments;
}
+ /**
+ * Gets all the used segments for the datasource that overlap with the given
+ * interval and are not completely overshadowed.
+ */
+ public Set<DataSegment> getAllUsedNonOvershadowedSegments(
+ String dataSource,
+ Interval interval
+ )
+ {
+ final SegmentTimeline timeline = usedSegmentsTimelinesPerDataSource.get(dataSource);
+ if (timeline == null) {
+ return Set.of();
+ }
+
+ return timeline.findNonOvershadowedObjectsInInterval(interval, Partitions.ONLY_COMPLETE);
+ }
+
/**
* Returns an iterable to go over all used segments in all data sources. The order in which segments are iterated
* is unspecified.
diff --git a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java
index 7cb48c694ef..5f8197d31c3 100644
--- a/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java
+++ b/server/src/main/java/org/apache/druid/guice/SQLMetadataStorageDruidModule.java
@@ -37,7 +37,6 @@ import org.apache.druid.metadata.SQLMetadataRuleManagerProvider;
import org.apache.druid.metadata.SQLMetadataSupervisorManager;
import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerProvider;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.SqlSegmentsMetadataManagerProvider;
import org.apache.druid.metadata.segment.SegmentMetadataTransactionFactory;
import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
@@ -85,11 +84,6 @@ public class SQLMetadataStorageDruidModule implements Module
@Override
public void configure(Binder binder)
{
- PolyBind.optionBinder(binder, Key.get(SegmentsMetadataManager.class))
- .addBinding(type)
- .to(SqlSegmentsMetadataManager.class)
- .in(LazySingleton.class);
-
PolyBind.optionBinder(binder, Key.get(SegmentsMetadataManagerProvider.class))
.addBinding(type)
.to(SqlSegmentsMetadataManagerProvider.class)
@@ -105,8 +99,8 @@ public class SQLMetadataStorageDruidModule implements Module
.to(SQLMetadataRuleManagerProvider.class)
.in(LazySingleton.class);
- // SegmentMetadataCache is bound for all services but is used only by the Overlord
- // Similar to some other classes bound here, such as IndexerSQLMetadataStorageCoordinator
+ // SegmentMetadataCache is bound for all services but is used only by the Overlord and Coordinator
+ // similar to some other classes bound here, such as IndexerSQLMetadataStorageCoordinator
binder.bind(SegmentMetadataCache.class)
.to(HeapMemorySegmentMetadataCache.class)
.in(LazySingleton.class);
diff --git a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java
index 473e462dd51..f84b9350ead 100644
--- a/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java
+++ b/server/src/main/java/org/apache/druid/indexing/overlord/IndexerMetadataStorageCoordinator.java
@@ -22,8 +22,10 @@ package org.apache.druid.indexing.overlord;
import org.apache.druid.java.util.common.Pair;
import org.apache.druid.metadata.PendingSegmentRecord;
import org.apache.druid.metadata.ReplaceTaskLock;
+import org.apache.druid.metadata.SortOrder;
import org.apache.druid.segment.SegmentSchemaMapping;
import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
+import org.apache.druid.server.http.DataSegmentPlus;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.SegmentTimeline;
@@ -44,6 +46,12 @@ import java.util.Set;
*/
public interface IndexerMetadataStorageCoordinator
{
+ /**
+ * @return Set of all datasource names for which there are used or unused
+ * segments present in the metadata store.
+ */
+ Set<String> retrieveAllDatasourceNames();
+
/**
* Retrieves all published segments that have partial or complete overlap with
* the given interval and are marked as used.
@@ -501,4 +509,87 @@ public interface IndexerMetadataStorageCoordinator
* @param segmentIds ids of the first segments which had the corresponding load spec
*/
Map<String, Set<String>> retrieveUpgradedToSegmentIds(String dataSource, Set<String> segmentIds);
+
+ /**
+ * Returns a list of unused segments and their associated metadata for a given datasource over an
+ * optional interval. The order in which segments are iterated is from earliest start-time, with ties being broken
+ * with earliest end-time first. Note: the iteration may not be as trivially cheap as for example, iteration over an
+ * ArrayList. Try (to some reasonable extent) to organize the code so that it iterates the returned iterable only
+ * once rather than several times.
+ *
+ * @param datasource the name of the datasource.
+ * @param interval an optional interval to search over. If none is specified, {@link org.apache.druid.java.util.common.Intervals#ETERNITY}
+ * @param limit an optional maximum number of results to return. If none is specified, the results are not limited.
+ * @param lastSegmentId an optional last segment id from which to search for results. All segments returned are >
+ * this segment lexigraphically if sortOrder is null or {@link SortOrder#ASC}, or < this segment
+ * lexigraphically if sortOrder is {@link SortOrder#DESC}. If none is specified, no such filter is used.
+ * @param sortOrder an optional order with which to return the matching segments by id, start time, end time.
+ * If none is specified, the order of the results is not guarenteed.
+ */
+ List<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
+ String datasource,
+ @Nullable Interval interval,
+ @Nullable Integer limit,
+ @Nullable String lastSegmentId,
+ @Nullable SortOrder sortOrder
+ );
+
+ /**
+ * Returns a list of up to {@code limit} unused segment intervals for the specified datasource. Segments are filtered
+ * based on the following criteria:
+ *
+ * <li> The start time of the segment must be no earlier than the specified {@code minStartTime} (if not null). </li>
+ * <li> The end time of the segment must be no later than the specified {@code maxEndTime}. </li>
+ * <li> The {@code used_status_last_updated} time of the segment must be no later than {@code maxUsedStatusLastUpdatedTime}.
+ * Segments that have no {@code used_status_last_updated} time (due to an upgrade from legacy Druid) will
+ * have {@code maxUsedStatusLastUpdatedTime} ignored. </li>
+ *
+ * @return list of intervals ordered by segment start time and then by end time. Note that the list may contain
+ * duplicate intervals.
+ *
+ */
+ List<Interval> getUnusedSegmentIntervals(
+ String dataSource,
+ @Nullable DateTime minStartTime,
+ DateTime maxEndTime,
+ int limit,
+ DateTime maxUsedStatusLastUpdatedTime
+ );
+
+ /**
+ * Returns the number of segment entries in the database whose state was changed as the result of this call (that is,
+ * the segments were marked as used). If the call results in a database error, an exception is relayed to the caller.
+ *
+ * @return Number of segments updated in the metadata store
+ */
+ int markAllNonOvershadowedSegmentsAsUsed(String dataSource);
+
+ /**
+ * Marks non-overshadowed unused segments for the given interval and optional list of versions
+ * as used. If versions are not specified, all versions of non-overshadowed unused segments in the interval
+ * will be marked as used. If an empty list of versions is passed, no segments are marked as used.
+ *
+ * @return Number of segments updated in the metadata store
+ */
+ int markNonOvershadowedSegmentsAsUsed(String dataSource, Interval interval, @Nullable List<String> versions);
+
+ /**
+ * Marks the given segment IDs as "used" only if there are not already overshadowed
+ * by other used segments. Qualifying segment IDs that are already marked as
+ * "used" are not updated.
+ *
+ * @return Number of segments updated
+ * @throws org.apache.druid.error.DruidException of category INVALID_INPUT if
+ * any of the given segment IDs
+ * do not exist in the metadata store.
+ */
+ int markNonOvershadowedSegmentsAsUsed(String dataSource, Set<SegmentId> segmentIds);
+
+ /**
+ * Returns true if the state of the segment entry is changed in the database as the result of this call (that is, the
+ * segment was marked as used), false otherwise. If the call results in a database error, an exception is relayed to
+ * the caller.
+ */
+ boolean markSegmentAsUsed(SegmentId segmentId);
+
}
diff --git a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
index 2f35c9f45d4..8f448168359 100644
--- a/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
+++ b/server/src/main/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinator.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
import com.google.common.collect.FluentIterable;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -91,6 +92,7 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -146,6 +148,17 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor
connector.createUpgradeSegmentsTable();
}
+ @Override
+ public Set<String> retrieveAllDatasourceNames()
+ {
+ final String sql = StringUtils.format("SELECT DISTINCT(dataSource) FROM %s", dbTables.getSegmentsTable());
+ return Set.copyOf(
+ connector.inReadOnlyTransaction(
+ (handle, status) -> handle.createQuery(sql).mapTo(String.class).list()
+ )
+ );
+ }
+
@Override
public Set<DataSegment> retrieveUsedSegmentsForIntervals(
final String dataSource,
@@ -282,6 +295,76 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor
);
}
+ @Override
+ public boolean markSegmentAsUsed(SegmentId segmentId)
+ {
+ return inWriteTransaction(
+ sql -> sql.markSegmentAsUsed(segmentId, DateTimes.nowUtc())
+ );
+ }
+
+ @Override
+ public int markNonOvershadowedSegmentsAsUsed(String dataSource, Set<SegmentId> segmentIds)
+ {
+ return inWriteTransaction(
+ sql -> sql.markNonOvershadowedSegmentsAsUsed(dataSource, segmentIds, DateTimes.nowUtc())
+ );
+ }
+
+ @Override
+ public int markNonOvershadowedSegmentsAsUsed(
+ String dataSource,
+ Interval interval,
+ @Nullable List<String> versions
+ )
+ {
+ return inWriteTransaction(
+ sql -> sql.markNonOvershadowedSegmentsAsUsed(dataSource, interval, versions, DateTimes.nowUtc())
+ );
+ }
+
+ @Override
+ public int markAllNonOvershadowedSegmentsAsUsed(String dataSource)
+ {
+ return inWriteTransaction(
+ sql -> sql.markAllNonOvershadowedSegmentsAsUsed(dataSource, DateTimes.nowUtc())
+ );
+ }
+
+ @Override
+ public List<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
+ String datasource,
+ @Nullable Interval interval,
+ @Nullable Integer limit,
+ @Nullable String lastSegmentId,
+ @Nullable SortOrder sortOrder
+ )
+ {
+ return inReadOnlyTransaction(
+ sql -> sql.iterateAllUnusedSegmentsForDatasource(datasource, interval, limit, lastSegmentId, sortOrder)
+ );
+ }
+
+ @Override
+ public List<Interval> getUnusedSegmentIntervals(
+ String dataSource,
+ @Nullable DateTime minStartTime,
+ DateTime maxEndTime,
+ int limit,
+ DateTime maxUsedStatusLastUpdatedTime
+ )
+ {
+ return inReadOnlyTransaction(
+ sql -> sql.retrieveUnusedSegmentIntervals(
+ dataSource,
+ minStartTime,
+ maxEndTime,
+ limit,
+ maxUsedStatusLastUpdatedTime
+ )
+ );
+ }
+
private SegmentTimeline getTimelineForIntervals(
final SegmentMetadataReadTransaction transaction,
final List<Interval> intervals
@@ -2492,6 +2575,10 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor
return upgradedToSegmentIds;
}
+ /**
+ * Performs a read-write transaction using the {@link SegmentMetadataTransactionFactory},
+ * which may use the segment metadata cache, if enabled and ready.
+ */
private <T> T inReadWriteDatasourceTransaction(
String dataSource,
SegmentMetadataTransaction.Callback<T> callback
@@ -2500,6 +2587,10 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor
return transactionFactory.inReadWriteDatasourceTransaction(dataSource, callback);
}
+ /**
+ * Performs a read-only transaction using the {@link SegmentMetadataTransactionFactory},
+ * which may use the segment metadata cache, if enabled and ready.
+ */
private <T> T inReadOnlyDatasourceTransaction(
String dataSource,
SegmentMetadataReadTransaction.Callback<T> callback
@@ -2507,4 +2598,52 @@ public class IndexerSQLMetadataStorageCoordinator implements IndexerMetadataStor
{
return transactionFactory.inReadOnlyDatasourceTransaction(dataSource, callback);
}
+
+ /**
+ * Performs a read-only transaction using the {@link SqlSegmentsMetadataQuery},
+ * which queries the metadata store directly.
+ */
+ private <T> T inReadOnlyTransaction(Function<SqlSegmentsMetadataQuery, T> sqlQuery)
+ {
+ try {
+ return connector.retryReadOnlyTransaction(
+ (handle, status) -> sqlQuery.apply(
+ SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables, jsonMapper)
+ ),
+ 2, 3
+ );
+ }
+ catch (Throwable t) {
+ Throwable rootCause = Throwables.getRootCause(t);
+ if (rootCause instanceof DruidException) {
+ throw (DruidException) rootCause;
+ } else {
+ throw t;
+ }
+ }
+ }
+
+ /**
+ * Performs a write transaction using the {@link SqlSegmentsMetadataQuery},
+ * which updates the metadata store directly.
+ */
+ private <T> T inWriteTransaction(Function<SqlSegmentsMetadataQuery, T> sqlUpdate)
+ {
+ try {
+ return connector.retryTransaction(
+ (handle, status) -> sqlUpdate.apply(
+ SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables, jsonMapper)
+ ),
+ 2, 3
+ );
+ }
+ catch (Throwable t) {
+ Throwable rootCause = Throwables.getRootCause(t);
+ if (rootCause instanceof DruidException) {
+ throw (DruidException) rootCause;
+ } else {
+ throw t;
+ }
+ }
+ }
}
diff --git a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java
index 847b1eba6fd..dc1bde217d1 100644
--- a/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SQLMetadataRuleManager.java
@@ -143,7 +143,7 @@ public class SQLMetadataRuleManager implements MetadataRuleManager
* the theoretical situation of two tasks scheduled in {@link #start()} calling {@link #poll()} concurrently, if
* the sequence of {@link #start()} - {@link #stop()} - {@link #start()} actions occurs quickly.
*
- * {@link SqlSegmentsMetadataManager} also have a similar issue.
+ * {@link SegmentsMetadataManager} also has a similar issue.
*/
private long currentStartOrder = -1;
private ScheduledExecutorService exec = null;
diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
index 97a77d5208a..8983364de77 100644
--- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManager.java
@@ -19,192 +19,58 @@
package org.apache.druid.metadata;
-import com.google.common.base.Optional;
import org.apache.druid.client.DataSourcesSnapshot;
-import org.apache.druid.client.ImmutableDruidDataSource;
-import org.apache.druid.server.http.DataSegmentPlus;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.SegmentId;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
-
-import javax.annotation.Nullable;
-import java.util.Collection;
-import java.util.List;
-import java.util.Set;
/**
- * The difference between this class and org.apache.druid.sql.calcite.schema.MetadataSegmentView is that this class
- * resides in Coordinator's memory, while org.apache.druid.sql.calcite.schema.MetadataSegmentView resides in Broker's
- * memory.
+ * Polls the metadata store periodically and builds a timeline of used segments
+ * (and schemas if schema caching on the Coordinator is enabled).
+ * <p>
+ * This class is provisioned by {@link SegmentsMetadataManagerProvider} and must
+ * be bound on the Coordinator/Overlord accordingly.
*/
public interface SegmentsMetadataManager
{
- void startPollingDatabasePeriodically();
-
- void stopPollingDatabasePeriodically();
-
- boolean isPollingDatabasePeriodically();
-
/**
- * Returns the number of segment entries in the database whose state was changed as the result of this call (that is,
- * the segments were marked as used). If the call results in a database error, an exception is relayed to the caller.
+ * Initializes the manager when the service is being started.
*/
- int markAsUsedAllNonOvershadowedSegmentsInDataSource(String dataSource);
+ void start();
/**
- * Marks non-overshadowed unused segments for the given interval and optional list of versions
- * as used. If versions are not specified, all versions of non-overshadowed unused segments in the interval
- * will be marked as used. If an empty list of versions is passed, no segments are marked as used.
- * @return Number of segments updated
- */
- int markAsUsedNonOvershadowedSegmentsInInterval(String dataSource, Interval interval, @Nullable List<String> versions);
-
- /**
- * Marks the given segment IDs as "used" only if there are not already overshadowed
- * by other used segments. Qualifying segment IDs that are already marked as
- * "used" are not updated.
- *
- * @return Number of segments updated
- * @throws org.apache.druid.error.DruidException of category INVALID_INPUT if
- * any of the given segment IDs
- * do not exist in the metadata store.
+ * Cleans up resources when the service is being shut down.
*/
- int markAsUsedNonOvershadowedSegments(String dataSource, Set<SegmentId> segmentIds);
-
- /**
- * Returns true if the state of the segment entry is changed in the database as the result of this call (that is, the
- * segment was marked as used), false otherwise. If the call results in a database error, an exception is relayed to
- * the caller.
- */
- boolean markSegmentAsUsed(String segmentId);
-
- /**
- * Returns the number of segment entries in the database whose state was changed as the result of this call (that is,
- * the segments were marked as unused). If the call results in a database error, an exception is relayed to the
- * caller.
- */
- int markAsUnusedAllSegmentsInDataSource(String dataSource);
-
- /**
- * Marks segments as unused that are <b>fully contained</b> in the given interval for an optional list of versions.
- * If versions are not specified, all versions of segments in the interval will be marked as unused. If an empty list
- * of versions is passed, no segments are marked as unused.
- * Segments that are already marked as unused are not updated.
- * @return The number of segments updated
- */
- int markAsUnusedSegmentsInInterval(String dataSource, Interval interval, @Nullable List<String> versions);
-
- int markSegmentsAsUnused(Set<SegmentId> segmentIds);
+ void stop();
/**
- * Returns true if the state of the segment entry is changed in the database as the result of this call (that is, the
- * segment was marked as unused), false otherwise. If the call results in a database error, an exception is relayed to
- * the caller.
+ * Starts polling segments from the metadata store upon becoming leader.
*/
- boolean markSegmentAsUnused(SegmentId segmentId);
-
- /**
- * If there are used segments belonging to the given data source this method returns them as an {@link
- * ImmutableDruidDataSource} object. If there are no used segments belonging to the given data source this method
- * returns null.
- */
- @Nullable ImmutableDruidDataSource getImmutableDataSourceWithUsedSegments(String dataSource);
-
- /**
- * Returns a set of {@link ImmutableDruidDataSource} objects containing information about all used segments. {@link
- * ImmutableDruidDataSource} objects in the returned collection are unique. If there are no used segments, this method
- * returns an empty collection.
- */
- Collection<ImmutableDruidDataSource> getImmutableDataSourcesWithAllUsedSegments();
-
- /**
- * Returns a snapshot of DruidDataSources and overshadowed segments
- */
- DataSourcesSnapshot getSnapshotOfDataSourcesWithAllUsedSegments();
-
- /**
- * Returns an iterable to go over all segments in all data sources. The order in which segments are iterated is
- * unspecified. Note: the iteration may not be as trivially cheap as, for example, iteration over an ArrayList. Try
- * (to some reasonable extent) to organize the code so that it iterates the returned iterable only once rather than
- * several times.
- */
- Iterable<DataSegment> iterateAllUsedSegments();
+ void startPollingDatabasePeriodically();
/**
- * Returns an iterable to go over all used and non-overshadowed segments of given data sources over given interval.
- * The order in which segments are iterated is unspecified. Note: the iteration may not be as trivially cheap as,
- * for example, iteration over an ArrayList. Try (to some reasonable extent) to organize the code so that it
- * iterates the returned iterable only once rather than several times.
- * If {@param requiresLatest} is true then a force metadatastore poll will be triggered. This can cause a longer
- * response time but will ensure that the latest segment information (at the time this method is called) is returned.
- * If {@param requiresLatest} is false then segment information from stale snapshot of up to the last periodic poll
- * period {@link SqlSegmentsMetadataManager#periodicPollDelay} will be used.
+ * Stops polling segments from the metadata store when leadership is lost.
*/
- Optional<Iterable<DataSegment>> iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(
- String datasource,
- Interval interval,
- boolean requiresLatest
- );
+ void stopPollingDatabasePeriodically();
/**
- * Returns an iterable to go over un-used segments and their associated metadata for a given datasource over an
- * optional interval. The order in which segments are iterated is from earliest start-time, with ties being broken
- * with earliest end-time first. Note: the iteration may not be as trivially cheap as for example, iteration over an
- * ArrayList. Try (to some reasonable extent) to organize the code so that it iterates the returned iterable only
- * once rather than several times.
- *
- * @param datasource the name of the datasource.
- * @param interval an optional interval to search over. If none is specified, {@link org.apache.druid.java.util.common.Intervals#ETERNITY}
- * @param limit an optional maximum number of results to return. If none is specified, the results are not limited.
- * @param lastSegmentId an optional last segment id from which to search for results. All segments returned are >
- * this segment lexigraphically if sortOrder is null or {@link SortOrder#ASC}, or < this segment
- * lexigraphically if sortOrder is {@link SortOrder#DESC}. If none is specified, no such filter is used.
- * @param sortOrder an optional order with which to return the matching segments by id, start time, end time.
- * If none is specified, the order of the results is not guarenteed.
+ * @return true if currently the leader and polling the metadata store.
*/
- Iterable<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
- String datasource,
- @Nullable Interval interval,
- @Nullable Integer limit,
- @Nullable String lastSegmentId,
- @Nullable SortOrder sortOrder
- );
+ boolean isPollingDatabasePeriodically();
/**
- * Retrieves all data source names for which there are segment in the database, regardless of whether those segments
- * are used or not. If there are no segments in the database, returns an empty set.
- *
- * Performance warning: this method makes a query into the database.
- *
- * This method might return a different set of data source names than may be observed via {@link
- * #getImmutableDataSourcesWithAllUsedSegments} method. This method will include a data source name even if there
- * are no used segments belonging to it, while {@link #getImmutableDataSourcesWithAllUsedSegments} won't return
- * such a data source.
+ * Returns the latest snapshot containing all used segments currently cached
+ * on the manager. This method returns immediately if the poll period has not
+ * elapsed since the latest snapshot was built. Otherwise, it blocks until
+ * the snapshot has been refreshed.
*/
- Set<String> retrieveAllDataSourceNames();
+ DataSourcesSnapshot getRecentDataSourcesSnapshot();
/**
- * Returns a list of up to {@code limit} unused segment intervals for the specified datasource. Segments are filtered
- * based on the following criteria:
- *
- * <li> The start time of the segment must be no earlier than the specified {@code minStartTime} (if not null). </li>
- * <li> The end time of the segment must be no later than the specified {@code maxEndTime}. </li>
- * <li> The {@code used_status_last_updated} time of the segment must be no later than {@code maxUsedStatusLastUpdatedTime}.
- * Segments that have no {@code used_status_last_updated} time (due to an upgrade from legacy Druid) will
- * have {@code maxUsedStatusLastUpdatedTime} ignored. </li>
- *
- * @return list of intervals ordered by segment start time and then by end time. Note that the list may contain
- * duplicate intervals.
+ * Forces the manager to poll the metadata store and update its snapshot.
+ * If a poll is already in progress, a new poll is not started. This method
+ * blocks until the poll finishes and the snapshot has been refreshed.
*
+ * @return The updated snapshot.
*/
- List<Interval> getUnusedSegmentIntervals(
- String dataSource,
- @Nullable DateTime minStartTime,
- DateTime maxEndTime,
- int limit,
- DateTime maxUsedStatusLastUpdatedTime
- );
+ DataSourcesSnapshot forceUpdateDataSourcesSnapshot();
/**
* Populates used_status_last_updated column in the segments table iteratively until there are no segments with a NULL
diff --git a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManagerConfig.java b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManagerConfig.java
index 033a703247f..f53f3108221 100644
--- a/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManagerConfig.java
+++ b/server/src/main/java/org/apache/druid/metadata/SegmentsMetadataManagerConfig.java
@@ -37,21 +37,24 @@ public class SegmentsMetadataManagerConfig
private final Period pollDuration;
@JsonProperty
- private final SegmentMetadataCache.UsageMode useCache;
+ private final SegmentMetadataCache.UsageMode useIncrementalCache;
@JsonCreator
public SegmentsMetadataManagerConfig(
@JsonProperty("pollDuration") Period pollDuration,
- @JsonProperty("useCache") SegmentMetadataCache.UsageMode useCache
+ @JsonProperty("useIncrementalCache") SegmentMetadataCache.UsageMode useIncrementalCache
)
{
this.pollDuration = Configs.valueOrDefault(pollDuration, Period.minutes(1));
- this.useCache = Configs.valueOrDefault(useCache, SegmentMetadataCache.UsageMode.NEVER);
+ this.useIncrementalCache = Configs.valueOrDefault(useIncrementalCache, SegmentMetadataCache.UsageMode.NEVER);
}
- public SegmentMetadataCache.UsageMode getCacheMode()
+ /**
+ * Usage mode of the incremental cache.
+ */
+ public SegmentMetadataCache.UsageMode getCacheUsageMode()
{
- return useCache;
+ return useIncrementalCache;
}
public Period getPollDuration()
diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
index 11b00742bc8..402617894b8 100644
--- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
+++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManager.java
@@ -22,34 +22,18 @@ package org.apache.druid.metadata;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import com.google.errorprone.annotations.concurrent.GuardedBy;
-import com.google.inject.Inject;
import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.ImmutableDruidDataSource;
-import org.apache.druid.error.DruidException;
-import org.apache.druid.error.InvalidInput;
-import org.apache.druid.guice.ManageLifecycle;
import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.JodaUtils;
-import org.apache.druid.java.util.common.MapUtils;
-import org.apache.druid.java.util.common.Pair;
import org.apache.druid.java.util.common.Stopwatch;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.concurrent.Execs;
-import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
-import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
-import org.apache.druid.java.util.common.parsers.CloseableIterator;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
@@ -57,21 +41,13 @@ import org.apache.druid.segment.SchemaPayload;
import org.apache.druid.segment.SegmentMetadata;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
import org.apache.druid.segment.metadata.SegmentSchemaCache;
-import org.apache.druid.server.http.DataSegmentPlus;
import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.Partitions;
import org.apache.druid.timeline.SegmentId;
-import org.apache.druid.timeline.SegmentTimeline;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.joda.time.DateTime;
import org.joda.time.Duration;
-import org.joda.time.Interval;
-import org.skife.jdbi.v2.BaseResultSetMapper;
import org.skife.jdbi.v2.Batch;
-import org.skife.jdbi.v2.FoldController;
import org.skife.jdbi.v2.Handle;
-import org.skife.jdbi.v2.Query;
-import org.skife.jdbi.v2.StatementContext;
import org.skife.jdbi.v2.TransactionCallback;
import org.skife.jdbi.v2.TransactionStatus;
@@ -79,14 +55,8 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
-import java.util.Set;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -94,12 +64,11 @@ import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
/**
- *
+ * Implementation of {@link SegmentsMetadataManager}, that periodically polls
+ * used segments from the metadata store to build a {@link DataSourcesSnapshot}.
*/
-@ManageLifecycle
public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
{
private static final EmittingLogger log = new EmittingLogger(SqlSegmentsMetadataManager.class);
@@ -186,7 +155,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
* The latest {@link DatabasePoll} represent {@link #poll()} calls which update {@link #dataSourcesSnapshot}, either
* periodically (see {@link PeriodicDatabasePoll}, {@link #startPollingDatabasePeriodically}, {@link
* #stopPollingDatabasePeriodically}) or "on demand" (see {@link OnDemandDatabasePoll}), when one of the methods that
- * accesses {@link #dataSourcesSnapshot}'s state (such as {@link #getImmutableDataSourceWithUsedSegments}) is
+ * accesses {@link #dataSourcesSnapshot}'s state is
* called when the Coordinator is not the leader and therefore SqlSegmentsMetadataManager isn't polling the database
* periodically.
*
@@ -205,8 +174,8 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
* #dataSourcesSnapshot}-accessing methods should be generally "wait free" for database polls.
*
* The notion and the complexity of "on demand" database polls was introduced to simplify the interface of {@link
- * SegmentsMetadataManager} and guarantee that it always returns consistent and relatively up-to-date data from methods
- * like {@link #getImmutableDataSourceWithUsedSegments}, while avoiding excessive repetitive polls. The last part
+ * SegmentsMetadataManager} and guarantee that it always returns consistent and relatively up-to-date data,
+ * while avoiding excessive repetitive polls. The last part
* is achieved via "hooking on" other polls by awaiting on {@link PeriodicDatabasePoll#firstPollCompletionFuture} or
* {@link OnDemandDatabasePoll#pollCompletionFuture}, see {@link #useLatestIfWithinDelayOrPerformNewDatabasePoll} method
* implementation for details.
@@ -249,7 +218,6 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
private Future<?> usedFlagLastUpdatedPopulationFuture;
- @Inject
public SqlSegmentsMetadataManager(
ObjectMapper jsonMapper,
Supplier<SegmentsMetadataManagerConfig> config,
@@ -270,11 +238,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
this.serviceEmitter = serviceEmitter;
}
- /**
- * Don't confuse this method with {@link #startPollingDatabasePeriodically}. This is a lifecycle starting method to
- * be executed just once for an instance of SqlSegmentsMetadataManager.
- */
- @LifecycleStart
+ @Override
public void start()
{
ReentrantReadWriteLock.WriteLock lock = startStopPollLock.writeLock();
@@ -290,11 +254,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
}
}
- /**
- * Don't confuse this method with {@link #stopPollingDatabasePeriodically}. This is a lifecycle stopping method to
- * be executed just once for an instance of SqlSegmentsMetadataManager.
- */
- @LifecycleStop
+ @Override
public void stop()
{
ReentrantReadWriteLock.WriteLock lock = startStopPollLock.writeLock();
@@ -643,279 +603,25 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
}
@Override
- public boolean markSegmentAsUsed(final String segmentId)
- {
- try {
- int numUpdatedDatabaseEntries = connector.getDBI().withHandle(
- handle -> handle
- .createStatement(StringUtils.format("UPDATE %s SET used=true, used_status_last_updated = :used_status_last_updated WHERE id = :id", getSegmentsTable()))
- .bind("id", segmentId)
- .bind("used_status_last_updated", DateTimes.nowUtc().toString())
- .execute()
- );
- // Unlike bulk markAsUsed methods: markAsUsedAllNonOvershadowedSegmentsInDataSource(),
- // markAsUsedNonOvershadowedSegmentsInInterval(), and markAsUsedNonOvershadowedSegments() we don't put the marked
- // segment into the respective data source, because we don't have it fetched from the database. It's probably not
- // worth complicating the implementation and making two database queries just to add the segment because it will
- // be anyway fetched during the next poll(). Segment putting that is done in the bulk markAsUsed methods is a nice
- // to have thing, but doesn't formally affect the external guarantees of SegmentsMetadataManager class.
- return numUpdatedDatabaseEntries > 0;
- }
- catch (RuntimeException e) {
- log.error(e, "Exception marking segment %s as used", segmentId);
- throw e;
- }
- }
-
- @Override
- public int markAsUsedAllNonOvershadowedSegmentsInDataSource(final String dataSource)
- {
- return doMarkAsUsedNonOvershadowedSegments(dataSource, null, null);
- }
-
- @Override
- public int markAsUsedNonOvershadowedSegmentsInInterval(
- final String dataSource,
- final Interval interval,
- @Nullable final List<String> versions
- )
- {
- Preconditions.checkNotNull(interval);
- return doMarkAsUsedNonOvershadowedSegments(dataSource, interval, versions);
- }
-
- private int doMarkAsUsedNonOvershadowedSegments(
- final String dataSourceName,
- final @Nullable Interval interval,
- final @Nullable List<String> versions
- )
+ public DataSourcesSnapshot getRecentDataSourcesSnapshot()
{
- final List<DataSegment> unusedSegments = new ArrayList<>();
- final SegmentTimeline timeline = new SegmentTimeline();
-
- connector.inReadOnlyTransaction(
- (handle, status) -> {
- final SqlSegmentsMetadataQuery queryTool =
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper);
-
- final List<Interval> intervals =
- interval == null ? Intervals.ONLY_ETERNITY : Collections.singletonList(interval);
-
- try (final CloseableIterator<DataSegment> iterator =
- queryTool.retrieveUsedSegments(dataSourceName, intervals, versions)) {
- timeline.addSegments(iterator);
- }
-
- try (final CloseableIterator<DataSegment> iterator =
- queryTool.retrieveUnusedSegments(dataSourceName, intervals, versions, null, null, null, null)) {
- while (iterator.hasNext()) {
- final DataSegment dataSegment = iterator.next();
- timeline.addSegments(Iterators.singletonIterator(dataSegment));
- unusedSegments.add(dataSegment);
- }
- }
-
- //noinspection ReturnOfNull: This consumer operates by side effects
- return null;
- }
- );
-
- return markNonOvershadowedSegmentsAsUsed(unusedSegments, timeline);
- }
-
- private int markNonOvershadowedSegmentsAsUsed(
- List<DataSegment> unusedSegments,
- SegmentTimeline timeline
- )
- {
- List<SegmentId> segmentIdsToMarkAsUsed = new ArrayList<>();
- for (DataSegment segment : unusedSegments) {
- if (!timeline.isOvershadowed(segment)) {
- segmentIdsToMarkAsUsed.add(segment.getId());
- }
- }
-
- return markSegmentsAsUsed(segmentIdsToMarkAsUsed);
- }
-
- @Override
- public int markAsUsedNonOvershadowedSegments(final String dataSource, final Set<SegmentId> segmentIds)
- {
- try {
- Pair<List<DataSegment>, SegmentTimeline> unusedSegmentsAndTimeline = connector
- .inReadOnlyTransaction(
- (handle, status) -> {
- List<DataSegment> unusedSegments = retrieveUnusedSegments(dataSource, segmentIds, handle);
- List<Interval> unusedSegmentsIntervals = JodaUtils.condenseIntervals(
- unusedSegments.stream().map(DataSegment::getInterval).collect(Collectors.toList())
- );
- try (CloseableIterator<DataSegment> usedSegmentsOverlappingUnusedSegmentsIntervals =
- retrieveUsedSegmentsOverlappingIntervals(dataSource, unusedSegmentsIntervals, handle)) {
- SegmentTimeline timeline = SegmentTimeline.forSegments(
- Iterators.concat(usedSegmentsOverlappingUnusedSegmentsIntervals, unusedSegments.iterator())
- );
- return new Pair<>(unusedSegments, timeline);
- }
- }
- );
-
- List<DataSegment> unusedSegments = unusedSegmentsAndTimeline.lhs;
- SegmentTimeline timeline = unusedSegmentsAndTimeline.rhs;
- return markNonOvershadowedSegmentsAsUsed(unusedSegments, timeline);
- }
- catch (Exception e) {
- Throwable rootCause = Throwables.getRootCause(e);
- if (rootCause instanceof DruidException) {
- throw (DruidException) rootCause;
- } else {
- throw e;
- }
- }
- }
-
- private List<DataSegment> retrieveUnusedSegments(
- final String dataSource,
- final Set<SegmentId> segmentIds,
- final Handle handle
- )
- {
- final List<DataSegmentPlus> retrievedSegments = SqlSegmentsMetadataQuery
- .forHandle(handle, connector, dbTables.get(), jsonMapper)
- .retrieveSegmentsById(dataSource, segmentIds);
-
- final Set<SegmentId> unknownSegmentIds = new HashSet<>(segmentIds);
- final List<DataSegment> unusedSegments = new ArrayList<>();
- for (DataSegmentPlus entry : retrievedSegments) {
- final DataSegment segment = entry.getDataSegment();
- unknownSegmentIds.remove(segment.getId());
- if (Boolean.FALSE.equals(entry.getUsed())) {
- unusedSegments.add(segment);
- }
- }
-
- if (!unknownSegmentIds.isEmpty()) {
- throw InvalidInput.exception(
- "Could not find segment IDs[%s] for datasource[%s]",
- unknownSegmentIds, dataSource
- );
- }
-
- return unusedSegments;
- }
-
- private CloseableIterator<DataSegment> retrieveUsedSegmentsOverlappingIntervals(
- final String dataSource,
- final Collection<Interval> intervals,
- final Handle handle
- )
- {
- return SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .retrieveUsedSegments(dataSource, intervals);
- }
-
- private int markSegmentsAsUsed(final List<SegmentId> segmentIds)
- {
- if (segmentIds.isEmpty()) {
- log.info("No segments found to mark as used.");
- return 0;
- }
-
- return connector.getDBI().withHandle(
- handle ->
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .markSegments(segmentIds, true, DateTimes.nowUtc())
- );
+ useLatestIfWithinDelayOrPerformNewDatabasePoll();
+ return dataSourcesSnapshot;
}
@Override
- public int markAsUnusedAllSegmentsInDataSource(final String dataSource)
+ public DataSourcesSnapshot forceUpdateDataSourcesSnapshot()
{
- try {
- return connector.getDBI().withHandle(
- handle ->
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .markSegmentsUnused(dataSource, Intervals.ETERNITY, DateTimes.nowUtc())
- );
- }
- catch (RuntimeException e) {
- log.error(e, "Exception marking all segments as unused in data source [%s]", dataSource);
- throw e;
- }
+ forceOrWaitOngoingDatabasePoll();
+ return dataSourcesSnapshot;
}
/**
- * This method does not update {@link #dataSourcesSnapshot}, see the comments in {@link #doPoll()} about
- * snapshot update. The update of the segment's state will be reflected after the next {@link DatabasePoll}.
+ * Returns the last snapshot built by the manager. This method always returns
+ * immediately, even if the last snapshot is older than the poll period.
*/
- @Override
- public boolean markSegmentAsUnused(final SegmentId segmentId)
- {
- try {
- final int numSegments = connector.getDBI().withHandle(
- handle ->
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .markSegments(List.of(segmentId), false, DateTimes.nowUtc())
- );
-
- return numSegments > 0;
- }
- catch (RuntimeException e) {
- log.error(e, "Exception marking segment [%s] as unused", segmentId);
- throw e;
- }
- }
-
- @Override
- public int markSegmentsAsUnused(Set<SegmentId> segmentIds)
- {
- return connector.getDBI().withHandle(
- handle ->
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .markSegments(segmentIds, false, DateTimes.nowUtc())
- );
- }
-
- @Override
- public int markAsUnusedSegmentsInInterval(
- final String dataSource,
- final Interval interval,
- @Nullable final List<String> versions
- )
- {
- Preconditions.checkNotNull(interval);
- try {
- return connector.getDBI().withHandle(
- handle ->
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper)
- .markSegmentsUnused(dataSource, interval, versions, DateTimes.nowUtc())
- );
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public @Nullable ImmutableDruidDataSource getImmutableDataSourceWithUsedSegments(String dataSourceName)
- {
- return getSnapshotOfDataSourcesWithAllUsedSegments().getDataSource(dataSourceName);
- }
-
- @Override
- public Collection<ImmutableDruidDataSource> getImmutableDataSourcesWithAllUsedSegments()
- {
- return getSnapshotOfDataSourcesWithAllUsedSegments().getDataSourcesWithAllUsedSegments();
- }
-
- @Override
- public DataSourcesSnapshot getSnapshotOfDataSourcesWithAllUsedSegments()
- {
- useLatestIfWithinDelayOrPerformNewDatabasePoll();
- return dataSourcesSnapshot;
- }
-
@VisibleForTesting
- DataSourcesSnapshot getDataSourcesSnapshot()
+ DataSourcesSnapshot getLatestDataSourcesSnapshot()
{
return dataSourcesSnapshot;
}
@@ -926,117 +632,20 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
return latestDatabasePoll;
}
- @Override
- public Iterable<DataSegment> iterateAllUsedSegments()
- {
- useLatestIfWithinDelayOrPerformNewDatabasePoll();
- return dataSourcesSnapshot.iterateAllUsedSegmentsInSnapshot();
- }
-
- @Override
- public Optional<Iterable<DataSegment>> iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(
- String datasource,
- Interval interval,
- boolean requiresLatest
- )
- {
- if (requiresLatest) {
- forceOrWaitOngoingDatabasePoll();
- } else {
- useLatestIfWithinDelayOrPerformNewDatabasePoll();
- }
- SegmentTimeline usedSegmentsTimeline
- = dataSourcesSnapshot.getUsedSegmentsTimelinesPerDataSource().get(datasource);
- return Optional.fromNullable(usedSegmentsTimeline)
- .transform(timeline -> timeline.findNonOvershadowedObjectsInInterval(interval, Partitions.ONLY_COMPLETE));
- }
-
- /**
- * Retrieves segments and their associated metadata for a given datasource that are marked unused and that are
- * *fully contained by* an optionally specified interval. If the interval specified is null, this method will
- * retrieve all unused segments.
- *
- * This call does not return any information about realtime segments.
- *
- * @param datasource The name of the datasource
- * @param interval an optional interval to search over.
- * @param limit an optional maximum number of results to return. If none is specified, the results are
- * not limited.
- * @param lastSegmentId an optional last segment id from which to search for results. All segments returned are >
- * this segment lexigraphically if sortOrder is null or {@link SortOrder#ASC}, or < this
- * segment lexigraphically if sortOrder is {@link SortOrder#DESC}. If none is specified, no
- * such filter is used.
- * @param sortOrder an optional order with which to return the matching segments by id, start time, end time. If
- * none is specified, the order of the results is not guarenteed.
-
- * Returns an iterable.
- */
- @Override
- public Iterable<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
- final String datasource,
- @Nullable final Interval interval,
- @Nullable final Integer limit,
- @Nullable final String lastSegmentId,
- @Nullable final SortOrder sortOrder
- )
- {
- return connector.inReadOnlyTransaction(
- (handle, status) -> {
- final SqlSegmentsMetadataQuery queryTool =
- SqlSegmentsMetadataQuery.forHandle(handle, connector, dbTables.get(), jsonMapper);
-
- final List<Interval> intervals =
- interval == null
- ? Intervals.ONLY_ETERNITY
- : Collections.singletonList(interval);
- try (final CloseableIterator<DataSegmentPlus> iterator =
- queryTool.retrieveUnusedSegmentsPlus(datasource, intervals, null, limit, lastSegmentId, sortOrder, null)) {
- return ImmutableList.copyOf(iterator);
- }
- }
- );
- }
-
- @Override
- public Set<String> retrieveAllDataSourceNames()
- {
- return connector.getDBI().withHandle(
- handle -> handle
- .createQuery(StringUtils.format("SELECT DISTINCT(datasource) FROM %s", getSegmentsTable()))
- .fold(
- new HashSet<>(),
- (Set<String> druidDataSources,
- Map<String, Object> stringObjectMap,
- FoldController foldController,
- StatementContext statementContext) -> {
- druidDataSources.add(MapUtils.getString(stringObjectMap, "datasource"));
- return druidDataSources;
- }
- )
- );
- }
-
@VisibleForTesting
void poll()
{
// See the comment to the pollLock field, explaining this synchronized block
synchronized (pollLock) {
- doPoll();
- }
- }
-
- /** This method is extracted from {@link #poll()} solely to reduce code nesting. */
- @GuardedBy("pollLock")
- private void doPoll()
- {
- if (centralizedDatasourceSchemaConfig.isEnabled()) {
- doPollSegmentAndSchema();
- } else {
- doPollSegments();
+ if (centralizedDatasourceSchemaConfig.isEnabled()) {
+ pollSegmentAndSchema();
+ } else {
+ pollSegments();
+ }
}
}
- private void doPollSegments()
+ private void pollSegments()
{
final DateTime startTime = DateTimes.nowUtc();
final Stopwatch stopwatch = Stopwatch.createStarted();
@@ -1078,7 +687,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
createDatasourcesSnapshot(startTime, segments);
}
- private void doPollSegmentAndSchema()
+ private void pollSegmentAndSchema()
{
final DateTime startTime = DateTimes.nowUtc();
final Stopwatch stopwatch = Stopwatch.createStarted();
@@ -1212,7 +821,7 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
/**
* For the garbage collector in Java, it's better to keep new objects short-living, but once they are old enough
- * (i. e. promoted to old generation), try to keep them alive. In {@link #poll()}, we fetch and deserialize all
+ * (i.e. promoted to old generation), try to keep them alive. In {@link #poll()}, we fetch and deserialize all
* existing segments each time, and then replace them in {@link #dataSourcesSnapshot}. This method allows to use
* already existing (old) segments when possible, effectively interning them a-la {@link String#intern} or {@link
* com.google.common.collect.Interner}, aiming to make the majority of {@link DataSegment} objects garbage soon after
@@ -1242,69 +851,4 @@ public class SqlSegmentsMetadataManager implements SegmentsMetadataManager
{
return dbTables.get().getSegmentSchemasTable();
}
-
- @Override
- public List<Interval> getUnusedSegmentIntervals(
- final String dataSource,
- @Nullable final DateTime minStartTime,
- final DateTime maxEndTime,
- final int limit,
- final DateTime maxUsedStatusLastUpdatedTime
- )
- {
- // Note that we handle the case where used_status_last_updated IS NULL here to allow smooth transition to Druid version that uses used_status_last_updated column
- return connector.inReadOnlyTransaction(
- new TransactionCallback<>()
- {
- @Override
- public List<Interval> inTransaction(Handle handle, TransactionStatus status)
- {
- final Query<Interval> sql = handle
- .createQuery(
- StringUtils.format(
- "SELECT start, %2$send%2$s FROM %1$s WHERE dataSource = :dataSource AND "
- + "%2$send%2$s <= :end AND used = false AND used_status_last_updated IS NOT NULL AND used_status_last_updated <= :used_status_last_updated %3$s ORDER BY start, %2$send%2$s",
- getSegmentsTable(),
- connector.getQuoteString(),
- null != minStartTime ? "AND start >= :start" : ""
- )
- )
- .setFetchSize(connector.getStreamingFetchSize())
- .setMaxRows(limit)
- .bind("dataSource", dataSource)
- .bind("end", maxEndTime.toString())
- .bind("used_status_last_updated", maxUsedStatusLastUpdatedTime.toString())
- .map(
- new BaseResultSetMapper<>()
- {
- @Override
- protected Interval mapInternal(int index, Map<String, Object> row)
- {
- return new Interval(
- DateTimes.of((String) row.get("start")),
- DateTimes.of((String) row.get("end"))
- );
- }
- }
- );
- if (null != minStartTime) {
- sql.bind("start", minStartTime.toString());
- }
-
- Iterator<Interval> iter = sql.iterator();
-
- List<Interval> result = Lists.newArrayListWithCapacity(limit);
- for (int i = 0; i < limit && iter.hasNext(); i++) {
- try {
- result.add(iter.next());
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- return result;
- }
- }
- );
- }
}
diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java
index 35add575115..f69dc950808 100644
--- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java
+++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProvider.java
@@ -24,6 +24,8 @@ import com.google.common.base.Supplier;
import com.google.inject.Inject;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.metadata.segment.SqlSegmentsMetadataManagerV2;
+import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
import org.apache.druid.segment.metadata.SegmentSchemaCache;
@@ -36,10 +38,12 @@ public class SqlSegmentsMetadataManagerProvider implements SegmentsMetadataManag
private final Lifecycle lifecycle;
private final ServiceEmitter serviceEmitter;
private final SegmentSchemaCache segmentSchemaCache;
+ private final SegmentMetadataCache segmentMetadataCache;
private final CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig;
@Inject
public SqlSegmentsMetadataManagerProvider(
+ SegmentMetadataCache segmentMetadataCache,
ObjectMapper jsonMapper,
Supplier<SegmentsMetadataManagerConfig> config,
Supplier<MetadataStorageTablesConfig> storageConfig,
@@ -57,6 +61,7 @@ public class SqlSegmentsMetadataManagerProvider implements SegmentsMetadataManag
this.lifecycle = lifecycle;
this.serviceEmitter = serviceEmitter;
this.segmentSchemaCache = segmentSchemaCache;
+ this.segmentMetadataCache = segmentMetadataCache;
this.centralizedDatasourceSchemaConfig = centralizedDatasourceSchemaConfig;
}
@@ -82,14 +87,15 @@ public class SqlSegmentsMetadataManagerProvider implements SegmentsMetadataManag
}
);
- return new SqlSegmentsMetadataManager(
- jsonMapper,
+ return new SqlSegmentsMetadataManagerV2(
+ segmentMetadataCache,
+ segmentSchemaCache,
+ connector,
config,
storageConfig,
- connector,
- segmentSchemaCache,
centralizedDatasourceSchemaConfig,
- serviceEmitter
+ serviceEmitter,
+ jsonMapper
);
}
}
diff --git a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
index 1bd02889978..b1d7d16a145 100644
--- a/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
+++ b/server/src/main/java/org/apache/druid/metadata/SqlSegmentsMetadataQuery.java
@@ -20,6 +20,7 @@
package org.apache.druid.metadata;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
@@ -28,10 +29,12 @@ import com.google.common.collect.Lists;
import com.google.common.collect.UnmodifiableIterator;
import org.apache.druid.common.utils.IdUtils;
import org.apache.druid.error.DruidException;
+import org.apache.druid.error.InvalidInput;
import org.apache.druid.java.util.common.CloseableIterators;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.JodaUtils;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.common.jackson.JacksonUtils;
import org.apache.druid.java.util.common.logger.Logger;
@@ -40,6 +43,7 @@ import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
import org.apache.druid.server.http.DataSegmentPlus;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.SegmentId;
+import org.apache.druid.timeline.SegmentTimeline;
import org.apache.druid.utils.CloseableUtils;
import org.joda.time.DateTime;
import org.joda.time.Interval;
@@ -53,6 +57,7 @@ import org.skife.jdbi.v2.util.StringMapper;
import javax.annotation.Nullable;
import java.io.IOException;
+import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -61,12 +66,13 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
+import java.util.Objects;
import java.util.Set;
import java.util.stream.Collectors;
/**
- * An object that helps {@link SqlSegmentsMetadataManager} and {@link IndexerSQLMetadataStorageCoordinator} make
- * queries to the metadata store segments table. Each instance of this class is scoped to a single handle and is meant
+ * An object that is used to query the segments table in the metadata store.
+ * Each instance of this class is scoped to a single {@link Handle} and is meant
* to be short-lived.
*/
public class SqlSegmentsMetadataQuery
@@ -125,12 +131,12 @@ public class SqlSegmentsMetadataQuery
* Retrieves segments for a given datasource that are marked used (i.e. published) in the metadata store, and that
* *overlap* any interval in a particular collection of intervals. If the collection of intervals is empty, this
* method will retrieve all used segments.
- *
+ * <p>
* You cannot assume that segments returned by this call are actually active. Because there is some delay between
* new segment publishing and the marking-unused of older segments, it is possible that some segments returned
* by this call are overshadowed by other segments. To check for this, use
- * {@link org.apache.druid.timeline.SegmentTimeline#forSegments(Iterable)}.
- *
+ * {@link SegmentTimeline#forSegments(Iterable)}.
+ * <p>
* This call does not return any information about realtime segments.
*
* @return a closeable iterator. You should close it when you are done.
@@ -239,6 +245,44 @@ public class SqlSegmentsMetadataQuery
}
}
+ /**
+ * Retrieves segments and their associated metadata for a given datasource that are marked unused and that are
+ * *fully contained by* an optionally specified interval. If the interval specified is null, this method will
+ * retrieve all unused segments.
+ *
+ * This call does not return any information about realtime segments.
+ *
+ * @param datasource The name of the datasource
+ * @param interval an optional interval to search over.
+ * @param limit an optional maximum number of results to return. If none is specified, the results are
+ * not limited.
+ * @param lastSegmentId an optional last segment id from which to search for results. All segments returned are >
+ * this segment lexigraphically if sortOrder is null or {@link SortOrder#ASC}, or < this
+ * segment lexigraphically if sortOrder is {@link SortOrder#DESC}. If none is specified, no
+ * such filter is used.
+ * @param sortOrder an optional order with which to return the matching segments by id, start time, end time. If
+ * none is specified, the order of the results is not guarenteed.
+
+ * Returns an iterable.
+ */
+ public List<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
+ final String datasource,
+ @Nullable final Interval interval,
+ @Nullable final Integer limit,
+ @Nullable final String lastSegmentId,
+ @Nullable final SortOrder sortOrder
+ )
+ {
+ final List<Interval> intervals = interval == null ? Intervals.ONLY_ETERNITY : List.of(interval);
+ try (final CloseableIterator<DataSegmentPlus> iterator =
+ retrieveUnusedSegmentsPlus(datasource, intervals, null, limit, lastSegmentId, sortOrder, null)) {
+ return ImmutableList.copyOf(iterator);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
/**
* Retrieves segments for a given datasource that are marked unused and that are <b>fully contained by</b> any interval
* in a particular collection of intervals. If the collection of intervals is empty, this method will retrieve all
@@ -564,15 +608,39 @@ public class SqlSegmentsMetadataQuery
}
/**
- * Marks the provided segments as either used or unused.
+ * Marks the given segment IDs as used.
*
- * For better performance, please try to
- * 1) ensure that the caller passes only used segments to this method when marking them as unused.
- * 2) Similarly, please try to call this method only on unused segments when marking segments as used with this method.
+ * @param segmentIds Segment IDs to update. For better performance, ensure that
+ * these segment IDs are not already marked as used.
+ * @param updateTime Updated segments will have their used_status_last_updated
+ * column set to this value
+ * @return Number of segments updated in the metadata store.
+ */
+ public int markSegmentsAsUsed(Set<SegmentId> segmentIds, DateTime updateTime)
+ {
+ return markSegments(segmentIds, true, updateTime);
+ }
+
+ /**
+ * Marks the given segment IDs as unused.
+ *
+ * @param segmentIds Segment IDs to update. For better performance, ensure that
+ * these segment IDs are not already marked as unused.
+ * @param updateTime Updated segments will have their used_status_last_updated
+ * column set to this value
+ * @return Number of segments updated in the metadata store.
+ */
+ public int markSegmentsAsUnused(Set<SegmentId> segmentIds, DateTime updateTime)
+ {
+ return markSegments(segmentIds, false, updateTime);
+ }
+
+ /**
+ * Marks the given segments as either used or unused.
*
* @return the number of segments actually modified.
*/
- public int markSegments(final Collection<SegmentId> segmentIds, final boolean used, DateTime updateTime)
+ private int markSegments(final Set<SegmentId> segmentIds, final boolean used, DateTime updateTime)
{
final String dataSource;
@@ -610,23 +678,8 @@ public class SqlSegmentsMetadataQuery
*
* @param interval Only used segments fully contained within this interval
* are eligible to be marked as unused
- * @param updateTime Updated segments will have their used_status_last_updated
- * column set to this value
- * @return Number of segments updated.
- */
- public int markSegmentsUnused(final String dataSource, final Interval interval, final DateTime updateTime)
- {
- return markSegmentsUnused(dataSource, interval, null, updateTime);
- }
-
- /**
- * Marks all used segments that are <b>fully contained by</b> a particular interval
- * filtered by an optional list of versions as unused.
- *
- * @param interval Only used segments fully contained within this interval
- * are eligible to be marked as unused
- * @param versions List of eligible segment versions. If null, all versions
- * are considered eligible to be marked as unused.
+ * @param versions List of eligible segment versions. If null or empty, all
+ * versions are considered eligible to be marked as unused.
* @param updateTime Updated segments will have their used_status_last_updated
* column set to this value
* @return Number of segments updated.
@@ -700,7 +753,7 @@ public class SqlSegmentsMetadataQuery
return stmt.execute();
} else {
// Retrieve, then drop, since we can't write a WHERE clause directly.
- final List<SegmentId> segments = ImmutableList.copyOf(
+ final Set<SegmentId> segments = ImmutableSet.copyOf(
Iterators.transform(
retrieveSegments(
dataSource,
@@ -720,6 +773,172 @@ public class SqlSegmentsMetadataQuery
}
}
+ public boolean markSegmentAsUsed(SegmentId segmentId, DateTime updateTime)
+ {
+ return markSegments(Set.of(segmentId), true, updateTime) > 0;
+ }
+
+ public int markAllNonOvershadowedSegmentsAsUsed(String dataSource, DateTime updateTime)
+ {
+ return markNonOvershadowedSegmentsAsUsedInternal(dataSource, null, null, updateTime);
+ }
+
+ public int markNonOvershadowedSegmentsAsUsed(
+ String dataSource,
+ Interval interval,
+ @Nullable List<String> versions,
+ DateTime updateTime
+ )
+ {
+ Preconditions.checkNotNull(interval);
+ return markNonOvershadowedSegmentsAsUsedInternal(dataSource, interval, versions, updateTime);
+ }
+
+ private int markNonOvershadowedSegmentsAsUsedInternal(
+ final String dataSourceName,
+ final @Nullable Interval interval,
+ final @Nullable List<String> versions,
+ DateTime updateTime
+ )
+ {
+ final List<DataSegment> unusedSegments = new ArrayList<>();
+ final SegmentTimeline timeline = new SegmentTimeline();
+
+ final List<Interval> intervals =
+ interval == null ? Intervals.ONLY_ETERNITY : Collections.singletonList(interval);
+
+ try (final CloseableIterator<DataSegment> iterator =
+ retrieveUsedSegments(dataSourceName, intervals, versions)) {
+ timeline.addSegments(iterator);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ try (final CloseableIterator<DataSegment> iterator =
+ retrieveUnusedSegments(dataSourceName, intervals, versions, null, null, null, null)) {
+ while (iterator.hasNext()) {
+ final DataSegment dataSegment = iterator.next();
+ timeline.add(dataSegment);
+ unusedSegments.add(dataSegment);
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return markNonOvershadowedSegmentsAsUsed(unusedSegments, timeline, updateTime);
+ }
+
+ private int markNonOvershadowedSegmentsAsUsed(
+ List<DataSegment> unusedSegments,
+ SegmentTimeline timeline,
+ DateTime updateTime
+ )
+ {
+ Set<SegmentId> nonOvershadowedSegments =
+ unusedSegments.stream()
+ .filter(segment -> !timeline.isOvershadowed(segment))
+ .map(DataSegment::getId)
+ .collect(Collectors.toSet());
+
+ return markSegmentsAsUsed(nonOvershadowedSegments, updateTime);
+ }
+
+ public int markNonOvershadowedSegmentsAsUsed(
+ final String dataSource,
+ final Set<SegmentId> segmentIds,
+ final DateTime updateTime
+ )
+ {
+ final List<DataSegment> unusedSegments = retrieveUnusedSegments(dataSource, segmentIds);
+ final List<Interval> unusedSegmentsIntervals = JodaUtils.condenseIntervals(
+ unusedSegments.stream().map(DataSegment::getInterval).collect(Collectors.toList())
+ );
+
+ // Create a timeline with all used and unused segments in this interval
+ final SegmentTimeline timeline = SegmentTimeline.forSegments(unusedSegments);
+
+ try (CloseableIterator<DataSegment> usedSegmentsOverlappingUnusedSegmentsIntervals =
+ retrieveUsedSegments(dataSource, unusedSegmentsIntervals)) {
+ timeline.addSegments(usedSegmentsOverlappingUnusedSegmentsIntervals);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return markNonOvershadowedSegmentsAsUsed(unusedSegments, timeline, updateTime);
+ }
+
+ private List<DataSegment> retrieveUnusedSegments(
+ final String dataSource,
+ final Set<SegmentId> segmentIds
+ )
+ {
+ final List<DataSegmentPlus> retrievedSegments = retrieveSegmentsById(dataSource, segmentIds);
+
+ final Set<SegmentId> unknownSegmentIds = new HashSet<>(segmentIds);
+ final List<DataSegment> unusedSegments = new ArrayList<>();
+ for (DataSegmentPlus entry : retrievedSegments) {
+ final DataSegment segment = entry.getDataSegment();
+ unknownSegmentIds.remove(segment.getId());
+ if (Boolean.FALSE.equals(entry.getUsed())) {
+ unusedSegments.add(segment);
+ }
+ }
+
+ if (!unknownSegmentIds.isEmpty()) {
+ throw InvalidInput.exception(
+ "Could not find segment IDs[%s] for datasource[%s]",
+ unknownSegmentIds, dataSource
+ );
+ }
+
+ return unusedSegments;
+ }
+
+ public List<Interval> retrieveUnusedSegmentIntervals(
+ final String dataSource,
+ @Nullable final DateTime minStartTime,
+ final DateTime maxEndTime,
+ final int limit,
+ final DateTime maxUsedStatusLastUpdatedTime
+ )
+ {
+ final boolean filterByStartTime = minStartTime != null;
+
+ // Handle cases where used_status_last_updated IS NULL for backward compatibility
+ final String sql = StringUtils.format(
+ "SELECT start, %2$send%2$s FROM %1$s"
+ + " WHERE dataSource = :dataSource AND used = false"
+ + " AND %2$send%2$s <= :end %3$s"
+ + " AND used_status_last_updated IS NOT NULL"
+ + " AND used_status_last_updated <= :used_status_last_updated"
+ + " ORDER BY start, %2$send%2$s",
+ dbTables.getSegmentsTable(),
+ connector.getQuoteString(),
+ filterByStartTime ? " AND start >= :start" : ""
+ );
+
+ Query<Map<String, Object>> query = handle
+ .createQuery(sql)
+ .setFetchSize(connector.getStreamingFetchSize())
+ .setMaxRows(limit)
+ .bind("dataSource", dataSource)
+ .bind("end", maxEndTime.toString())
+ .bind("used_status_last_updated", maxUsedStatusLastUpdatedTime.toString());
+
+ if (filterByStartTime) {
+ query.bind("start", minStartTime.toString());
+ }
+
+ List<Interval> unusedIntervals = query
+ .map((index, r, ctx) -> mapToInterval(r, dataSource))
+ .list();
+
+ return unusedIntervals.stream().filter(Objects::nonNull).collect(Collectors.toList());
+ }
+
/**
* Retrieve the used segment for a given id if it exists in the metadata store and null otherwise
*/
@@ -1233,6 +1452,21 @@ public class SqlSegmentsMetadataQuery
return sql;
}
+ @Nullable
+ private Interval mapToInterval(ResultSet resultSet, String dataSource)
+ {
+ try {
+ return new Interval(
+ DateTimes.of(resultSet.getString("start")),
+ DateTimes.of(resultSet.getString("end"))
+ );
+ }
+ catch (Throwable t) {
+ log.error(t, "Could not read an interval of datasource[%s]", dataSource);
+ return null;
+ }
+ }
+
private ResultIterator<DataSegment> getDataSegmentResultIterator(Query<Map<String, Object>> sql)
{
return sql.map((index, r, ctx) -> JacksonUtils.readValue(jsonMapper, r.getBytes(2), DataSegment.class))
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java
index 1866d7e10eb..116832dfe31 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransaction.java
@@ -266,13 +266,13 @@ class SqlSegmentMetadataTransaction implements SegmentMetadataTransaction
@Override
public boolean markSegmentAsUnused(SegmentId segmentId, DateTime updateTime)
{
- return query.markSegments(Set.of(segmentId), false, updateTime) > 0;
+ return query.markSegmentsAsUnused(Set.of(segmentId), updateTime) > 0;
}
@Override
public int markSegmentsAsUnused(Set<SegmentId> segmentIds, DateTime updateTime)
{
- return query.markSegments(segmentIds, false, updateTime);
+ return query.markSegmentsAsUnused(segmentIds, updateTime);
}
@Override
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransactionFactory.java b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransactionFactory.java
index a894cfc301a..2942ed1a086 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransactionFactory.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentMetadataTransactionFactory.java
@@ -23,6 +23,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.inject.Inject;
import org.apache.druid.client.indexing.IndexingService;
import org.apache.druid.discovery.DruidLeaderSelector;
+import org.apache.druid.discovery.NodeRole;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.guice.annotations.Self;
import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
@@ -34,6 +37,8 @@ import org.apache.druid.query.DruidMetrics;
import org.skife.jdbi.v2.Handle;
import org.skife.jdbi.v2.TransactionStatus;
+import java.util.Set;
+
/**
* Factory for {@link SegmentMetadataTransaction}s. If the
* {@link SegmentMetadataCache} is enabled and ready, the transaction may
@@ -41,6 +46,15 @@ import org.skife.jdbi.v2.TransactionStatus;
* <p>
* This class serves as a wrapper over the {@link SQLMetadataConnector} to
* perform transactions specific to segment metadata.
+ * <p>
+ * Only the Overlord can perform write operations on the metadata store or the
+ * cache via this transaction factory. The Coordinator performs read operations
+ * directly on the metadata store and uses the cache only to build the timeline
+ * in {@link SqlSegmentsMetadataManagerV2}. There is currently only one method
+ * called by the Coordinator that could benefit from reading from the cache,
+ * {@code MetadataResource.getUsedSegmentsInDataSourceForIntervals()}, but for
+ * now, it continues to read directly from the metadata store for consistency
+ * with older Druid versions.
*/
public class SqlSegmentMetadataTransactionFactory implements SegmentMetadataTransactionFactory
{
@@ -56,12 +70,15 @@ public class SqlSegmentMetadataTransactionFactory implements SegmentMetadataTran
private final SegmentMetadataCache segmentMetadataCache;
private final ServiceEmitter emitter;
+ private final boolean isNotOverlord;
+
@Inject
public SqlSegmentMetadataTransactionFactory(
ObjectMapper jsonMapper,
MetadataStorageTablesConfig tablesConfig,
SQLMetadataConnector connector,
@IndexingService DruidLeaderSelector leaderSelector,
+ @Self Set<NodeRole> nodeRoles,
SegmentMetadataCache segmentMetadataCache,
ServiceEmitter emitter
)
@@ -72,6 +89,8 @@ public class SqlSegmentMetadataTransactionFactory implements SegmentMetadataTran
this.leaderSelector = leaderSelector;
this.segmentMetadataCache = segmentMetadataCache;
this.emitter = emitter;
+
+ this.isNotOverlord = !nodeRoles.contains(NodeRole.OVERLORD);
}
public int getMaxRetries()
@@ -90,8 +109,11 @@ public class SqlSegmentMetadataTransactionFactory implements SegmentMetadataTran
final SegmentMetadataTransaction sqlTransaction
= createSqlTransaction(dataSource, handle, status);
- // For read-only transactions, use cache only if it is already synced
- if (segmentMetadataCache.isSyncedForRead()) {
+ if (isNotOverlord) {
+ // Read directly from the metadata store if not Overlord
+ return executeReadAndClose(sqlTransaction, callback);
+ } else if (segmentMetadataCache.isSyncedForRead()) {
+ // Use cache as it is already synced with the metadata store
emitTransactionCount(Metric.READ_ONLY_TRANSACTIONS, dataSource);
return segmentMetadataCache.readCacheForDataSource(dataSource, dataSourceCache -> {
final SegmentMetadataReadTransaction cachedTransaction
@@ -113,6 +135,10 @@ public class SqlSegmentMetadataTransactionFactory implements SegmentMetadataTran
SegmentMetadataTransaction.Callback<T> callback
)
{
+ if (isNotOverlord) {
+ throw DruidException.defensive("Only Overlord can perform write transactions on segment metadata.");
+ }
+
return connector.retryTransaction(
(handle, status) -> {
final SegmentMetadataTransaction sqlTransaction
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2.java b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2.java
new file mode 100644
index 00000000000..893950c8362
--- /dev/null
+++ b/server/src/main/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata.segment;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Supplier;
+import org.apache.druid.client.DataSourcesSnapshot;
+import org.apache.druid.guice.ManageLifecycle;
+import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
+import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.metadata.MetadataStorageTablesConfig;
+import org.apache.druid.metadata.SQLMetadataConnector;
+import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
+import org.apache.druid.metadata.SqlSegmentsMetadataManager;
+import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
+import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
+import org.apache.druid.segment.metadata.SegmentSchemaCache;
+import org.apache.druid.server.coordinator.DruidCompactionConfig;
+
+/**
+ * Implementation V2 of {@link SegmentsMetadataManager}, that can use the
+ * segments cached in {@link SegmentMetadataCache} to build a {@link DataSourcesSnapshot}.
+ * <p>
+ * This class acts as a wrapper over {@link SqlSegmentsMetadataManager} and the
+ * {@link SegmentMetadataCache}. If the cache is enabled, an additional poll is
+ * not done and the segments already present in the cache are used to build the
+ * snapshot. If the {@link SegmentMetadataCache} is disabled, the polling is
+ * delegated to the legacy implementation in {@link SqlSegmentsMetadataManager}.
+ * <p>
+ * The Coordinator always uses the snapshot to perform various segment management
+ * duties such as loading, balancing, etc.
+ * The Overlord uses the snapshot only when compaction supervisors are enabled.
+ * Thus, when running the Overlord as a standalone service (i.e. not combined
+ * with the Coordinator), {@link #startPollingDatabasePeriodically()} and
+ * {@link #stopPollingDatabasePeriodically()} are called based on the current
+ * state of {@link DruidCompactionConfig#isUseSupervisors()}.
+ */
+@ManageLifecycle
+public class SqlSegmentsMetadataManagerV2 implements SegmentsMetadataManager
+{
+ private static final Logger log = new Logger(SqlSegmentsMetadataManagerV2.class);
+
+ private final SegmentsMetadataManager delegate;
+ private final SegmentMetadataCache segmentMetadataCache;
+ private final SegmentsMetadataManagerConfig managerConfig;
+ private final CentralizedDatasourceSchemaConfig schemaConfig;
+
+ public SqlSegmentsMetadataManagerV2(
+ SegmentMetadataCache segmentMetadataCache,
+ SegmentSchemaCache segmentSchemaCache,
+ SQLMetadataConnector connector,
+ Supplier<SegmentsMetadataManagerConfig> managerConfig,
+ Supplier<MetadataStorageTablesConfig> tablesConfig,
+ CentralizedDatasourceSchemaConfig centralizedDatasourceSchemaConfig,
+ ServiceEmitter serviceEmitter,
+ ObjectMapper jsonMapper
+ )
+ {
+ this.delegate = new SqlSegmentsMetadataManager(
+ jsonMapper,
+ managerConfig, tablesConfig, connector, segmentSchemaCache,
+ centralizedDatasourceSchemaConfig, serviceEmitter
+ );
+ this.managerConfig = managerConfig.get();
+ this.segmentMetadataCache = segmentMetadataCache;
+ this.schemaConfig = centralizedDatasourceSchemaConfig;
+
+ // Segment metadata cache currently cannot handle schema updates
+ if (segmentMetadataCache.isEnabled() && schemaConfig.isEnabled()) {
+ throw new IllegalArgumentException(
+ "Segment metadata incremental cache['druid.manager.segments.useIncrementalCache']"
+ + " and segment schema cache['druid.centralizedDatasourceSchema.enabled']"
+ + " must not be enabled together."
+ );
+ }
+ }
+
+ /**
+ * @return true if segment metadata cache is enabled.
+ */
+ private boolean useIncrementalCache()
+ {
+ return segmentMetadataCache.isEnabled();
+ }
+
+ @Override
+ @LifecycleStart
+ public void start()
+ {
+ delegate.start();
+ }
+
+ @Override
+ @LifecycleStop
+ public void stop()
+ {
+ delegate.stop();
+ }
+
+ @Override
+ public void startPollingDatabasePeriodically()
+ {
+ if (useIncrementalCache()) {
+ log.info("Using segments in metadata cache to build timeline.");
+ } else {
+ log.info("Starting poll of segments from metadata store.");
+ delegate.startPollingDatabasePeriodically();
+ }
+ }
+
+ @Override
+ public void stopPollingDatabasePeriodically()
+ {
+ if (useIncrementalCache()) {
+ // Cache does not stop polling until service is stopped
+ } else {
+ log.info("Stopping poll of segments from metadata store.");
+ delegate.stopPollingDatabasePeriodically();
+ }
+ }
+
+ @Override
+ public boolean isPollingDatabasePeriodically()
+ {
+ // When cache is being used, this will return true even after
+ // stopPollingDatabasePeriodically has been called
+ return useIncrementalCache() || delegate.isPollingDatabasePeriodically();
+ }
+
+ @Override
+ public DataSourcesSnapshot getRecentDataSourcesSnapshot()
+ {
+ if (useIncrementalCache()) {
+ return segmentMetadataCache.getDataSourcesSnapshot();
+ } else {
+ return delegate.getRecentDataSourcesSnapshot();
+ }
+ }
+
+ @Override
+ public DataSourcesSnapshot forceUpdateDataSourcesSnapshot()
+ {
+ if (useIncrementalCache()) {
+ long timeoutMillis = managerConfig.getPollDuration().toStandardDuration().getMillis() * 2;
+ segmentMetadataCache.awaitNextSync(timeoutMillis);
+ return segmentMetadataCache.getDataSourcesSnapshot();
+ } else {
+ return delegate.forceUpdateDataSourcesSnapshot();
+ }
+ }
+
+ // Methods delegated to SqlSegmentsMetadataManager V1 implementation
+
+ @Override
+ public void populateUsedFlagLastUpdatedAsync()
+ {
+ delegate.populateUsedFlagLastUpdatedAsync();
+ }
+
+ @Override
+ public void stopAsyncUsedFlagLastUpdatedUpdate()
+ {
+ delegate.stopAsyncUsedFlagLastUpdatedUpdate();
+ }
+}
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
index 06061219ddc..e954a4f8b4a 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCache.java
@@ -30,6 +30,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import com.google.errorprone.annotations.ThreadSafe;
import com.google.errorprone.annotations.concurrent.GuardedBy;
import com.google.inject.Inject;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.error.DruidException;
import org.apache.druid.error.InternalServerError;
import org.apache.druid.java.util.common.DateTimes;
@@ -61,6 +62,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.concurrent.CancellationException;
import java.util.concurrent.ConcurrentHashMap;
@@ -132,6 +134,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
datasourceToSegmentCache = new ConcurrentHashMap<>();
private final AtomicReference<DateTime> syncFinishTime = new AtomicReference<>();
+ private final AtomicReference<DataSourcesSnapshot> datasourcesSnapshot = new AtomicReference<>(null);
@Inject
public HeapMemorySegmentMetadataCache(
@@ -144,7 +147,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
)
{
this.jsonMapper = jsonMapper;
- this.cacheMode = config.get().getCacheMode();
+ this.cacheMode = config.get().getCacheUsageMode();
this.pollDuration = config.get().getPollDuration().toStandardDuration();
this.tablesConfig = tablesConfig.get();
this.connector = connector;
@@ -199,6 +202,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
pollExecutor.shutdownNow();
datasourceToSegmentCache.forEach((datasource, cache) -> cache.stop());
datasourceToSegmentCache.clear();
+ datasourcesSnapshot.set(null);
syncFinishTime.set(null);
updateCacheState(CacheState.STOPPED, "Stopped sync with metadata store");
@@ -213,13 +217,15 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
if (isEnabled()) {
if (currentCacheState == CacheState.STOPPED) {
throw DruidException.defensive("Cache has not been started yet");
- } else {
+ } else if (currentCacheState == CacheState.FOLLOWER) {
updateCacheState(CacheState.LEADER_FIRST_SYNC_PENDING, "We are now leader");
// Cancel the current sync so that a fresh one is scheduled and cache becomes ready sooner
if (nextSyncFuture != null && !nextSyncFuture.isDone()) {
nextSyncFuture.cancel(true);
}
+ } else {
+ log.info("We are already the leader. Cache is in state[%s].", currentCacheState);
}
}
}
@@ -247,10 +253,27 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
return isEnabled() && isCacheReady.get();
}
+ @Override
+ public DataSourcesSnapshot getDataSourcesSnapshot()
+ {
+ verifyCacheIsUsableAndAwaitSyncIf(isEnabled());
+ return datasourcesSnapshot.get();
+ }
+
+ @Override
+ public void awaitNextSync(long timeoutMillis)
+ {
+ final DateTime lastSyncTime = syncFinishTime.get();
+ final Supplier<Boolean> lastSyncTimeIsNotUpdated =
+ () -> Objects.equals(syncFinishTime.get(), lastSyncTime);
+
+ waitForCacheToFinishSyncWhile(lastSyncTimeIsNotUpdated, timeoutMillis);
+ }
+
@Override
public <T> T readCacheForDataSource(String dataSource, Action<T> readAction)
{
- verifyCacheIsUsableAndAwaitSync();
+ verifyCacheIsUsableAndAwaitSyncIf(isEnabled());
try (final HeapMemoryDatasourceSegmentCache datasourceCache = getCacheWithReference(dataSource)) {
return datasourceCache.withReadLock(
() -> {
@@ -269,7 +292,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
@Override
public <T> T writeCacheForDataSource(String dataSource, Action<T> writeAction)
{
- verifyCacheIsUsableAndAwaitSync();
+ verifyCacheIsUsableAndAwaitSyncIf(cacheMode == UsageMode.ALWAYS);
try (final HeapMemoryDatasourceSegmentCache datasourceCache = getCacheWithReference(dataSource)) {
return datasourceCache.withWriteLock(
() -> {
@@ -316,11 +339,11 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
/**
* Verifies that the cache is enabled, started and has become leader.
* Also waits for the cache to be synced with the metadata store after becoming
- * leader if {@link #cacheMode} is set to {@link UsageMode#ALWAYS}.
+ * leader if {@code shouldWait} is true.
*
* @throws DruidException if the cache is disabled, stopped or not leader.
*/
- private void verifyCacheIsUsableAndAwaitSync()
+ private void verifyCacheIsUsableAndAwaitSyncIf(boolean shouldWait)
{
if (!isEnabled()) {
throw DruidException.defensive("Segment metadata cache is not enabled.");
@@ -334,9 +357,9 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
throw InternalServerError.exception("Not leader yet. Segment metadata cache is not usable.");
case LEADER_FIRST_SYNC_PENDING:
case LEADER_FIRST_SYNC_STARTED:
- if (cacheMode == UsageMode.ALWAYS) {
- waitForCacheToFinishSync();
- verifyCacheIsUsableAndAwaitSync();
+ if (shouldWait) {
+ waitForCacheToFinishSyncWhile(this::isLeaderSyncPending, READY_TIMEOUT_MILLIS);
+ verifyCacheIsUsableAndAwaitSyncIf(true);
}
case LEADER_READY:
// Cache is now ready for use
@@ -344,22 +367,31 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
}
}
+ private boolean isLeaderSyncPending()
+ {
+ synchronized (cacheStateLock) {
+ return currentCacheState == CacheState.LEADER_FIRST_SYNC_PENDING
+ || currentCacheState == CacheState.LEADER_FIRST_SYNC_STARTED;
+ }
+ }
+
/**
- * Waits for cache to become ready if we are leader and current state is
- * {@link CacheState#LEADER_FIRST_SYNC_PENDING} or
- * {@link CacheState#LEADER_FIRST_SYNC_STARTED}.
+ * Waits for cache to finish sync as long as the wait condition is true, or
+ * until the timeout elapses.
*/
- private void waitForCacheToFinishSync()
+ private void waitForCacheToFinishSyncWhile(Supplier<Boolean> waitCondition, long timeoutMillis)
{
+ final Stopwatch totalWaitTime = Stopwatch.createStarted();
+
synchronized (cacheStateLock) {
log.info("Waiting for cache to finish sync with metadata store.");
- while (currentCacheState == CacheState.LEADER_FIRST_SYNC_PENDING
- || currentCacheState == CacheState.LEADER_FIRST_SYNC_STARTED) {
+ while (waitCondition.get() && totalWaitTime.millisElapsed() <= timeoutMillis) {
try {
- cacheStateLock.wait(READY_TIMEOUT_MILLIS);
+ cacheStateLock.wait(timeoutMillis);
}
catch (InterruptedException e) {
log.noStackTrace().info(e, "Interrupted while waiting for cache to be ready");
+ throw DruidException.defensive(e, "Interrupted while waiting for cache to be ready");
}
catch (Exception e) {
log.error(e, "Error while waiting for cache to be ready");
@@ -377,8 +409,13 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
log.info("%s. Cache is now in state[%s].", message, currentCacheState);
isCacheReady.set(currentCacheState == CacheState.LEADER_READY);
+ notifyThreadsWaitingOnCacheSync();
+ }
+ }
- // Notify threads waiting for cache to change state
+ private void notifyThreadsWaitingOnCacheSync()
+ {
+ synchronized (cacheStateLock) {
cacheStateLock.notifyAll();
}
}
@@ -407,6 +444,8 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
previousSyncDurationMillis
)
);
+ } else {
+ notifyThreadsWaitingOnCacheSync();
}
}
@@ -492,7 +531,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
if (syncFinishTime.get() == null) {
retrieveAllUsedSegments(datasourceToSummary);
} else {
- retrieveAllSegmentIds(datasourceToSummary);
+ retrieveUsedSegmentIds(datasourceToSummary);
updateSegmentIdsInCache(datasourceToSummary, syncStartTime);
retrieveUsedSegmentPayloads(datasourceToSummary);
}
@@ -500,7 +539,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
updateUsedSegmentPayloadsInCache(datasourceToSummary);
retrieveAllPendingSegments(datasourceToSummary);
updatePendingSegmentsInCache(datasourceToSummary, syncStartTime);
- markCacheSynced();
+ markCacheSynced(syncStartTime);
syncFinishTime.set(DateTimes.nowUtc());
return totalSyncDuration.millisElapsed();
@@ -509,9 +548,12 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
/**
* Marks the cache for all datasources as synced and emit total stats.
*/
- private void markCacheSynced()
+ private void markCacheSynced(DateTime syncStartTime)
{
+ final Stopwatch updateDuration = Stopwatch.createStarted();
+
final Set<String> cachedDatasources = Set.copyOf(datasourceToSegmentCache.keySet());
+ final Map<String, Set<DataSegment>> datasourceToUsedSegments = new HashMap<>();
for (String dataSource : cachedDatasources) {
final HeapMemoryDatasourceSegmentCache cache = datasourceToSegmentCache.getOrDefault(
@@ -539,15 +581,22 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
emitMetric(dataSource, Metric.CACHED_USED_SEGMENTS, stats.getNumUsedSegments());
emitMetric(dataSource, Metric.CACHED_UNUSED_SEGMENTS, stats.getNumUnusedSegments());
emitMetric(dataSource, Metric.CACHED_PENDING_SEGMENTS, stats.getNumPendingSegments());
+
+ datasourceToUsedSegments.put(dataSource, cache.findUsedSegmentsOverlappingAnyOf(List.of()));
}
}
+
+ datasourcesSnapshot.set(
+ DataSourcesSnapshot.fromUsedSegments(datasourceToUsedSegments, syncStartTime)
+ );
+ emitMetric(Metric.UPDATE_SNAPSHOT_DURATION_MILLIS, updateDuration.millisElapsed());
}
/**
* Retrieves all used segment IDs from the metadata store.
* Populates the summary for the datasources found in metadata store.
*/
- private void retrieveAllSegmentIds(
+ private void retrieveUsedSegmentIds(
Map<String, DatasourceSegmentSummary> datasourceToSummary
)
{
@@ -629,7 +678,7 @@ public class HeapMemorySegmentMetadataCache implements SegmentMetadataCache
/**
* Updates the cache of each datasource with the segment IDs fetched from the
- * metadata store in {@link #retrieveAllSegmentIds}. The update done on each
+ * metadata store in {@link #retrieveUsedSegmentIds}. The update done on each
* datasource cache is atomic.
*/
private void updateSegmentIdsInCache(
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
index adaec00e991..6ce9e245614 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/Metric.java
@@ -105,6 +105,11 @@ public class Metric
*/
public static final String RETRIEVE_PENDING_SEGMENTS_DURATION_MILLIS = METRIC_NAME_PREFIX + "fetchPending/time";
+ /**
+ * Time taken to update the datasource snapshot in the cache.
+ */
+ public static final String UPDATE_SNAPSHOT_DURATION_MILLIS = METRIC_NAME_PREFIX + "updateSnapshot/time";
+
// CACHE UPDATE METRICS
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/NoopSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/NoopSegmentMetadataCache.java
index 7dc7c080ea6..23a28553c8e 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/cache/NoopSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/NoopSegmentMetadataCache.java
@@ -19,6 +19,8 @@
package org.apache.druid.metadata.segment.cache;
+import org.apache.druid.client.DataSourcesSnapshot;
+
public class NoopSegmentMetadataCache implements SegmentMetadataCache
{
private static final NoopSegmentMetadataCache INSTANCE = new NoopSegmentMetadataCache();
@@ -64,6 +66,18 @@ public class NoopSegmentMetadataCache implements SegmentMetadataCache
return false;
}
+ @Override
+ public void awaitNextSync(long timeoutMillis)
+ {
+
+ }
+
+ @Override
+ public DataSourcesSnapshot getDataSourcesSnapshot()
+ {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public <T> T readCacheForDataSource(String dataSource, Action<T> readAction)
{
diff --git a/server/src/main/java/org/apache/druid/metadata/segment/cache/SegmentMetadataCache.java b/server/src/main/java/org/apache/druid/metadata/segment/cache/SegmentMetadataCache.java
index 671aba0c9c5..ace920b846d 100644
--- a/server/src/main/java/org/apache/druid/metadata/segment/cache/SegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/metadata/segment/cache/SegmentMetadataCache.java
@@ -20,10 +20,11 @@
package org.apache.druid.metadata.segment.cache;
import com.fasterxml.jackson.annotation.JsonCreator;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.error.InvalidInput;
/**
- * Cache for metadata of pending segments and committed segments maintained by
+ * Cache for metadata of pending segments and used segments maintained by
* the Overlord to improve performance of segment allocation and other task actions.
* <p>
* Not to be confused with {@link org.apache.druid.segment.metadata.AbstractSegmentMetadataCache}
@@ -64,6 +65,17 @@ public interface SegmentMetadataCache
*/
boolean isSyncedForRead();
+ /**
+ * Waits until the cache finishes the next sync with the metadata store or
+ * until the timeout elapses, whichever is sooner.
+ */
+ void awaitNextSync(long timeoutMillis);
+
+ /**
+ * @return Latest snapshot of the datasources with all their used segments.
+ */
+ DataSourcesSnapshot getDataSourcesSnapshot();
+
/**
* Performs a thread-safe read action on the cache for the given datasource.
* Read actions can be concurrent with other reads but are mutually exclusive
diff --git a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
index 24489e60acd..448d11e2142 100644
--- a/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
+++ b/server/src/main/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCache.java
@@ -41,8 +41,8 @@ import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
+import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.query.DruidMetrics;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
@@ -109,7 +109,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
private final ColumnTypeMergePolicy columnTypeMergePolicy;
private final SegmentSchemaCache segmentSchemaCache;
private final SegmentSchemaBackFillQueue segmentSchemaBackfillQueue;
- private final SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
+ private final SegmentsMetadataManager segmentsMetadataManager;
private final Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier;
private final ServiceEmitter emitter;
private volatile SegmentReplicationStatus segmentReplicationStatus = null;
@@ -134,7 +134,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
ServiceEmitter emitter,
SegmentSchemaCache segmentSchemaCache,
SegmentSchemaBackFillQueue segmentSchemaBackfillQueue,
- SqlSegmentsMetadataManager sqlSegmentsMetadataManager,
+ SegmentsMetadataManager segmentsMetadataManager,
Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier
)
{
@@ -143,7 +143,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
this.columnTypeMergePolicy = config.getMetadataColumnTypeMergePolicy();
this.segmentSchemaCache = segmentSchemaCache;
this.segmentSchemaBackfillQueue = segmentSchemaBackfillQueue;
- this.sqlSegmentsMetadataManager = sqlSegmentsMetadataManager;
+ this.segmentsMetadataManager = segmentsMetadataManager;
this.segmentsMetadataManagerConfigSupplier = segmentsMetadataManagerConfigSupplier;
this.emitter = emitter;
this.coldSchemaExec = Execs.scheduledSingleThreaded("DruidColdSchema-ScheduledExecutor-%d");
@@ -437,12 +437,10 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
final DataSourceInformation cold
)
{
- if (hot == null && cold == null) {
- return Optional.empty();
- } else if (hot != null && cold == null) {
+ if (hot == null) {
+ return Optional.ofNullable(cold);
+ } else if (cold == null) {
return Optional.of(hot);
- } else if (hot == null && cold != null) {
- return Optional.of(cold);
} else {
final Map<String, ColumnType> columnTypes = new LinkedHashMap<>();
@@ -572,7 +570,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
int datasources = 0, dataSourceWithColdSegments = 0, totalColdSegments = 0;
Collection<ImmutableDruidDataSource> immutableDataSources =
- sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments();
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSourcesWithAllUsedSegments();
for (ImmutableDruidDataSource dataSource : immutableDataSources) {
datasources++;
@@ -884,7 +882,7 @@ public class CoordinatorSegmentMetadataCache extends AbstractSegmentMetadataCach
}
ImmutableDruidDataSource druidDataSource =
- sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(segment.getDataSource());
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSource(segment.getDataSource());
if (druidDataSource != null && druidDataSource.getSegment(id) != null) {
markSegmentAsNeedRefresh(id);
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorOverlordServiceConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorOverlordServiceConfig.java
index 3f9df32c61d..03a20bbc1ff 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorOverlordServiceConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorOverlordServiceConfig.java
@@ -49,6 +49,9 @@ public class CoordinatorOverlordServiceConfig
}
}
+ /**
+ * @return true if Coordinator and Overlord are running as a single process.
+ */
public boolean isEnabled()
{
return enabled;
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
index 0a6ac741445..de6ef45a053 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
@@ -104,7 +104,6 @@ import org.joda.time.Duration;
import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -231,7 +230,7 @@ public class DruidCoordinator
public Map<String, Object2LongMap<String>> getTierToDatasourceToUnderReplicatedCount(boolean useClusterView)
{
- final Iterable<DataSegment> dataSegments = metadataManager.segments().iterateAllUsedSegments();
+ final Iterable<DataSegment> dataSegments = metadataManager.iterateAllUsedSegments();
return computeUnderReplicated(dataSegments, useClusterView);
}
@@ -251,7 +250,7 @@ public class DruidCoordinator
final Object2IntOpenHashMap<String> datasourceToUnavailableSegments = new Object2IntOpenHashMap<>();
- final Iterable<DataSegment> dataSegments = metadataManager.segments().iterateAllUsedSegments();
+ final Iterable<DataSegment> dataSegments = metadataManager.iterateAllUsedSegments();
for (DataSegment segment : dataSegments) {
SegmentReplicaCount replicaCount = segmentReplicationStatus.getReplicaCountsInCluster(segment.getId());
if (replicaCount != null && (replicaCount.totalLoaded() > 0 || replicaCount.required() == 0)) {
@@ -272,7 +271,7 @@ public class DruidCoordinator
final Object2IntOpenHashMap<String> datasourceToDeepStorageOnlySegments = new Object2IntOpenHashMap<>();
- final Iterable<DataSegment> dataSegments = metadataManager.segments().iterateAllUsedSegments();
+ final Iterable<DataSegment> dataSegments = metadataManager.iterateAllUsedSegments();
for (DataSegment segment : dataSegments) {
SegmentReplicaCount replicaCount = segmentReplicationStatus.getReplicaCountsInCluster(segment.getId());
if (replicaCount != null && replicaCount.totalLoaded() == 0 && replicaCount.required() == 0) {
@@ -286,10 +285,9 @@ public class DruidCoordinator
public Map<String, Double> getDatasourceToLoadStatus()
{
final Map<String, Double> loadStatus = new HashMap<>();
- final Collection<ImmutableDruidDataSource> dataSources =
- metadataManager.segments().getImmutableDataSourcesWithAllUsedSegments();
+ final DataSourcesSnapshot snapshot = metadataManager.segments().getRecentDataSourcesSnapshot();
- for (ImmutableDruidDataSource dataSource : dataSources) {
+ for (ImmutableDruidDataSource dataSource : snapshot.getDataSourcesWithAllUsedSegments()) {
final Set<DataSegment> segments = Sets.newHashSet(dataSource.getSegments());
final int numPublishedSegments = segments.size();
@@ -349,8 +347,7 @@ public class DruidCoordinator
{
return new CompactionRunSimulator(compactionStatusTracker, overlordClient).simulateRunWithConfig(
metadataManager.configs().getCurrentCompactionConfig().withClusterConfig(updateRequest),
- metadataManager.segments()
- .getSnapshotOfDataSourcesWithAllUsedSegments(),
+ metadataManager.segments().getRecentDataSourcesSnapshot(),
CompactionEngine.NATIVE
);
}
@@ -374,6 +371,7 @@ public class DruidCoordinator
}
started = true;
+ metadataManager.startCache();
coordLeaderSelector.registerListener(
new DruidLeaderSelector.Listener()
{
@@ -402,6 +400,7 @@ public class DruidCoordinator
}
coordLeaderSelector.unregisterListener();
+ metadataManager.stopCache();
started = false;
stopAllDutyGroups();
}
@@ -580,7 +579,7 @@ public class DruidCoordinator
config.getCoordinatorIndexingPeriod()
);
if (killUnusedConfig.isCleanupEnabled()) {
- duties.add(new KillUnusedSegments(metadataManager.segments(), overlordClient, killUnusedConfig));
+ duties.add(new KillUnusedSegments(metadataManager.indexer(), overlordClient, killUnusedConfig));
}
if (config.getKillConfigs().pendingSegments().isCleanupEnabled()) {
duties.add(new KillStalePendingSegments(overlordClient));
@@ -608,7 +607,7 @@ public class DruidCoordinator
)
);
duties.add(
- new KillCompactionConfig(killConfigs.compactionConfigs(), metadataManager.segments(), metadataManager.configs())
+ new KillCompactionConfig(killConfigs.compactionConfigs(), metadataManager.indexer(), metadataManager.configs())
);
if (centralizedDatasourceSchemaConfig.isEnabled()) {
duties.add(new KillUnreferencedSegmentSchema(killConfigs.segmentSchemas(), metadataManager.schemas()));
@@ -666,8 +665,11 @@ public class DruidCoordinator
if (rootCause instanceof HttpResponseException) {
HttpResponseStatus status = ((HttpResponseException) rootCause).getResponse().getStatus();
if (status.getCode() == 404) {
- log.info("Could not update segments via Overlord API. Updating metadata store directly.");
- return metadataManager.segments().markSegmentsAsUnused(segmentIds);
+ log.warn(
+ "Could not mark segments as unused since Overlord is on an older version."
+ + " Upgrade the Overlord to a newer version to allow updating segments."
+ );
+ return 0;
}
}
@@ -724,12 +726,12 @@ public class DruidCoordinator
// use a future snapshotTime to ensure that compaction always runs
dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(
metadataManager.segments()
- .getSnapshotOfDataSourcesWithAllUsedSegments()
+ .getRecentDataSourcesSnapshot()
.iterateAllUsedSegmentsInSnapshot(),
DateTimes.nowUtc().plusMinutes(60)
);
} else {
- dataSourcesSnapshot = metadataManager.segments().getSnapshotOfDataSourcesWithAllUsedSegments();
+ dataSourcesSnapshot = metadataManager.segments().getRecentDataSourcesSnapshot();
}
final DruidCoordinatorRuntimeParams params = DruidCoordinatorRuntimeParams
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/MetadataManager.java b/server/src/main/java/org/apache/druid/server/coordinator/MetadataManager.java
index 5cb1369ff36..91df6ae428f 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/MetadataManager.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/MetadataManager.java
@@ -25,7 +25,9 @@ import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.metadata.MetadataRuleManager;
import org.apache.druid.metadata.MetadataSupervisorManager;
import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
import org.apache.druid.segment.metadata.SegmentSchemaManager;
+import org.apache.druid.timeline.DataSegment;
/**
* Contains all metadata managers used by the Coordinator.
@@ -39,6 +41,7 @@ public class MetadataManager
private final MetadataRuleManager metadataRuleManager;
private final IndexerMetadataStorageCoordinator storageCoordinator;
private final SegmentSchemaManager segmentSchemaManager;
+ private final SegmentMetadataCache segmentMetadataCache;
@Inject
public MetadataManager(
@@ -48,7 +51,8 @@ public class MetadataManager
MetadataSupervisorManager metadataSupervisorManager,
MetadataRuleManager metadataRuleManager,
IndexerMetadataStorageCoordinator storageCoordinator,
- SegmentSchemaManager segmentSchemaManager
+ SegmentSchemaManager segmentSchemaManager,
+ SegmentMetadataCache segmentMetadataCache
)
{
this.auditManager = auditManager;
@@ -58,22 +62,35 @@ public class MetadataManager
this.metadataRuleManager = metadataRuleManager;
this.storageCoordinator = storageCoordinator;
this.segmentSchemaManager = segmentSchemaManager;
+ this.segmentMetadataCache = segmentMetadataCache;
}
public void onLeaderStart()
{
segmentsMetadataManager.startPollingDatabasePeriodically();
segmentsMetadataManager.populateUsedFlagLastUpdatedAsync();
+ segmentMetadataCache.becomeLeader();
metadataRuleManager.start();
}
public void onLeaderStop()
{
metadataRuleManager.stop();
+ segmentMetadataCache.stopBeingLeader();
segmentsMetadataManager.stopPollingDatabasePeriodically();
segmentsMetadataManager.stopAsyncUsedFlagLastUpdatedUpdate();
}
+ public void startCache()
+ {
+ segmentMetadataCache.start();
+ }
+
+ public void stopCache()
+ {
+ segmentMetadataCache.stop();
+ }
+
public boolean isStarted()
{
return segmentsMetadataManager.isPollingDatabasePeriodically();
@@ -113,4 +130,16 @@ public class MetadataManager
{
return segmentSchemaManager;
}
+
+ /**
+ * Returns an iterable to go over all segments in all data sources. The order in which segments are iterated is
+ * unspecified. Note: the iteration may not be as trivially cheap as, for example, iteration over an ArrayList. Try
+ * (to some reasonable extent) to organize the code so that it iterates the returned iterable only once rather than
+ * several times.
+ */
+ public Iterable<DataSegment> iterateAllUsedSegments()
+ {
+ return segments().getRecentDataSourcesSnapshot().iterateAllUsedSegmentsInSnapshot();
+ }
+
}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java
index d2dc33365a0..900feb44ffc 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillCompactionConfig.java
@@ -22,10 +22,10 @@ package org.apache.druid.server.coordinator.duty;
import com.google.common.collect.ImmutableList;
import org.apache.druid.audit.AuditInfo;
import org.apache.druid.common.config.ConfigManager;
+import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.java.util.RetryableException;
import org.apache.druid.java.util.common.RetryUtils;
import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.server.coordinator.CoordinatorConfigManager;
import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -49,17 +49,17 @@ public class KillCompactionConfig extends MetadataCleanupDuty
private static final Logger log = new Logger(KillCompactionConfig.class);
private static final int UPDATE_NUM_RETRY = 5;
- private final SegmentsMetadataManager sqlSegmentsMetadataManager;
+ private final IndexerMetadataStorageCoordinator storageCoordinator;
private final CoordinatorConfigManager configManager;
public KillCompactionConfig(
MetadataCleanupConfig config,
- SegmentsMetadataManager sqlSegmentsMetadataManager,
+ IndexerMetadataStorageCoordinator storageCoordinator,
CoordinatorConfigManager configManager
)
{
super("compaction configs", config, Stats.Kill.COMPACTION_CONFIGS);
- this.sqlSegmentsMetadataManager = sqlSegmentsMetadataManager;
+ this.storageCoordinator = storageCoordinator;
this.configManager = configManager;
}
@@ -94,7 +94,7 @@ public class KillCompactionConfig extends MetadataCleanupDuty
// Get all active datasources
// Note that we get all active datasources after getting compaction config to prevent race condition if new
// datasource and config are added.
- Set<String> activeDatasources = sqlSegmentsMetadataManager.retrieveAllDataSourceNames();
+ Set<String> activeDatasources = storageCoordinator.retrieveAllDatasourceNames();
final Map<String, DataSourceCompactionConfig> updated = current
.getCompactionConfigs()
.stream()
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
index cd1c1746912..64d81a066f9 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/duty/KillUnusedSegments.java
@@ -24,10 +24,10 @@ import com.google.common.collect.Sets;
import org.apache.druid.collections.CircularList;
import org.apache.druid.common.guava.FutureUtils;
import org.apache.druid.indexer.TaskStatusPlus;
+import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.JodaUtils;
import org.apache.druid.java.util.common.logger.Logger;
-import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.rpc.indexing.OverlordClient;
import org.apache.druid.server.coordinator.CoordinatorDynamicConfig;
import org.apache.druid.server.coordinator.DruidCoordinatorRuntimeParams;
@@ -97,14 +97,14 @@ public class KillUnusedSegments implements CoordinatorDuty
private DateTime lastKillTime;
- private final SegmentsMetadataManager segmentsMetadataManager;
+ private final IndexerMetadataStorageCoordinator storageCoordinator;
private final OverlordClient overlordClient;
private String prevDatasourceKilled;
private CircularList<String> datasourceCircularKillList;
public KillUnusedSegments(
- SegmentsMetadataManager segmentsMetadataManager,
+ IndexerMetadataStorageCoordinator storageCoordinator,
OverlordClient overlordClient,
KillUnusedSegmentsConfig killConfig
)
@@ -133,7 +133,7 @@ public class KillUnusedSegments implements CoordinatorDuty
this.maxIntervalToKill
);
- this.segmentsMetadataManager = segmentsMetadataManager;
+ this.storageCoordinator = storageCoordinator;
this.overlordClient = overlordClient;
this.datasourceToLastKillIntervalEnd = new ConcurrentHashMap<>();
}
@@ -166,7 +166,7 @@ public class KillUnusedSegments implements CoordinatorDuty
final Set<String> dataSourcesToKill;
if (CollectionUtils.isNullOrEmpty(dynamicConfig.getSpecificDataSourcesToKillUnusedSegmentsIn())) {
- dataSourcesToKill = segmentsMetadataManager.retrieveAllDataSourceNames();
+ dataSourcesToKill = storageCoordinator.retrieveAllDatasourceNames();
} else {
dataSourcesToKill = dynamicConfig.getSpecificDataSourcesToKillUnusedSegmentsIn();
}
@@ -290,7 +290,7 @@ public class KillUnusedSegments implements CoordinatorDuty
}
final List<Interval> unusedSegmentIntervals = limitToPeriod(
- segmentsMetadataManager.getUnusedSegmentIntervals(
+ storageCoordinator.getUnusedSegmentIntervals(
dataSource,
minStartTime,
maxEndTime,
diff --git a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
index 5f95a52d1c5..2aebf306c3a 100644
--- a/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/DataSourcesResource.java
@@ -19,7 +19,6 @@
package org.apache.druid.server.http;
-import com.google.common.base.Optional;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
@@ -31,13 +30,13 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.druid.audit.AuditEntry;
import org.apache.druid.audit.AuditManager;
import org.apache.druid.client.CoordinatorServerView;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.DruidDataSource;
import org.apache.druid.client.DruidServer;
import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.client.ImmutableSegmentLoadInfo;
import org.apache.druid.client.SegmentLoadInfo;
import org.apache.druid.common.guava.FutureUtils;
-import org.apache.druid.common.utils.IdUtils;
import org.apache.druid.error.DruidException;
import org.apache.druid.guice.annotations.PublicApi;
import org.apache.druid.java.util.common.DateTimes;
@@ -107,6 +106,10 @@ import java.util.stream.Collectors;
public class DataSourcesResource
{
private static final Logger log = new Logger(DataSourcesResource.class);
+
+ /**
+ * Default offset of 14 days.
+ */
private static final long DEFAULT_LOADSTATUS_INTERVAL_OFFSET = 14 * 24 * 60 * 60 * 1000;
private final CoordinatorServerView serverInventoryView;
@@ -184,11 +187,6 @@ public class DataSourcesResource
return Response.ok(getSimpleDatasource(dataSourceName)).build();
}
- private interface SegmentUpdateOperation
- {
- int perform();
- }
-
private interface RemoteSegmentUpdateOperation
{
ListenableFuture<SegmentUpdateResponse> perform();
@@ -204,12 +202,9 @@ public class DataSourcesResource
@ResourceFilters(DatasourceResourceFilter.class)
public Response markAsUsedAllNonOvershadowedSegments(@PathParam("dataSourceName") final String dataSourceName)
{
- SegmentUpdateOperation metadataOperation = () -> segmentsMetadataManager
- .markAsUsedAllNonOvershadowedSegmentsInDataSource(dataSourceName);
RemoteSegmentUpdateOperation remoteOperation = () -> overlordClient
.markNonOvershadowedSegmentsAsUsed(dataSourceName);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
/**
@@ -231,28 +226,9 @@ public class DataSourcesResource
.entity(SegmentsToUpdateFilter.INVALID_PAYLOAD_ERROR_MESSAGE)
.build();
} else {
- SegmentUpdateOperation metadataOperation = () -> {
- final Interval interval = payload.getInterval();
- final List<String> versions = payload.getVersions();
- if (interval != null) {
- return segmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(dataSourceName, interval, versions);
- } else {
- final Set<String> segmentIds = payload.getSegmentIds();
- if (segmentIds == null || segmentIds.isEmpty()) {
- return 0;
- }
-
- return segmentsMetadataManager.markAsUsedNonOvershadowedSegments(
- dataSourceName,
- IdUtils.getValidSegmentIds(dataSourceName, segmentIds)
- );
- }
- };
-
RemoteSegmentUpdateOperation remoteOperation
= () -> overlordClient.markNonOvershadowedSegmentsAsUsed(dataSourceName, payload);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
}
@@ -277,43 +253,9 @@ public class DataSourcesResource
.entity(SegmentsToUpdateFilter.INVALID_PAYLOAD_ERROR_MESSAGE)
.build();
} else {
- SegmentUpdateOperation metadataOperation = () -> {
- final Interval interval = payload.getInterval();
- final List<String> versions = payload.getVersions();
- final int numUpdatedSegments;
- if (interval != null) {
- numUpdatedSegments = segmentsMetadataManager.markAsUnusedSegmentsInInterval(dataSourceName, interval, versions);
- } else {
- final Set<SegmentId> segmentIds =
- payload.getSegmentIds()
- .stream()
- .map(idStr -> SegmentId.tryParse(dataSourceName, idStr))
- .filter(Objects::nonNull)
- .collect(Collectors.toSet());
-
- // Filter out segmentIds that do not belong to this datasource
- numUpdatedSegments = segmentsMetadataManager.markSegmentsAsUnused(
- segmentIds.stream()
- .filter(segmentId -> segmentId.getDataSource().equals(dataSourceName))
- .collect(Collectors.toSet())
- );
- }
- auditManager.doAudit(
- AuditEntry.builder()
- .key(dataSourceName)
- .type("segment.markUnused")
- .payload(payload)
- .auditInfo(AuthorizationUtils.buildAuditInfo(req))
- .request(AuthorizationUtils.buildRequestInfo("coordinator", req))
- .build()
- );
- return numUpdatedSegments;
- };
-
RemoteSegmentUpdateOperation remoteOperation
= () -> overlordClient.markSegmentsAsUnused(dataSourceName, payload);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
}
@@ -323,74 +265,41 @@ public class DataSourcesResource
return Response.noContent().build();
}
- private static Response performSegmentUpdate(String dataSourceName, SegmentUpdateOperation operation)
+ private static Response updateSegmentsViaOverlord(
+ String dataSourceName,
+ RemoteSegmentUpdateOperation operation
+ )
{
try {
- int numChangedSegments = operation.perform();
- return Response.ok(new SegmentUpdateResponse(numChangedSegments)).build();
+ SegmentUpdateResponse response = FutureUtils.getUnchecked(operation.perform(), true);
+ return Response.ok(response).build();
}
catch (DruidException e) {
return ServletResourceUtils.buildErrorResponseFrom(e);
}
catch (Exception e) {
- log.error(e, "Error occurred while updating segments for datasource[%s]", dataSourceName);
+ final Throwable rootCause = Throwables.getRootCause(e);
+ if (rootCause instanceof HttpResponseException) {
+ HttpResponseStatus status = ((HttpResponseException) rootCause).getResponse().getStatus();
+ if (status.getCode() == 404) {
+ final String errorMessage = "Could not update segments since Overlord is on an older version.";
+ log.error(errorMessage);
+ return ServletResourceUtils.buildErrorResponseFrom(
+ DruidException.forPersona(DruidException.Persona.OPERATOR)
+ .ofCategory(DruidException.Category.NOT_FOUND)
+ .build(errorMessage)
+ );
+ }
+ }
+
+ log.error(e, "Error occurred while updating segments for datasource[%s].", dataSourceName);
return Response
.serverError()
- .entity(ImmutableMap.of("error", "Server error", "message", Throwables.getRootCause(e).toString()))
+ .entity(Map.of("error", "Unknown server error", "message", rootCause.toString()))
.build();
}
}
- private static RemoteOrMetadataUpdate updateSegmentsViaOverlord(
- String dataSourceName,
- RemoteSegmentUpdateOperation operation
- )
- {
- return new RemoteOrMetadataUpdate(dataSourceName, operation);
- }
-
- private static class RemoteOrMetadataUpdate
- {
- private final String dataSourceName;
- private final RemoteSegmentUpdateOperation remoteOperation;
-
- private RemoteOrMetadataUpdate(
- String dataSourceName,
- RemoteSegmentUpdateOperation remoteOperation
- )
- {
- this.dataSourceName = dataSourceName;
- this.remoteOperation = remoteOperation;
- }
-
- Response orUpdateMetadataIf404(SegmentUpdateOperation operation)
- {
- try {
- SegmentUpdateResponse response = FutureUtils.getUnchecked(remoteOperation.perform(), true);
- return Response.ok(response).build();
- }
- catch (DruidException e) {
- return ServletResourceUtils.buildErrorResponseFrom(e);
- }
- catch (Exception e) {
- final Throwable rootCause = Throwables.getRootCause(e);
- if (rootCause instanceof HttpResponseException) {
- HttpResponseStatus status = ((HttpResponseException) rootCause).getResponse().getStatus();
- if (status.getCode() == 404) {
- log.info("Could not update segments via Overlord API. Updating metadata store directly.");
- return performSegmentUpdate(dataSourceName, operation);
- }
- }
-
- log.error(e, "Error occurred while updating segments for datasource[%s]", dataSourceName);
- return Response
- .serverError()
- .entity(ImmutableMap.of("error", "Unknown server error", "message", rootCause.toString()))
- .build();
- }
- }
- }
-
/**
* @deprecated Use {@code OverlordDataSourcesResource#markAllSegmentsAsUnused}
* or {@link #killUnusedSegmentsInInterval} instead.
@@ -410,26 +319,9 @@ public class DataSourcesResource
if (Boolean.parseBoolean(kill)) {
return killUnusedSegmentsInInterval(dataSourceName, interval, req);
} else {
- SegmentUpdateOperation metadataOperation = () -> {
- int numUpdatedSegments = segmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(dataSourceName);
- if (numUpdatedSegments > 0) {
- auditManager.doAudit(
- AuditEntry.builder()
- .key(dataSourceName)
- .type("segment.markUnused")
- .payload(new SegmentUpdateResponse(numUpdatedSegments))
- .auditInfo(AuthorizationUtils.buildAuditInfo(req))
- .request(AuthorizationUtils.buildRequestInfo("coordinator", req))
- .build()
- );
- }
- return numUpdatedSegments;
- };
-
RemoteSegmentUpdateOperation remoteOperation
= () -> overlordClient.markSegmentsAsUnused(dataSourceName);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
}
@@ -555,17 +447,20 @@ public class DataSourcesResource
theInterval = Intervals.of(interval.replace('_', '/'));
}
- Optional<Iterable<DataSegment>> segments = segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(
- dataSourceName,
- theInterval,
- forceMetadataRefresh
- );
+ final DataSourcesSnapshot snapshot;
+ if (forceMetadataRefresh) {
+ snapshot = segmentsMetadataManager.forceUpdateDataSourcesSnapshot();
+ } else {
+ snapshot = segmentsMetadataManager.getRecentDataSourcesSnapshot();
+ }
- if (!segments.isPresent()) {
+ final ImmutableDruidDataSource immutableDruidDataSource = snapshot.getDataSource(dataSourceName);
+ if (immutableDruidDataSource == null) {
return logAndCreateDataSourceNotFoundResponse(dataSourceName);
}
- if (Iterables.size(segments.get()) == 0) {
+ final Set<DataSegment> segments = snapshot.getAllUsedNonOvershadowedSegments(dataSourceName, theInterval);
+ if (segments.isEmpty()) {
return Response
.status(Response.Status.NO_CONTENT)
.entity("No used segment found for the given datasource and interval")
@@ -574,7 +469,7 @@ public class DataSourcesResource
if (simple != null) {
// Calculate response for simple mode
- SegmentsLoadStatistics segmentsLoadStatistics = computeSegmentLoadStatistics(segments.get());
+ SegmentsLoadStatistics segmentsLoadStatistics = computeSegmentLoadStatistics(segments);
return Response.ok(
ImmutableMap.of(
dataSourceName,
@@ -584,7 +479,7 @@ public class DataSourcesResource
} else if (full != null) {
// Calculate response for full mode
Map<String, Object2LongMap<String>> segmentLoadMap =
- coordinator.getTierToDatasourceToUnderReplicatedCount(segments.get(), computeUsingClusterView != null);
+ coordinator.getTierToDatasourceToUnderReplicatedCount(segments, computeUsingClusterView != null);
if (segmentLoadMap.isEmpty()) {
return Response.serverError()
.entity("Coordinator segment replicant lookup is not initialized yet. Try again later.")
@@ -593,7 +488,7 @@ public class DataSourcesResource
return Response.ok(segmentLoadMap).build();
} else {
// Calculate response for default mode
- SegmentsLoadStatistics segmentsLoadStatistics = computeSegmentLoadStatistics(segments.get());
+ SegmentsLoadStatistics segmentsLoadStatistics = computeSegmentLoadStatistics(segments);
return Response.ok(
ImmutableMap.of(
dataSourceName,
@@ -779,13 +674,10 @@ public class DataSourcesResource
).build();
}
- SegmentUpdateOperation metadataOperation
- = () -> segmentsMetadataManager.markSegmentAsUnused(segmentId) ? 1 : 0;
RemoteSegmentUpdateOperation remoteOperation
= () -> overlordClient.markSegmentAsUnused(segmentId);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
/**
@@ -811,13 +703,10 @@ public class DataSourcesResource
).build();
}
- SegmentUpdateOperation metadataOperation
- = () -> segmentsMetadataManager.markSegmentAsUsed(segmentIdString) ? 1 : 0;
RemoteSegmentUpdateOperation remoteOperation
= () -> overlordClient.markSegmentAsUsed(segmentId);
- return updateSegmentsViaOverlord(dataSourceName, remoteOperation)
- .orUpdateMetadataIf404(metadataOperation);
+ return updateSegmentsViaOverlord(dataSourceName, remoteOperation);
}
@GET
diff --git a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
index ec8f3206ff5..c2c747df4f6 100644
--- a/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
+++ b/server/src/main/java/org/apache/druid/server/http/MetadataResource.java
@@ -114,9 +114,9 @@ public class MetadataResource
Collection<ImmutableDruidDataSource> druidDataSources = null;
final TreeSet<String> dataSourceNamesPreAuth;
if (includeUnused) {
- dataSourceNamesPreAuth = new TreeSet<>(segmentsMetadataManager.retrieveAllDataSourceNames());
+ dataSourceNamesPreAuth = new TreeSet<>(metadataStorageCoordinator.retrieveAllDatasourceNames());
} else {
- druidDataSources = segmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments();
+ druidDataSources = segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSourcesWithAllUsedSegments();
dataSourceNamesPreAuth = druidDataSources
.stream()
.map(ImmutableDruidDataSource::getName)
@@ -171,7 +171,7 @@ public class MetadataResource
}
Collection<ImmutableDruidDataSource> dataSourcesWithUsedSegments =
- segmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments();
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSourcesWithAllUsedSegments();
if (dataSources != null && !dataSources.isEmpty()) {
dataSourcesWithUsedSegments = dataSourcesWithUsedSegments
.stream()
@@ -206,7 +206,7 @@ public class MetadataResource
String includeRealtimeSegments
)
{
- DataSourcesSnapshot dataSourcesSnapshot = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments();
+ DataSourcesSnapshot dataSourcesSnapshot = segmentsMetadataManager.getRecentDataSourcesSnapshot();
Collection<ImmutableDruidDataSource> dataSourcesWithUsedSegments =
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments();
if (dataSources != null && !dataSources.isEmpty()) {
@@ -294,8 +294,7 @@ public class MetadataResource
@ResourceFilters(DatasourceResourceFilter.class)
public Response getDataSourceWithUsedSegments(@PathParam("dataSourceName") final String dataSourceName)
{
- ImmutableDruidDataSource dataSource =
- segmentsMetadataManager.getImmutableDataSourceWithUsedSegments(dataSourceName);
+ ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
@@ -312,8 +311,7 @@ public class MetadataResource
@QueryParam("full") @Nullable String full
)
{
- ImmutableDruidDataSource dataSource =
- segmentsMetadataManager.getImmutableDataSourceWithUsedSegments(dataSourceName);
+ ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
@@ -380,7 +378,7 @@ public class MetadataResource
final SortOrder theSortOrder = sortOrder == null ? null : SortOrder.fromValue(sortOrder);
final Interval theInterval = interval != null ? Intervals.of(interval.replace('_', '/')) : null;
- final Iterable<DataSegmentPlus> unusedSegments = segmentsMetadataManager.iterateAllUnusedSegmentsForDatasource(
+ final Iterable<DataSegmentPlus> unusedSegments = metadataStorageCoordinator.iterateAllUnusedSegmentsForDatasource(
dataSource,
theInterval,
limit,
@@ -413,7 +411,7 @@ public class MetadataResource
@QueryParam("includeUnused") @Nullable Boolean includeUnused
)
{
- ImmutableDruidDataSource dataSource = segmentsMetadataManager.getImmutableDataSourceWithUsedSegments(dataSourceName);
+ ImmutableDruidDataSource dataSource = getDataSource(dataSourceName);
if (dataSource == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
@@ -496,4 +494,9 @@ public class MetadataResource
}
return Response.status(Response.Status.OK).entity(broadcastSegments).build();
}
+
+ private ImmutableDruidDataSource getDataSource(String dataSource)
+ {
+ return segmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSource(dataSource);
+ }
}
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java
new file mode 100644
index 00000000000..7007607a2b0
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorMarkUsedTest.java
@@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.discovery.NodeRole;
+import org.apache.druid.error.DruidException;
+import org.apache.druid.error.DruidExceptionMatcher;
+import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
+import org.apache.druid.indexing.overlord.Segments;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
+import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector;
+import org.apache.druid.server.metrics.NoopServiceEmitter;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.SegmentId;
+import org.apache.druid.timeline.partition.LinearShardSpec;
+import org.hamcrest.MatcherAssert;
+import org.joda.time.Duration;
+import org.joda.time.Interval;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Unit tests for {@link IndexerSQLMetadataStorageCoordinator} methods which
+ * mark segments as used.
+ * <p>
+ * These tests have been kept out of {@link IndexerSQLMetadataStorageCoordinatorTest}
+ * as that class is already too bloated.
+ */
+public class IndexerSQLMetadataStorageCoordinatorMarkUsedTest extends IndexerSqlMetadataStorageCoordinatorTestBase
+{
+ @Rule
+ public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule
+ = new TestDerbyConnector.DerbyConnectorRule();
+
+ private IndexerMetadataStorageCoordinator storageCoordinator;
+
+ private final DataSegment wikiSegment1 =
+ CreateDataSegments.ofDatasource(TestDataSource.WIKI).startingAt("2012-03-15").eachOfSizeInMb(500).get(0);
+ private final DataSegment wikiSegment2 =
+ CreateDataSegments.ofDatasource(TestDataSource.WIKI).startingAt("2012-01-05").eachOfSizeInMb(500).get(0);
+
+ @Before
+ public void setup()
+ {
+ derbyConnector = derbyConnectorRule.getConnector();
+
+ final SqlSegmentMetadataTransactionFactory transactionFactory = new SqlSegmentMetadataTransactionFactory(
+ mapper,
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
+ derbyConnector,
+ new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
+ NoopSegmentMetadataCache.instance(),
+ NoopServiceEmitter.instance()
+ )
+ {
+ @Override
+ public int getMaxRetries()
+ {
+ return MAX_SQL_MEATADATA_RETRY_FOR_TEST;
+ }
+ };
+
+ storageCoordinator = new IndexerSQLMetadataStorageCoordinator(
+ transactionFactory,
+ mapper,
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
+ derbyConnector,
+ null,
+ CentralizedDatasourceSchemaConfig.create()
+ );
+
+ derbyConnector.createSegmentTable();
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed1()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+ final Set<SegmentId> segmentIds = Set.of(
+ koalaSegment1.getId(),
+ koalaSegment2.getId(),
+ koalaSegment3.getId()
+ );
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(2, storageCoordinator.markNonOvershadowedSegmentsAsUsed(TestDataSource.KOALA, segmentIds));
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ Set.of(koalaSegment1, koalaSegment2),
+ retrieveAllUsedSegments(TestDataSource.KOALA)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_inEternityIntervalWithVersions()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ 2,
+ storageCoordinator.markNonOvershadowedSegmentsAsUsed(
+ TestDataSource.KOALA,
+ Intervals.ETERNITY,
+ ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
+ )
+ );
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ Set.of(koalaSegment1, koalaSegment2),
+ retrieveAllUsedSegments(TestDataSource.KOALA)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_withEmptyVersions()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ 0,
+ storageCoordinator.markNonOvershadowedSegmentsAsUsed(
+ TestDataSource.KOALA,
+ Intervals.of("2017/2018"),
+ ImmutableList.of()
+ )
+ );
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_inEternityIntervalWithEmptyVersions()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ 0,
+ storageCoordinator.markNonOvershadowedSegmentsAsUsed(
+ TestDataSource.KOALA,
+ Intervals.ETERNITY,
+ ImmutableList.of()
+ )
+ );
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_inFiniteIntervalWithVersions()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ 2,
+ storageCoordinator.markNonOvershadowedSegmentsAsUsed(
+ TestDataSource.KOALA,
+ Intervals.of("2017-10-15/2017-10-18"),
+ ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
+ )
+ );
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ Set.of(koalaSegment1, koalaSegment2),
+ retrieveAllUsedSegments(TestDataSource.KOALA)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_withNonExistentVersions()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ 0,
+ storageCoordinator.markNonOvershadowedSegmentsAsUsed(
+ TestDataSource.KOALA,
+ Intervals.ETERNITY,
+ ImmutableList.of("foo", "bar")
+ )
+ );
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_invalidDataSource()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+ final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2);
+ final Set<SegmentId> segmentIds = Set.of(koalaSegment1.getId(), koalaSegment2.getId());
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+
+ MatcherAssert.assertThat(
+ Assert.assertThrows(
+ DruidException.class,
+ () -> storageCoordinator.markNonOvershadowedSegmentsAsUsed("wrongDataSource", segmentIds)
+ ),
+ DruidExceptionMatcher
+ .invalidInput()
+ .expectMessageContains("Could not find segment IDs")
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_withInvalidSegmentIds()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+ final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
+
+ final Set<SegmentId> segmentIds = Set.of(koalaSegment1.getId(), koalaSegment2.getId());
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+
+ MatcherAssert.assertThat(
+ Assert.assertThrows(
+ DruidException.class,
+ () -> storageCoordinator.markNonOvershadowedSegmentsAsUsed(TestDataSource.KOALA, segmentIds)
+ ),
+ DruidExceptionMatcher
+ .invalidInput()
+ .expectMessageContains("Could not find segment IDs")
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
+ final Interval theInterval = Intervals.of("2017-10-15T00:00:00.000/2017-10-18T00:00:00.000");
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+
+ // 2 out of 3 segments match the interval
+ Assert.assertEquals(2, storageCoordinator.markNonOvershadowedSegmentsAsUsed(TestDataSource.KOALA, theInterval, null));
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(
+ Set.of(koalaSegment1, koalaSegment2),
+ retrieveAllUsedSegments(TestDataSource.KOALA)
+ );
+ }
+
+ @Test
+ public void test_markNonOvershadowedSegmentsAsUsed_withOverlappingInterval()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-16T20:19:12.565Z"
+ );
+
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-19T00:00:00.000/2017-10-22T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+
+ // Overshadowed by koalaSegment2
+ final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
+
+ publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
+ final Interval theInterval = Intervals.of("2017-10-16T00:00:00.000/2017-10-20T00:00:00.000");
+
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+
+ // 1 out of 3 segments match the interval, other 2 overlap, only the segment fully contained will be marked unused
+ Assert.assertEquals(1, storageCoordinator.markNonOvershadowedSegmentsAsUsed(TestDataSource.KOALA, theInterval, null));
+
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments(TestDataSource.WIKI)
+ );
+ Assert.assertEquals(Set.of(koalaSegment2), retrieveAllUsedSegments(TestDataSource.KOALA));
+ }
+
+ @Test
+ public void test_getUnusedSegmentIntervals()
+ {
+ publishSegments(wikiSegment1, wikiSegment2);
+
+ int numChangedSegments = storageCoordinator.markSegmentsAsUnused(
+ TestDataSource.WIKI,
+ Set.of(wikiSegment1.getId(), wikiSegment2.getId())
+ );
+ Assert.assertEquals(2, numChangedSegments);
+
+ // Publish an unused segment with used_status_last_updated 2 hours ago
+ final DataSegment koalaSegment1 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+ publishUnusedSegments(koalaSegment1);
+ derbyConnectorRule.segments().updateUsedStatusLastUpdated(
+ koalaSegment1.getId().toString(),
+ DateTimes.nowUtc().minus(Duration.standardHours(2))
+ );
+
+ // Publish an unused segment with used_status_last_updated 2 days ago
+ final DataSegment koalaSegment2 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-16T00:00:00.000/2017-10-17T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+ publishUnusedSegments(koalaSegment2);
+ derbyConnectorRule.segments().updateUsedStatusLastUpdated(
+ koalaSegment2.getId().toString(),
+ DateTimes.nowUtc().minus(Duration.standardDays(2))
+ );
+
+ // Publish an unused segment and set used_status_last_updated to null
+ final DataSegment koalaSegment3 = createSegment(
+ TestDataSource.KOALA,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+ publishUnusedSegments(koalaSegment3);
+
+ Assert.assertEquals(
+ ImmutableList.of(wikiSegment2.getInterval()),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.of("2000"),
+ DateTimes.of("3000"),
+ 1,
+ DateTimes.COMPARE_DATE_AS_STRING_MAX
+ )
+ );
+
+ // Test the DateTime maxEndTime argument of getUnusedSegmentIntervals
+ Assert.assertEquals(
+ ImmutableList.of(wikiSegment2.getInterval()),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.of("2000"),
+ DateTimes.of(2012, 1, 7, 0, 0),
+ 1,
+ DateTimes.COMPARE_DATE_AS_STRING_MAX
+ )
+ );
+ Assert.assertEquals(
+ ImmutableList.of(wikiSegment1.getInterval()),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.of(2012, 1, 7, 0, 0),
+ DateTimes.of(2012, 4, 7, 0, 0),
+ 1,
+ DateTimes.COMPARE_DATE_AS_STRING_MAX
+ )
+ );
+ Assert.assertEquals(
+ ImmutableList.of(),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.of(2012, 1, 7, 0, 0),
+ DateTimes.of(2012, 1, 7, 0, 0),
+ 1,
+ DateTimes.COMPARE_DATE_AS_STRING_MAX
+ )
+ );
+
+ Assert.assertEquals(
+ ImmutableList.of(wikiSegment2.getInterval(), wikiSegment1.getInterval()),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.of("2000"),
+ DateTimes.of("3000"),
+ 5,
+ DateTimes.COMPARE_DATE_AS_STRING_MAX
+ )
+ );
+
+ // Test a buffer period that should exclude some segments
+
+ // The wikipedia datasource has segments generated with last used time equal to roughly the time of test run. None of these segments should be selected with a bufer period of 1 day
+ Assert.assertEquals(
+ ImmutableList.of(),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.WIKI,
+ DateTimes.COMPARE_DATE_AS_STRING_MIN,
+ DateTimes.of("3000"),
+ 5,
+ DateTimes.nowUtc().minus(Duration.parse("PT86400S"))
+ )
+ );
+
+ // koalaSegment3 has a null used_status_last_updated which should mean getUnusedSegmentIntervals never returns it
+ // koalaSegment2 has a used_status_last_updated older than 1 day which means it should be returned
+ // The last of the 3 segments in koala has a used_status_last_updated date less than one day and should not be returned
+ Assert.assertEquals(
+ ImmutableList.of(koalaSegment2.getInterval()),
+ storageCoordinator.getUnusedSegmentIntervals(
+ TestDataSource.KOALA,
+ DateTimes.COMPARE_DATE_AS_STRING_MIN,
+ DateTimes.of("3000"),
+ 5,
+ DateTimes.nowUtc().minus(Duration.parse("PT86400S"))
+ )
+ );
+ }
+
+ private Set<DataSegment> retrieveAllUsedSegments(String dataSource)
+ {
+ return storageCoordinator.retrieveAllUsedSegments(dataSource, Segments.INCLUDING_OVERSHADOWED);
+ }
+
+ private void publishUnusedSegments(DataSegment... segments)
+ {
+ publishSegments(segments);
+ for (DataSegment segment : segments) {
+ storageCoordinator.markSegmentAsUnused(segment.getId());
+ }
+ }
+
+ private void publishSegments(DataSegment... segments)
+ {
+ storageCoordinator.commitSegments(Set.of(segments), null);
+ }
+
+ private static DataSegment createSegment(
+ String dataSource,
+ String interval,
+ String version
+ )
+ {
+ return new DataSegment(
+ dataSource,
+ Intervals.of(interval),
+ version,
+ Map.of(),
+ List.of(),
+ List.of(),
+ new LinearShardSpec(1),
+ 9,
+ 1234L
+ );
+ }
+
+ private static DataSegment createNewSegment1(String datasource)
+ {
+ return createSegment(
+ datasource,
+ "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+ }
+
+ private static DataSegment createNewSegment2(String datasource)
+ {
+ return createSegment(
+ datasource,
+ "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ "2017-10-15T20:19:12.565Z"
+ );
+ }
+}
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
index 6671fe4e133..b902b1be608 100644
--- a/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSQLMetadataStorageCoordinatorTest.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.apache.druid.data.input.StringTuple;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexing.overlord.DataSourceMetadata;
import org.apache.druid.indexing.overlord.ObjectMetadata;
import org.apache.druid.indexing.overlord.SegmentCreateRequest;
@@ -173,6 +174,7 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnector,
leaderSelector,
+ Set.of(NodeRole.OVERLORD),
segmentMetadataCache,
emitter
)
@@ -4068,7 +4070,6 @@ public class IndexerSQLMetadataStorageCoordinatorTest extends IndexerSqlMetadata
private void insertUsedSegments(Set<DataSegment> segments, Map<String, String> upgradedFromSegmentIdMap)
{
- final String table = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
- insertUsedSegments(segments, upgradedFromSegmentIdMap, derbyConnector, table, mapper);
+ insertUsedSegments(segments, upgradedFromSegmentIdMap, derbyConnectorRule, mapper);
}
}
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java
index c075b7ae890..75d2cd9d2b6 100644
--- a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest.java
@@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import org.apache.druid.discovery.NodeRole;
import org.apache.druid.indexing.overlord.DataSourceMetadata;
import org.apache.druid.indexing.overlord.SegmentPublishResult;
import org.apache.druid.java.util.common.Intervals;
@@ -97,6 +98,7 @@ public class IndexerSqlMetadataStorageCoordinatorSchemaPersistenceTest extends
derbyConnectorRule.metadataTablesConfigSupplier().get(),
derbyConnector,
new TestDruidLeaderSelector(),
+ Set.of(NodeRole.OVERLORD),
NoopSegmentMetadataCache.instance(),
NoopServiceEmitter.instance()
);
diff --git a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
index 1deb4452561..a5e30ae90a7 100644
--- a/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
+++ b/server/src/test/java/org/apache/druid/metadata/IndexerSqlMetadataStorageCoordinatorTestBase.java
@@ -558,8 +558,7 @@ public class IndexerSqlMetadataStorageCoordinatorTestBase
public static void insertUsedSegments(
Set<DataSegment> dataSegments,
Map<String, String> upgradedFromSegmentIdMap,
- SQLMetadataConnector connector,
- String table,
+ TestDerbyConnector.DerbyConnectorRule derbyConnectorRule,
ObjectMapper jsonMapper
)
{
@@ -579,16 +578,18 @@ public class IndexerSqlMetadataStorageCoordinatorTestBase
);
}
- insertSegments(usedSegments, connector, table, jsonMapper);
+ insertSegments(usedSegments, derbyConnectorRule, jsonMapper);
}
public static void insertSegments(
Set<DataSegmentPlus> dataSegments,
- SQLMetadataConnector connector,
- String table,
+ TestDerbyConnector.DerbyConnectorRule derbyConnectorRule,
ObjectMapper jsonMapper
)
{
+ final TestDerbyConnector connector = derbyConnectorRule.getConnector();
+ final String table = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
+
connector.retryWithHandle(
handle -> {
PreparedBatch preparedBatch = handle.prepareBatch(
diff --git a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
index b3110912509..433981ac330 100644
--- a/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SQLMetadataRuleManagerTest.java
@@ -327,7 +327,7 @@ public class SQLMetadataRuleManagerTest
}
@Test
- public void testRemoveRulesOlderThanWithActiveDatasourceShouldNotDelete() throws Exception
+ public void testRemoveRulesOlderThanWithActiveDatasourceShouldNotDelete()
{
List<Rule> rules = ImmutableList.of(
new IntervalLoadRule(
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProviderTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProviderTest.java
index 36fc1c4e7da..0841f586a5d 100644
--- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProviderTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerProviderTest.java
@@ -22,6 +22,8 @@ package org.apache.druid.metadata;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Suppliers;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
+import org.apache.druid.metadata.segment.SqlSegmentsMetadataManagerV2;
+import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
import org.apache.druid.segment.metadata.SegmentSchemaCache;
@@ -48,6 +50,7 @@ public class SqlSegmentsMetadataManagerProviderTest
final Lifecycle lifecycle = new Lifecycle();
final SegmentSchemaCache segmentSchemaCache = new SegmentSchemaCache(new NoopServiceEmitter());
SqlSegmentsMetadataManagerProvider provider = new SqlSegmentsMetadataManagerProvider(
+ NoopSegmentMetadataCache.instance(),
jsonMapper,
Suppliers.ofInstance(config),
derbyConnectorRule.metadataTablesConfigSupplier(),
@@ -58,7 +61,7 @@ public class SqlSegmentsMetadataManagerProviderTest
NoopServiceEmitter.instance()
);
SegmentsMetadataManager manager = provider.get();
- Assert.assertTrue(manager instanceof SqlSegmentsMetadataManager);
+ Assert.assertTrue(manager instanceof SqlSegmentsMetadataManagerV2);
final MetadataStorageTablesConfig storageConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
final String segmentsTable = storageConfig.getSegmentsTable();
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerSchemaPollTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerSchemaPollTest.java
index 788de1fa0ff..2ac01f864a1 100644
--- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerSchemaPollTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerSchemaPollTest.java
@@ -53,31 +53,14 @@ public class SqlSegmentsMetadataManagerSchemaPollTest extends SqlSegmentsMetadat
@Before
public void setUp() throws Exception
{
- connector = derbyConnectorRule.getConnector();
- SegmentsMetadataManagerConfig config = new SegmentsMetadataManagerConfig(Period.seconds(3), null);
-
- segmentSchemaCache = new SegmentSchemaCache(new NoopServiceEmitter());
+ setUp(derbyConnectorRule);
+ segmentSchemaCache = new SegmentSchemaCache(NoopServiceEmitter.instance());
segmentSchemaManager = new SegmentSchemaManager(
derbyConnectorRule.metadataTablesConfigSupplier().get(),
jsonMapper,
connector
);
- sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
- jsonMapper,
- Suppliers.ofInstance(config),
- derbyConnectorRule.metadataTablesConfigSupplier(),
- connector,
- segmentSchemaCache,
- CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
- );
- sqlSegmentsMetadataManager.start();
- storageConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
-
- connector.createSegmentSchemasTable();
- connector.createSegmentTable();
-
publishSegment(segment1);
publishSegment(segment2);
}
@@ -85,10 +68,7 @@ public class SqlSegmentsMetadataManagerSchemaPollTest extends SqlSegmentsMetadat
@After
public void teardown()
{
- if (sqlSegmentsMetadataManager.isPollingDatabasePeriodically()) {
- sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
- }
- sqlSegmentsMetadataManager.stop();
+ teardownManager();
}
@Test(timeout = 60_000)
@@ -141,7 +121,7 @@ public class SqlSegmentsMetadataManagerSchemaPollTest extends SqlSegmentsMetadat
);
sqlSegmentsMetadataManager.start();
- DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertNull(dataSourcesSnapshot);
Assert.assertFalse(segmentSchemaCache.getSchemaForSegment(segment1.getId()).isPresent());
Assert.assertFalse(segmentSchemaCache.getSchemaForSegment(segment2.getId()).isPresent());
@@ -159,11 +139,7 @@ public class SqlSegmentsMetadataManagerSchemaPollTest extends SqlSegmentsMetadat
Assert.assertEquals(schemaMetadata1, segmentSchemaCache.getSchemaForSegment(segment1.getId()).get());
Assert.assertEquals(schemaMetadata2, segmentSchemaCache.getSchemaForSegment(segment2.getId()).get());
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
- Assert.assertEquals(
- ImmutableSet.of("wikipedia"),
- sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
- );
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableList.of("wikipedia"),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
index c1f9a0d544f..987d93dcd27 100644
--- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTest.java
@@ -19,7 +19,6 @@
package org.apache.druid.metadata;
-import com.google.common.base.Optional;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
@@ -27,24 +26,16 @@ import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.ImmutableDruidDataSource;
-import org.apache.druid.error.DruidException;
-import org.apache.druid.error.DruidExceptionMatcher;
-import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.segment.TestDataSource;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
-import org.apache.druid.segment.metadata.SegmentSchemaCache;
-import org.apache.druid.segment.metadata.SegmentSchemaManager;
import org.apache.druid.server.coordinator.CreateDataSegments;
import org.apache.druid.server.metrics.NoopServiceEmitter;
import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.partition.NoneShardSpec;
-import org.hamcrest.MatcherAssert;
-import org.joda.time.DateTime;
-import org.joda.time.Duration;
+import org.assertj.core.util.Sets;
import org.joda.time.Interval;
import org.joda.time.Period;
import org.junit.After;
@@ -53,7 +44,6 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
-import java.io.IOException;
import java.util.Set;
import java.util.stream.Collectors;
@@ -87,61 +77,30 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
private final DataSegment wikiSegment2 =
CreateDataSegments.ofDatasource(TestDataSource.WIKI).startingAt("2012-01-05").eachOfSizeInMb(500).get(0);
- private void publishUnusedSegments(DataSegment... segments) throws IOException
+ private void publishUnusedSegments(DataSegment... segments)
{
for (DataSegment segment : segments) {
publishSegment(segment);
- sqlSegmentsMetadataManager.markSegmentAsUnused(segment.getId());
+ markSegmentsAsUnused(segment.getId());
}
}
private void publishWikiSegments()
{
- try {
- publishSegment(wikiSegment1);
- publishSegment(wikiSegment2);
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
+ publishSegment(wikiSegment1);
+ publishSegment(wikiSegment2);
}
@Before
- public void setUp()
+ public void setUp() throws Exception
{
- connector = derbyConnectorRule.getConnector();
- SegmentsMetadataManagerConfig config = new SegmentsMetadataManagerConfig(Period.seconds(3), null);
- storageConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
-
- segmentSchemaCache = new SegmentSchemaCache(NoopServiceEmitter.instance());
- segmentSchemaManager = new SegmentSchemaManager(
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- jsonMapper,
- connector
- );
-
- sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
- jsonMapper,
- Suppliers.ofInstance(config),
- derbyConnectorRule.metadataTablesConfigSupplier(),
- connector,
- segmentSchemaCache,
- CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
- );
- sqlSegmentsMetadataManager.start();
-
- connector.createSegmentSchemasTable();
- connector.createSegmentTable();
+ setUp(derbyConnectorRule);
}
@After
- public void teardown()
+ public void tearDown()
{
- if (sqlSegmentsMetadataManager.isPollingDatabasePeriodically()) {
- sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
- }
- sqlSegmentsMetadataManager.stop();
+ teardownManager();
}
@Test
@@ -151,18 +110,10 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
sqlSegmentsMetadataManager.poll();
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
Assert.assertTrue(
- sqlSegmentsMetadataManager.retrieveAllDataSourceNames().isEmpty()
- );
- Assert.assertEquals(
- 0,
sqlSegmentsMetadataManager
- .getImmutableDataSourcesWithAllUsedSegments()
- .stream()
- .map(ImmutableDruidDataSource::getName).count()
- );
- Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.WIKI));
- Assert.assertTrue(
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments()).isEmpty()
+ .getRecentDataSourcesSnapshot()
+ .getDataSourcesWithAllUsedSegments()
+ .isEmpty()
);
}
@@ -170,18 +121,14 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
public void testPollPeriodically()
{
publishWikiSegments();
- DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertNull(dataSourcesSnapshot);
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
// This call make sure that the first poll is completed
sqlSegmentsMetadataManager.useLatestSnapshotIfWithinDelay();
Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
- Assert.assertEquals(
- ImmutableSet.of(TestDataSource.WIKI),
- sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
- );
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableList.of(TestDataSource.WIKI),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
@@ -195,7 +142,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
);
Assert.assertEquals(
ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(dataSourcesSnapshot.iterateAllUsedSegmentsInSnapshot())
+ retrieveAllUsedSegments()
);
}
@@ -203,7 +150,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
public void testPollOnDemand()
{
publishWikiSegments();
- DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertNull(dataSourcesSnapshot);
// This should return false and not wait/poll anything as we did not schedule periodic poll
Assert.assertFalse(sqlSegmentsMetadataManager.useLatestSnapshotIfWithinDelay());
@@ -212,11 +159,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
sqlSegmentsMetadataManager.forceOrWaitOngoingDatabasePoll();
Assert.assertFalse(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.OnDemandDatabasePoll);
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
- Assert.assertEquals(
- ImmutableSet.of(TestDataSource.WIKI),
- sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
- );
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableList.of(TestDataSource.WIKI),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
@@ -230,7 +173,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
);
Assert.assertEquals(
ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(dataSourcesSnapshot.iterateAllUsedSegmentsInSnapshot())
+ retrieveAllUsedSegments()
);
}
@@ -238,14 +181,14 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
public void testPollPeriodicallyAndOnDemandInterleave() throws Exception
{
publishWikiSegments();
- DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ DataSourcesSnapshot dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertNull(dataSourcesSnapshot);
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
// This call make sure that the first poll is completed
sqlSegmentsMetadataManager.useLatestSnapshotIfWithinDelay();
Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableList.of(TestDataSource.WIKI),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
@@ -260,7 +203,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.OnDemandDatabasePoll);
// New datasource should now be in the snapshot since we just force on demand poll.
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableList.of(TestDataSource.KOALA, TestDataSource.WIKI),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
@@ -273,12 +216,12 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
publishSegment(createNewSegment1(newDataSource3));
// This time wait for periodic poll (not doing on demand poll so we have to wait a bit...)
- while (sqlSegmentsMetadataManager.getDataSourcesSnapshot().getDataSource(newDataSource3) == null) {
+ while (sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot().getDataSource(newDataSource3) == null) {
Thread.sleep(1000);
}
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
Assert.assertTrue(sqlSegmentsMetadataManager.getLatestDatabasePoll() instanceof SqlSegmentsMetadataManager.PeriodicDatabasePoll);
- dataSourcesSnapshot = sqlSegmentsMetadataManager.getDataSourcesSnapshot();
+ dataSourcesSnapshot = sqlSegmentsMetadataManager.getLatestDataSourcesSnapshot();
Assert.assertEquals(
ImmutableSet.of(TestDataSource.KOALA, "wikipedia3", TestDataSource.WIKI),
dataSourcesSnapshot.getDataSourcesWithAllUsedSegments()
@@ -289,29 +232,29 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
}
@Test
- public void testPrepareImmutableDataSourceWithUsedSegmentsAwaitsPollOnRestart() throws IOException
+ public void testPrepareImmutableDataSourceWithUsedSegmentsAwaitsPollOnRestart()
{
publishWikiSegments();
DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
Assert.assertEquals(
- ImmutableSet.of(koalaSegment),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA).getSegments())
+ Set.of(koalaSegment),
+ Set.copyOf(sqlSegmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSource(TestDataSource.KOALA).getSegments())
);
}
@Test
- public void testGetDataSourceWithUsedSegmentsAwaitsPollOnRestart() throws IOException
+ public void testGetDataSourceWithUsedSegmentsAwaitsPollOnRestart()
{
publishWikiSegments();
DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
Assert.assertEquals(
- ImmutableSet.of(koalaSegment),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA).getSegments())
+ Set.of(koalaSegment),
+ Set.copyOf(sqlSegmentsMetadataManager.getRecentDataSourcesSnapshot().getDataSource(TestDataSource.KOALA).getSegments())
);
}
@Test
- public void testPrepareImmutableDataSourcesWithAllUsedSegmentsAwaitsPollOnRestart() throws IOException
+ public void testPrepareImmutableDataSourcesWithAllUsedSegmentsAwaitsPollOnRestart()
{
publishWikiSegments();
DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
@@ -319,7 +262,8 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment),
ImmutableSet.copyOf(
sqlSegmentsMetadataManager
- .getImmutableDataSourcesWithAllUsedSegments()
+ .getRecentDataSourcesSnapshot()
+ .getDataSourcesWithAllUsedSegments()
.stream()
.flatMap((ImmutableDruidDataSource dataSource) -> dataSource.getSegments().stream())
.iterator()
@@ -328,26 +272,22 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
}
@Test
- public void testIterateAllUsedSegmentsAwaitsPollOnRestart() throws IOException
+ public void testIterateAllUsedSegmentsAwaitsPollOnRestart()
{
publishWikiSegments();
DataSegment koalaSegment = pollThenStopThenPublishKoalaSegment();
Assert.assertEquals(
ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
+ retrieveAllUsedSegments()
);
}
- private DataSegment pollThenStopThenPublishKoalaSegment() throws IOException
+ private DataSegment pollThenStopThenPublishKoalaSegment()
{
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
sqlSegmentsMetadataManager.poll();
sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
Assert.assertFalse(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
- Assert.assertEquals(
- ImmutableSet.of(TestDataSource.WIKI),
- sqlSegmentsMetadataManager.retrieveAllDataSourceNames()
- );
final DataSegment koalaSegment = createNewSegment1(TestDataSource.KOALA);
publishSegment(koalaSegment);
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
@@ -360,7 +300,7 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
* even if one of the entries is corrupted.
*/
@Test
- public void testPollWithCorruptedSegment() throws IOException
+ public void testPollWithCorruptedSegment()
{
publishWikiSegments();
@@ -372,158 +312,13 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
+ DataSourcesSnapshot snapshot = sqlSegmentsMetadataManager.getRecentDataSourcesSnapshot();
Assert.assertEquals(
TestDataSource.WIKI,
- Iterables.getOnlyElement(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).getName()
- );
- }
-
- @Test
- public void testGetUnusedSegmentIntervals() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
-
- // Allow null values of used_status_last_updated to test upgrade from older Druid versions
- allowUsedFlagLastUpdatedToBeNullable();
-
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
- int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(TestDataSource.WIKI);
- Assert.assertEquals(2, numChangedSegments);
-
- // Publish an unused segment with used_status_last_updated 2 hours ago
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
- publishUnusedSegments(koalaSegment1);
- derbyConnectorRule.segments().updateUsedStatusLastUpdated(
- koalaSegment1.getId().toString(),
- DateTimes.nowUtc().minus(Duration.standardHours(2))
- );
-
- // Publish an unused segment with used_status_last_updated 2 days ago
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-16T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
- publishUnusedSegments(koalaSegment2);
- derbyConnectorRule.segments().updateUsedStatusLastUpdated(
- koalaSegment2.getId().toString(),
- DateTimes.nowUtc().minus(Duration.standardDays(2))
- );
-
- // Publish an unused segment and set used_status_last_updated to null
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
- publishUnusedSegments(koalaSegment3);
- updateUsedStatusLastUpdatedToNull(koalaSegment3);
-
- Assert.assertEquals(
- ImmutableList.of(wikiSegment2.getInterval()),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- null,
- DateTimes.of("3000"),
- 1,
- DateTimes.COMPARE_DATE_AS_STRING_MAX
- )
- );
-
- // Test the DateTime maxEndTime argument of getUnusedSegmentIntervals
- Assert.assertEquals(
- ImmutableList.of(wikiSegment2.getInterval()),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- null,
- DateTimes.of(2012, 1, 7, 0, 0),
- 1,
- DateTimes.COMPARE_DATE_AS_STRING_MAX
- )
- );
- Assert.assertEquals(
- ImmutableList.of(wikiSegment1.getInterval()),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- DateTimes.of(2012, 1, 7, 0, 0),
- DateTimes.of(2012, 4, 7, 0, 0),
- 1,
- DateTimes.COMPARE_DATE_AS_STRING_MAX
- )
- );
- Assert.assertEquals(
- ImmutableList.of(),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- DateTimes.of(2012, 1, 7, 0, 0),
- DateTimes.of(2012, 1, 7, 0, 0),
- 1,
- DateTimes.COMPARE_DATE_AS_STRING_MAX
- )
- );
-
- Assert.assertEquals(
- ImmutableList.of(wikiSegment2.getInterval(), wikiSegment1.getInterval()),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- null,
- DateTimes.of("3000"),
- 5,
- DateTimes.COMPARE_DATE_AS_STRING_MAX
- )
- );
-
- // Test a buffer period that should exclude some segments
-
- // The wikipedia datasource has segments generated with last used time equal to roughly the time of test run. None of these segments should be selected with a bufer period of 1 day
- Assert.assertEquals(
- ImmutableList.of(),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.WIKI,
- DateTimes.COMPARE_DATE_AS_STRING_MIN,
- DateTimes.of("3000"),
- 5,
- DateTimes.nowUtc().minus(Duration.parse("PT86400S"))
- )
- );
-
- // koalaSegment3 has a null used_status_last_updated which should mean getUnusedSegmentIntervals never returns it
- // koalaSegment2 has a used_status_last_updated older than 1 day which means it should be returned
- // The last of the 3 segments in koala has a used_status_last_updated date less than one day and should not be returned
- Assert.assertEquals(
- ImmutableList.of(koalaSegment2.getInterval()),
- sqlSegmentsMetadataManager.getUnusedSegmentIntervals(
- TestDataSource.KOALA,
- DateTimes.COMPARE_DATE_AS_STRING_MIN,
- DateTimes.of("3000"),
- 5,
- DateTimes.nowUtc().minus(Duration.parse("PT86400S"))
- )
+ Iterables.getOnlyElement(snapshot.getDataSourcesWithAllUsedSegments()).getName()
);
}
- @Test(timeout = 60_000)
- public void testMarkAsUnusedAllSegmentsInDataSource() throws IOException, InterruptedException
- {
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- publishSegment(createNewSegment1(TestDataSource.KOALA));
-
- awaitDataSourceAppeared(TestDataSource.KOALA);
- int numChangedSegments = sqlSegmentsMetadataManager.markAsUnusedAllSegmentsInDataSource(TestDataSource.KOALA);
- Assert.assertEquals(1, numChangedSegments);
- awaitDataSourceDisappeared(TestDataSource.KOALA);
- Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
- }
-
private static DataSegment createNewSegment1(String datasource)
{
return createSegment(
@@ -542,894 +337,153 @@ public class SqlSegmentsMetadataManagerTest extends SqlSegmentsMetadataManagerTe
);
}
- @Test(timeout = 60_000)
- public void testMarkSegmentAsUnused() throws IOException, InterruptedException
+ @Test
+ public void test_poll_doesNotRetrieveUnusedSegments()
{
+ publishWikiSegments();
+ final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
+ final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
+
+ publishSegment(koalaSegment1);
+ publishSegment(koalaSegment2);
+
+ // Poll all segments
sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
sqlSegmentsMetadataManager.poll();
Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
+ retrieveAllUsedSegments()
);
- publishSegment(koalaSegment);
- awaitDataSourceAppeared(TestDataSource.KOALA);
- Assert.assertNotNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
-
- Assert.assertTrue(sqlSegmentsMetadataManager.markSegmentAsUnused(koalaSegment.getId()));
- awaitDataSourceDisappeared(TestDataSource.KOALA);
- Assert.assertNull(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(TestDataSource.KOALA));
- }
+ // Mark the koala segments as unused
+ Assert.assertEquals(2, markSegmentsAsUnused(koalaSegment1.getId(), koalaSegment2.getId()));
- private void awaitDataSourceAppeared(String datasource) throws InterruptedException
- {
- while (sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(datasource) == null) {
- Thread.sleep(5);
- }
+ // Verify that subsequent poll only retrieves the used segments
+ sqlSegmentsMetadataManager.poll();
+ Assert.assertEquals(
+ Set.of(wikiSegment1, wikiSegment2),
+ retrieveAllUsedSegments()
+ );
}
- private void awaitDataSourceDisappeared(String dataSource) throws InterruptedException
+ @Test
+ public void testStopAndStart()
{
- while (sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(dataSource) != null) {
- Thread.sleep(5);
- }
+ // Simulate successive losing and getting the coordinator leadership
+ sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
+ sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
+ sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
+ sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
}
@Test
- public void testMarkAsUsedNonOvershadowedSegments() throws Exception
+ public void testIterateAllUsedNonOvershadowedSegmentsForDatasourceInterval()
{
publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
+ final Interval theInterval = Intervals.of("2012-03-15T00:00:00.000/2012-03-20T00:00:00.000");
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
+ // Re-create SqlSegmentsMetadataManager with a higher poll duration
+ final SegmentsMetadataManagerConfig config = new SegmentsMetadataManagerConfig(Period.seconds(1), null);
+ sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
+ jsonMapper,
+ Suppliers.ofInstance(config),
+ derbyConnectorRule.metadataTablesConfigSupplier(),
+ derbyConnectorRule.getConnector(),
+ segmentSchemaCache,
+ CentralizedDatasourceSchemaConfig.create(),
+ NoopServiceEmitter.instance()
);
+ sqlSegmentsMetadataManager.start();
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
+ Set<DataSegment> segments = sqlSegmentsMetadataManager
+ .getRecentDataSourcesSnapshot()
+ .getAllUsedNonOvershadowedSegments(TestDataSource.WIKI, theInterval);
+ Assert.assertEquals(Set.of(wikiSegment1), segments);
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
+ final DataSegment wikiSegment3 = createSegment(
+ TestDataSource.WIKI,
+ "2012-03-16T00:00:00.000/2012-03-17T00:00:00.000",
"2017-10-15T20:19:12.565Z"
);
+ publishSegment(wikiSegment3);
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
- final Set<SegmentId> segmentIds = ImmutableSet.of(
- koalaSegment1.getId(),
- koalaSegment2.getId(),
- koalaSegment3.getId()
- );
+ // New segment is not returned since we call without force poll
+ segments = sqlSegmentsMetadataManager
+ .getRecentDataSourcesSnapshot()
+ .getAllUsedNonOvershadowedSegments(TestDataSource.WIKI, theInterval);
+ Assert.assertEquals(Set.of(wikiSegment1), segments);
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(TestDataSource.KOALA, segmentIds));
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
+ // New segment is returned since we call with force poll
+ segments = sqlSegmentsMetadataManager
+ .forceUpdateDataSourcesSnapshot()
+ .getAllUsedNonOvershadowedSegments(TestDataSource.WIKI, theInterval);
+ Assert.assertEquals(Set.of(wikiSegment1, wikiSegment3), segments);
}
@Test
- public void testMarkAsUsedNonOvershadowedSegmentsInEternityIntervalWithVersions() throws Exception
+ public void testPopulateUsedFlagLastUpdated()
{
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
+ allowUsedFlagLastUpdatedToBeNullable();
+ final DataSegment koalaSegment = createSegment(
TestDataSource.KOALA,
"2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
"2017-10-15T20:19:12.565Z"
);
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(
- 2,
- sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
- TestDataSource.KOALA,
- Intervals.ETERNITY,
- ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
- )
- );
- sqlSegmentsMetadataManager.poll();
+ publishUnusedSegments(koalaSegment);
+ updateUsedStatusLastUpdatedToNull(koalaSegment);
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
+ Assert.assertEquals(1, getCountOfRowsWithLastUsedNull());
+ sqlSegmentsMetadataManager.populateUsedFlagLastUpdated();
+ Assert.assertEquals(0, getCountOfRowsWithLastUsedNull());
}
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInIntervalWithEmptyVersions() throws Exception
+ private int getCountOfRowsWithLastUsedNull()
{
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
+ return derbyConnectorRule.getConnector().retryWithHandle(
+ handle -> handle.select(
+ StringUtils.format(
+ "SELECT ID FROM %1$s WHERE USED_STATUS_LAST_UPDATED IS NULL",
+ derbyConnectorRule.segments().getTableName()
+ )
+ ).size()
);
+ }
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
+ private void updateSegmentPayload(DataSegment segment, byte[] payload)
+ {
+ derbyConnectorRule.segments().update(
+ "UPDATE %1$s SET PAYLOAD = ? WHERE ID = ?",
+ payload,
+ segment.getId().toString()
);
+ }
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
- TestDataSource.KOALA,
- Intervals.of("2017/2018"),
- ImmutableList.of()
- )
+ private void updateUsedStatusLastUpdatedToNull(DataSegment segment)
+ {
+ derbyConnectorRule.segments().update(
+ "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = NULL WHERE ID = ?",
+ segment.getId().toString()
);
- sqlSegmentsMetadataManager.poll();
+ }
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
+ /**
+ * Alters the column used_status_last_updated to be nullable. This is used to
+ * test backward compatibility with versions of Druid without this column
+ * present in the segments table.
+ */
+ private void allowUsedFlagLastUpdatedToBeNullable()
+ {
+ derbyConnectorRule.segments().update(
+ "ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL"
);
}
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInEternityIntervalWithEmptyVersions() throws Exception
+
+ private Set<DataSegment> retrieveAllUsedSegments()
{
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
- TestDataSource.KOALA,
- Intervals.ETERNITY,
- ImmutableList.of()
- )
- );
- sqlSegmentsMetadataManager.poll();
-
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInFiniteIntervalWithVersions() throws Exception
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(
- 2,
- sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
- TestDataSource.KOALA,
- Intervals.of("2017-10-15/2017-10-18"),
- ImmutableList.of("2017-10-15T20:19:12.565Z", "2017-10-16T20:19:12.565Z")
- )
- );
- sqlSegmentsMetadataManager.poll();
-
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsWithNonExistentVersions() throws Exception
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3);
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(
- TestDataSource.KOALA,
- Intervals.ETERNITY,
- ImmutableList.of("foo", "bar")
- )
- );
- sqlSegmentsMetadataManager.poll();
-
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInvalidDataSource() throws Exception
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
- final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
-
- publishUnusedSegments(koalaSegment1, koalaSegment2);
- final ImmutableSet<SegmentId> segmentIds =
- ImmutableSet.of(koalaSegment1.getId(), koalaSegment2.getId());
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
-
- MatcherAssert.assertThat(
- Assert.assertThrows(
- DruidException.class,
- () -> sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments("wrongDataSource", segmentIds)
- ),
- DruidExceptionMatcher
- .invalidInput()
- .expectMessageContains("Could not find segment IDs")
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsWithInvalidSegmentIds()
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
- final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
-
- final ImmutableSet<SegmentId> segmentIds =
- ImmutableSet.of(koalaSegment1.getId(), koalaSegment2.getId());
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
-
- MatcherAssert.assertThat(
- Assert.assertThrows(
- DruidException.class,
- () -> sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegments(TestDataSource.KOALA, segmentIds)
- ),
- DruidExceptionMatcher
- .invalidInput()
- .expectMessageContains("Could not find segment IDs")
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInInterval() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
-
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
- final Interval theInterval = Intervals.of("2017-10-15T00:00:00.000/2017-10-18T00:00:00.000");
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
-
- // 2 out of 3 segments match the interval
- Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUsedNonOvershadowedSegmentsInIntervalWithOverlappingInterval() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-16T20:19:12.565Z"
- );
-
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-22T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- // Overshadowed by koalaSegment2
- final DataSegment koalaSegment4 = createNewSegment2(TestDataSource.KOALA);
-
- publishUnusedSegments(koalaSegment1, koalaSegment2, koalaSegment3, koalaSegment4);
- final Interval theInterval = Intervals.of("2017-10-16T00:00:00.000/2017-10-20T00:00:00.000");
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
-
- // 1 out of 3 segments match the interval, other 2 overlap, only the segment fully contained will be marked unused
- Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUsedNonOvershadowedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkSegmentsAsUnused() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
- final DataSegment koalaSegment2 = createNewSegment1(TestDataSource.KOALA);
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
-
- final ImmutableSet<SegmentId> segmentIds =
- ImmutableSet.of(koalaSegment1.getId(), koalaSegment1.getId());
-
- Assert.assertEquals(segmentIds.size(), sqlSegmentsMetadataManager.markSegmentsAsUnused(segmentIds));
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
+ return Sets.newHashSet(
+ sqlSegmentsMetadataManager.getRecentDataSourcesSnapshot().iterateAllUsedSegmentsInSnapshot()
);
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInInterval() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createNewSegment1(TestDataSource.KOALA);
- final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-15T00:00:00.000/2017-10-18T00:00:00.000");
-
- // 2 out of 3 segments match the interval
- Assert.assertEquals(2, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInIntervalAndVersions() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DateTime now = DateTimes.nowUtc();
- final String v1 = now.toString();
- final String v2 = now.plus(Duration.standardDays(1)).toString();
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- v1
- );
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- v2
- );
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- v2
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-15/2017-10-18");
-
- Assert.assertEquals(
- 2,
- sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
- TestDataSource.KOALA,
- theInterval,
- ImmutableList.of(v1, v2)
- )
- );
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInIntervalAndNonExistentVersions() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DateTime now = DateTimes.nowUtc();
- final String v1 = now.toString();
- final String v2 = now.plus(Duration.standardDays(1)).toString();
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- v1
- );
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- v2
- );
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- v2
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-15/2017-10-18");
-
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
- TestDataSource.KOALA,
- theInterval,
- ImmutableList.of("foo", "bar", "baz")
- )
- );
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInIntervalWithEmptyVersions() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DateTime now = DateTimes.nowUtc();
- final String v1 = now.toString();
- final String v2 = now.plus(Duration.standardDays(1)).toString();
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- v1
- );
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- v2
- );
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- v2
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-15/2017-10-18");
-
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
- TestDataSource.KOALA,
- theInterval,
- ImmutableList.of()
- )
- );
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInEternityIntervalWithEmptyVersions() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DateTime now = DateTimes.nowUtc();
- final String v1 = now.toString();
- final String v2 = now.plus(Duration.standardDays(1)).toString();
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-16T00:00:00.000",
- v1
- );
- final DataSegment koalaSegment2 = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- v2
- );
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-20T00:00:00.000",
- v2
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-15/2017-10-18");
-
- Assert.assertEquals(
- 0,
- sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(
- TestDataSource.KOALA,
- theInterval,
- ImmutableList.of()
- )
- );
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment2, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testMarkAsUnusedSegmentsInIntervalWithOverlappingInterval() throws IOException
- {
- publishWikiSegments();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.poll();
- Assert.assertTrue(sqlSegmentsMetadataManager.isPollingDatabasePeriodically());
-
- final DataSegment koalaSegment1 = createSegment(
- TestDataSource.KOALA,
- "2017-10-15T00:00:00.000/2017-10-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
- final DataSegment koalaSegment2 = createNewSegment2(TestDataSource.KOALA);
- final DataSegment koalaSegment3 = createSegment(
- TestDataSource.KOALA,
- "2017-10-19T00:00:00.000/2017-10-22T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishSegment(koalaSegment1);
- publishSegment(koalaSegment2);
- publishSegment(koalaSegment3);
- final Interval theInterval = Intervals.of("2017-10-16T00:00:00.000/2017-10-20T00:00:00.000");
-
- // 1 out of 3 segments match the interval, other 2 overlap, only the segment fully contained will be marked unused
- Assert.assertEquals(1, sqlSegmentsMetadataManager.markAsUnusedSegmentsInInterval(TestDataSource.KOALA, theInterval, null));
-
- sqlSegmentsMetadataManager.poll();
- Assert.assertEquals(
- ImmutableSet.of(wikiSegment1, wikiSegment2, koalaSegment1, koalaSegment3),
- ImmutableSet.copyOf(sqlSegmentsMetadataManager.iterateAllUsedSegments())
- );
- }
-
- @Test
- public void testStopAndStart()
- {
- // Simulate successive losing and getting the coordinator leadership
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.startPollingDatabasePeriodically();
- sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
- }
-
- @Test
- public void testIterateAllUsedNonOvershadowedSegmentsForDatasourceInterval() throws Exception
- {
- publishWikiSegments();
- final Interval theInterval = Intervals.of("2012-03-15T00:00:00.000/2012-03-20T00:00:00.000");
-
- // Re-create SqlSegmentsMetadataManager with a higher poll duration
- final SegmentsMetadataManagerConfig config = new SegmentsMetadataManagerConfig(Period.seconds(1), null);
- sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
- jsonMapper,
- Suppliers.ofInstance(config),
- derbyConnectorRule.metadataTablesConfigSupplier(),
- derbyConnectorRule.getConnector(),
- segmentSchemaCache,
- CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
- );
- sqlSegmentsMetadataManager.start();
-
- Optional<Iterable<DataSegment>> segments = sqlSegmentsMetadataManager
- .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, true);
- Assert.assertTrue(segments.isPresent());
- Set<DataSegment> dataSegmentSet = ImmutableSet.copyOf(segments.get());
- Assert.assertEquals(1, dataSegmentSet.size());
- Assert.assertTrue(dataSegmentSet.contains(wikiSegment1));
-
- final DataSegment wikiSegment3 = createSegment(
- TestDataSource.WIKI,
- "2012-03-16T00:00:00.000/2012-03-17T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
- publishSegment(wikiSegment3);
-
- // New segment is not returned since we call without force poll
- segments = sqlSegmentsMetadataManager
- .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, false);
- Assert.assertTrue(segments.isPresent());
- dataSegmentSet = ImmutableSet.copyOf(segments.get());
- Assert.assertEquals(1, dataSegmentSet.size());
- Assert.assertTrue(dataSegmentSet.contains(wikiSegment1));
-
- // New segment is returned since we call with force poll
- segments = sqlSegmentsMetadataManager
- .iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(TestDataSource.WIKI, theInterval, true);
- Assert.assertTrue(segments.isPresent());
- dataSegmentSet = ImmutableSet.copyOf(segments.get());
- Assert.assertEquals(2, dataSegmentSet.size());
- Assert.assertTrue(dataSegmentSet.contains(wikiSegment1));
- Assert.assertTrue(dataSegmentSet.contains(wikiSegment3));
- }
-
- @Test
- public void testPopulateUsedFlagLastUpdated() throws IOException
- {
- allowUsedFlagLastUpdatedToBeNullable();
- final DataSegment koalaSegment = createSegment(
- TestDataSource.KOALA,
- "2017-10-17T00:00:00.000/2017-10-18T00:00:00.000",
- "2017-10-15T20:19:12.565Z"
- );
-
- publishUnusedSegments(koalaSegment);
- updateUsedStatusLastUpdatedToNull(koalaSegment);
-
- Assert.assertEquals(1, getCountOfRowsWithLastUsedNull());
- sqlSegmentsMetadataManager.populateUsedFlagLastUpdated();
- Assert.assertEquals(0, getCountOfRowsWithLastUsedNull());
- }
-
- private int getCountOfRowsWithLastUsedNull()
- {
- return derbyConnectorRule.getConnector().retryWithHandle(
- handle -> handle.select(
- StringUtils.format(
- "SELECT ID FROM %1$s WHERE USED_STATUS_LAST_UPDATED IS NULL",
- derbyConnectorRule.segments().getTableName()
- )
- ).size()
- );
- }
-
- private void updateSegmentPayload(DataSegment segment, byte[] payload)
- {
- derbyConnectorRule.segments().update(
- "UPDATE %1$s SET PAYLOAD = ? WHERE ID = ?",
- payload,
- segment.getId().toString()
- );
- }
-
- private void updateUsedStatusLastUpdatedToNull(DataSegment segment)
- {
- derbyConnectorRule.segments().update(
- "UPDATE %1$s SET USED_STATUS_LAST_UPDATED = NULL WHERE ID = ?",
- segment.getId().toString()
- );
- }
-
- /**
- * Alters the column used_status_last_updated to be nullable. This is used to
- * test backward compatibility with versions of Druid without this column
- * present in the segments table.
- */
- private void allowUsedFlagLastUpdatedToBeNullable()
- {
- derbyConnectorRule.segments().update(
- "ALTER TABLE %1$s ALTER COLUMN USED_STATUS_LAST_UPDATED NULL"
- );
- }
+ }
}
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTestBase.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTestBase.java
index 18112c29b4c..1f7b0d8247c 100644
--- a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTestBase.java
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataManagerTestBase.java
@@ -20,22 +20,28 @@
package org.apache.druid.metadata;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.TestHelper;
+import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
import org.apache.druid.segment.metadata.SegmentSchemaCache;
import org.apache.druid.segment.metadata.SegmentSchemaManager;
+import org.apache.druid.server.metrics.NoopServiceEmitter;
import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.partition.NoneShardSpec;
+import org.joda.time.DateTime;
+import org.joda.time.Period;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.tweak.HandleCallback;
-import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class SqlSegmentsMetadataManagerTestBase
{
@@ -64,11 +70,66 @@ public class SqlSegmentsMetadataManagerTestBase
0
);
- protected void publishSegment(final DataSegment segment) throws IOException
+ protected void setUp(TestDerbyConnector.DerbyConnectorRule derbyConnectorRule) throws Exception
+ {
+ config = new SegmentsMetadataManagerConfig(Period.seconds(3), null);
+ connector = derbyConnectorRule.getConnector();
+ storageConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
+
+ segmentSchemaCache = new SegmentSchemaCache(NoopServiceEmitter.instance());
+ segmentSchemaManager = new SegmentSchemaManager(
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
+ jsonMapper,
+ connector
+ );
+
+ sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
+ jsonMapper,
+ Suppliers.ofInstance(config),
+ derbyConnectorRule.metadataTablesConfigSupplier(),
+ connector,
+ segmentSchemaCache,
+ CentralizedDatasourceSchemaConfig.create(),
+ NoopServiceEmitter.instance()
+ );
+ sqlSegmentsMetadataManager.start();
+
+ connector.createSegmentSchemasTable();
+ connector.createSegmentTable();
+ }
+
+ protected void teardownManager()
+ {
+ if (sqlSegmentsMetadataManager.isPollingDatabasePeriodically()) {
+ sqlSegmentsMetadataManager.stopPollingDatabasePeriodically();
+ }
+ sqlSegmentsMetadataManager.stop();
+ }
+
+ protected void publishSegment(final DataSegment segment)
{
publishSegment(connector, storageConfig, jsonMapper, segment);
}
+ protected int markSegmentsAsUnused(SegmentId... segmentIds)
+ {
+ return markSegmentsAsUnused(Set.of(segmentIds), connector, storageConfig, jsonMapper, DateTimes.nowUtc());
+ }
+
+ public static int markSegmentsAsUnused(
+ Set<SegmentId> segmentIds,
+ SQLMetadataConnector connector,
+ MetadataStorageTablesConfig storageConfig,
+ ObjectMapper jsonMapper,
+ DateTime updateTime
+ )
+ {
+ return connector.retryWithHandle(
+ handle -> SqlSegmentsMetadataQuery.forHandle(handle, connector, storageConfig, jsonMapper)
+ .markSegmentsAsUnused(segmentIds, updateTime)
+ );
+ }
+
protected static DataSegment createSegment(
String dataSource,
String interval,
@@ -99,46 +160,15 @@ public class SqlSegmentsMetadataManagerTestBase
final MetadataStorageTablesConfig config,
final ObjectMapper jsonMapper,
final DataSegment segment
- ) throws IOException
- {
- String now = DateTimes.nowUtc().toString();
- publishSegment(
- connector,
- config,
- segment.getId().toString(),
- segment.getDataSource(),
- now,
- segment.getInterval().getStart().toString(),
- segment.getInterval().getEnd().toString(),
- (segment.getShardSpec() instanceof NoneShardSpec) ? false : true,
- segment.getVersion(),
- true,
- jsonMapper.writeValueAsBytes(segment),
- now
- );
- }
-
- private static void publishSegment(
- final SQLMetadataConnector connector,
- final MetadataStorageTablesConfig config,
- final String segmentId,
- final String dataSource,
- final String createdDate,
- final String start,
- final String end,
- final boolean partitioned,
- final String version,
- final boolean used,
- final byte[] payload,
- final String usedFlagLastUpdated
)
{
+ final String now = DateTimes.nowUtc().toString();
try {
final DBI dbi = connector.getDBI();
List<Map<String, Object>> exists = dbi.withHandle(
handle ->
handle.createQuery(StringUtils.format("SELECT id FROM %s WHERE id=:id", config.getSegmentsTable()))
- .bind("id", segmentId)
+ .bind("id", segment.getId().toString())
.list()
);
@@ -153,19 +183,20 @@ public class SqlSegmentsMetadataManagerTestBase
connector.getQuoteString()
);
+ final byte[] payload = jsonMapper.writeValueAsBytes(segment);
dbi.withHandle(
(HandleCallback<Void>) handle -> {
handle.createStatement(publishStatement)
- .bind("id", segmentId)
- .bind("dataSource", dataSource)
- .bind("created_date", createdDate)
- .bind("start", start)
- .bind("end", end)
- .bind("partitioned", partitioned)
- .bind("version", version)
- .bind("used", used)
+ .bind("id", segment.getId().toString())
+ .bind("dataSource", segment.getDataSource())
+ .bind("created_date", now)
+ .bind("start", segment.getInterval().getStart().toString())
+ .bind("end", segment.getInterval().getEnd().toString())
+ .bind("partitioned", !(segment.getShardSpec() instanceof NoneShardSpec))
+ .bind("version", segment.getVersion())
+ .bind("used", true)
.bind("payload", payload)
- .bind("used_status_last_updated", usedFlagLastUpdated)
+ .bind("used_status_last_updated", now)
.execute();
return null;
diff --git a/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java
new file mode 100644
index 00000000000..9dd1b916c4e
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/metadata/SqlSegmentsMetadataQueryTest.java
@@ -0,0 +1,317 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.metadata.storage.derby.DerbyConnector;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.segment.TestHelper;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.SegmentId;
+import org.joda.time.DateTime;
+import org.joda.time.Interval;
+import org.joda.time.Period;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+public class SqlSegmentsMetadataQueryTest
+{
+ @Rule
+ public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule
+ = new TestDerbyConnector.DerbyConnectorRule();
+
+ private static final DateTime JAN_1 = DateTimes.of("2025-01-01");
+ private static final String V1 = JAN_1.toString();
+ private static final String V2 = JAN_1.plusDays(1).toString();
+
+ private static final List<DataSegment> WIKI_SEGMENTS_2X5D
+ = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
+ .forIntervals(5, Granularities.DAY)
+ .withNumPartitions(2)
+ .startingAt(JAN_1)
+ .withVersion(V1)
+ .eachOfSizeInMb(500);
+
+ @Before
+ public void setUp()
+ {
+ derbyConnectorRule.getConnector().createSegmentTable();
+ insertSegments(WIKI_SEGMENTS_2X5D.toArray(new DataSegment[0]));
+ }
+
+ @Test
+ public void test_markSegmentsAsUnused()
+ {
+ // Check segments currently present in the metadata store
+ Assert.assertEquals(Set.copyOf(WIKI_SEGMENTS_2X5D), retrieveAllUsedSegments());
+ Assert.assertTrue(retrieveAllUnusedSegments().isEmpty());
+
+ // Mark segments as unused and verify the results
+ final Set<DataSegment> segmentsToUpdate = Set.of(WIKI_SEGMENTS_2X5D.get(0), WIKI_SEGMENTS_2X5D.get(1));
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsAsUnused(getIds(segmentsToUpdate), DateTimes.nowUtc())
+ );
+ Assert.assertEquals(2, numUpdatedSegments);
+ Assert.assertEquals(segmentsToUpdate, retrieveAllUnusedSegments());
+
+ // Verify that these segments are not present in used segments set
+ Set<DataSegment> usedSegments = retrieveAllUsedSegments();
+ Assert.assertEquals(8, usedSegments.size());
+
+ segmentsToUpdate.forEach(
+ updatedSegment -> Assert.assertFalse(usedSegments.contains(updatedSegment))
+ );
+ }
+
+ @Test
+ public void test_markSegmentsAsUsed()
+ {
+ // Mark segments as unused and verify the results
+ final Set<DataSegment> segmentsToUpdate = Set.of(WIKI_SEGMENTS_2X5D.get(0), WIKI_SEGMENTS_2X5D.get(1));
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsAsUnused(getIds(segmentsToUpdate), DateTimes.nowUtc())
+ );
+ Assert.assertEquals(2, numUpdatedSegments);
+ Assert.assertEquals(segmentsToUpdate, retrieveAllUnusedSegments());
+
+ // Mark segments as used again and verify the results
+ numUpdatedSegments = update(
+ sql -> sql.markSegmentsAsUsed(getIds(segmentsToUpdate), DateTimes.nowUtc())
+ );
+ Assert.assertEquals(2, numUpdatedSegments);
+ Assert.assertEquals(Set.copyOf(WIKI_SEGMENTS_2X5D), retrieveAllUsedSegments());
+ Assert.assertTrue(retrieveAllUnusedSegments().isEmpty());
+ }
+
+ @Test
+ public void test_markSegmentsAsUnused_forEmptySegmentIds_isNoop()
+ {
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsAsUnused(Set.of(), DateTimes.nowUtc())
+ );
+ Assert.assertEquals(0, numUpdatedSegments);
+ Assert.assertEquals(Set.copyOf(WIKI_SEGMENTS_2X5D), retrieveAllUsedSegments());
+ }
+
+ @Test
+ public void test_markSegmentsUnused_forEternityInterval()
+ {
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsUnused(TestDataSource.WIKI, Intervals.ETERNITY, null, DateTimes.nowUtc())
+ );
+ Assert.assertEquals(WIKI_SEGMENTS_2X5D.size(), numUpdatedSegments);
+ Assert.assertEquals(Set.copyOf(WIKI_SEGMENTS_2X5D), retrieveAllUnusedSegments());
+ Assert.assertTrue(retrieveAllUsedSegments().isEmpty());
+ }
+
+ @Test
+ public void test_markSegmentsUnused_forSingleVersion()
+ {
+ // Insert v2 segments
+ insertSegments(
+ WIKI_SEGMENTS_2X5D.stream().map(
+ segment -> DataSegment.builder(segment).version(V2).build()
+ ).toArray(DataSegment[]::new)
+ );
+
+ // Update segments for 2 days
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsUnused(
+ TestDataSource.WIKI,
+ new Interval(JAN_1, Period.days(2)),
+ List.of(V1),
+ DateTimes.nowUtc()
+ )
+ );
+ Assert.assertEquals(4, numUpdatedSegments);
+ Assert.assertEquals(4, retrieveAllUnusedSegments().size());
+ Assert.assertEquals(16, retrieveAllUsedSegments().size());
+ }
+
+ @Test
+ public void test_markSegmentsUnused_forMultipleVersions()
+ {
+ // Insert v2 segments
+ insertSegments(
+ WIKI_SEGMENTS_2X5D.stream().map(
+ segment -> DataSegment.builder(segment).version(V2).build()
+ ).toArray(DataSegment[]::new)
+ );
+
+ // Update segments for 2 days
+ final List<String> versionsToUpdate = List.of(V1, V2);
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsUnused(
+ TestDataSource.WIKI,
+ new Interval(JAN_1, Period.days(2)),
+ versionsToUpdate,
+ DateTimes.nowUtc()
+ )
+ );
+ Assert.assertEquals(8, numUpdatedSegments);
+ Assert.assertEquals(8, retrieveAllUnusedSegments().size());
+ Assert.assertEquals(12, retrieveAllUsedSegments().size());
+ }
+
+ @Test
+ public void test_markSegmentsUnused_forAllVersions()
+ {
+ // Insert v2 segments
+ insertSegments(
+ WIKI_SEGMENTS_2X5D.stream().map(
+ segment -> DataSegment.builder(segment).version(V2).build()
+ ).toArray(DataSegment[]::new)
+ );
+
+ // Update segments for 2 days
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsUnused(
+ TestDataSource.WIKI,
+ new Interval(JAN_1, Period.days(2)),
+ null,
+ DateTimes.nowUtc()
+ )
+ );
+ Assert.assertEquals(8, numUpdatedSegments);
+ Assert.assertEquals(8, retrieveAllUnusedSegments().size());
+ Assert.assertEquals(12, retrieveAllUsedSegments().size());
+ }
+
+ @Test
+ public void test_markSegmentsUnused_forEmptyVersions_isNoop()
+ {
+ int numUpdatedSegments = update(
+ sql -> sql.markSegmentsUnused(TestDataSource.WIKI, Intervals.ETERNITY, List.of(), DateTimes.nowUtc())
+ );
+ Assert.assertEquals(0, numUpdatedSegments);
+ Assert.assertEquals(Set.copyOf(WIKI_SEGMENTS_2X5D), retrieveAllUsedSegments());
+ Assert.assertTrue(retrieveAllUnusedSegments().isEmpty());
+ }
+
+ @Test
+ public void test_retrieveSegmentForId()
+ {
+ final DataSegment segmentJan1 = WIKI_SEGMENTS_2X5D.get(0);
+ Assert.assertEquals(
+ segmentJan1,
+ read(sql -> sql.retrieveSegmentForId(segmentJan1.getId()))
+ );
+ }
+
+ @Test
+ public void test_retrieveSegmentForId_returnsNull_forUnknownId()
+ {
+ Assert.assertNull(
+ read(
+ sql -> sql.retrieveSegmentForId(SegmentId.dummy(TestDataSource.WIKI))
+ )
+ );
+ }
+
+ /**
+ * Reads segments from the metadata store using a
+ * {@link SqlSegmentsMetadataQuery} object.
+ */
+ private <T> T read(Function<SqlSegmentsMetadataQuery, T> function)
+ {
+ final DerbyConnector connector = derbyConnectorRule.getConnector();
+ final MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
+ return connector.inReadOnlyTransaction(
+ (handle, status) -> function.apply(
+ SqlSegmentsMetadataQuery.forHandle(handle, connector, tablesConfig, TestHelper.JSON_MAPPER)
+ )
+ );
+ }
+
+ /**
+ * Reads a set of segments from the metadata store using a
+ * {@link SqlSegmentsMetadataQuery} object.
+ */
+ private <T> Set<T> readAsSet(Function<SqlSegmentsMetadataQuery, CloseableIterator<T>> iterableReader)
+ {
+ final DerbyConnector connector = derbyConnectorRule.getConnector();
+ final MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
+
+ return connector.inReadOnlyTransaction((handle, status) -> {
+ final SqlSegmentsMetadataQuery query =
+ SqlSegmentsMetadataQuery.forHandle(handle, connector, tablesConfig, TestHelper.JSON_MAPPER);
+
+ try (CloseableIterator<T> iterator = iterableReader.apply(query)) {
+ return ImmutableSet.copyOf(iterator);
+ }
+ });
+ }
+
+ /**
+ * Executes an update using a {@link SqlSegmentsMetadataQuery} object.
+ */
+ private <T> T update(Function<SqlSegmentsMetadataQuery, T> function)
+ {
+ final DerbyConnector connector = derbyConnectorRule.getConnector();
+ final MetadataStorageTablesConfig tablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
+ return connector.retryWithHandle(
+ handle -> function.apply(
+ SqlSegmentsMetadataQuery.forHandle(handle, connector, tablesConfig, TestHelper.JSON_MAPPER)
+ )
+ );
+ }
+
+ private Set<DataSegment> retrieveAllUsedSegments()
+ {
+ return readAsSet(
+ sql -> sql.retrieveUsedSegments(TestDataSource.WIKI, List.of())
+ );
+ }
+
+ private Set<DataSegment> retrieveAllUnusedSegments()
+ {
+ return readAsSet(
+ sql -> sql.retrieveUnusedSegments(TestDataSource.WIKI, List.of(), null, null, null, null, null)
+ );
+ }
+
+ private void insertSegments(DataSegment... segments)
+ {
+ IndexerSqlMetadataStorageCoordinatorTestBase.insertUsedSegments(
+ Set.of(segments),
+ Map.of(),
+ derbyConnectorRule,
+ TestHelper.JSON_MAPPER
+ );
+ }
+
+ private static Set<SegmentId> getIds(Set<DataSegment> segments)
+ {
+ return segments.stream().map(DataSegment::getId).collect(Collectors.toSet());
+ }
+}
diff --git a/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java b/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java
new file mode 100644
index 00000000000..9f339ac59ae
--- /dev/null
+++ b/server/src/test/java/org/apache/druid/metadata/segment/SqlSegmentsMetadataManagerV2Test.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.metadata.segment;
+
+import com.google.common.base.Suppliers;
+import org.apache.druid.client.DataSourcesSnapshot;
+import org.apache.druid.error.ExceptionMatcher;
+import org.apache.druid.java.util.common.DateTimes;
+import org.apache.druid.java.util.common.granularity.Granularities;
+import org.apache.druid.java.util.metrics.StubServiceEmitter;
+import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
+import org.apache.druid.metadata.SqlSegmentsMetadataManagerTestBase;
+import org.apache.druid.metadata.TestDerbyConnector;
+import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache;
+import org.apache.druid.metadata.segment.cache.Metric;
+import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
+import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
+import org.apache.druid.segment.TestDataSource;
+import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
+import org.apache.druid.server.coordinator.CreateDataSegments;
+import org.apache.druid.server.coordinator.simulate.BlockingExecutorService;
+import org.apache.druid.server.coordinator.simulate.WrappingScheduledExecutorService;
+import org.apache.druid.timeline.DataSegment;
+import org.assertj.core.util.Sets;
+import org.hamcrest.MatcherAssert;
+import org.joda.time.DateTime;
+import org.joda.time.Period;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Set;
+
+public class SqlSegmentsMetadataManagerV2Test extends SqlSegmentsMetadataManagerTestBase
+{
+ @Rule
+ public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule =
+ new TestDerbyConnector.DerbyConnectorRule(CentralizedDatasourceSchemaConfig.create(true));
+
+ private SegmentsMetadataManager manager;
+ private BlockingExecutorService segmentMetadataCacheExec;
+ private StubServiceEmitter emitter;
+
+ private static final DateTime JAN_1 = DateTimes.of("2025-01-01");
+
+ private static final List<DataSegment> WIKI_SEGMENTS_1X5D
+ = CreateDataSegments.ofDatasource(TestDataSource.WIKI)
+ .forIntervals(5, Granularities.DAY)
+ .startingAt(JAN_1)
+ .eachOfSize(500);
+
+ @Before
+ public void setup() throws Exception
+ {
+ setUp(derbyConnectorRule);
+ connector.createPendingSegmentsTable();
+
+ emitter = new StubServiceEmitter();
+
+ WIKI_SEGMENTS_1X5D.forEach(super::publishSegment);
+ }
+
+ private void initManager(
+ SegmentMetadataCache.UsageMode cacheMode
+ )
+ {
+ segmentMetadataCacheExec = new BlockingExecutorService("test");
+ SegmentMetadataCache segmentMetadataCache = new HeapMemorySegmentMetadataCache(
+ jsonMapper,
+ Suppliers.ofInstance(new SegmentsMetadataManagerConfig(Period.seconds(1), cacheMode)),
+ Suppliers.ofInstance(storageConfig),
+ connector,
+ (poolSize, name) -> new WrappingScheduledExecutorService(name, segmentMetadataCacheExec, false),
+ emitter
+ );
+ segmentMetadataCache.start();
+ segmentMetadataCache.becomeLeader();
+
+ manager = new SqlSegmentsMetadataManagerV2(
+ segmentMetadataCache,
+ segmentSchemaCache,
+ connector,
+ Suppliers.ofInstance(config),
+ derbyConnectorRule.metadataTablesConfigSupplier(),
+ CentralizedDatasourceSchemaConfig.create(),
+ emitter,
+ jsonMapper
+ );
+ manager.start();
+ }
+
+ private void syncSegmentMetadataCache()
+ {
+ segmentMetadataCacheExec.finishNextPendingTasks(2);
+ segmentMetadataCacheExec.finishNextPendingTasks(2);
+ }
+
+ @After
+ public void tearDown()
+ {
+ if (manager == null) {
+ return;
+ }
+
+ if (manager.isPollingDatabasePeriodically()) {
+ manager.stopPollingDatabasePeriodically();
+ }
+ manager.stop();
+ }
+
+ @Test
+ public void test_manager_usesCachedSegments_ifCacheIsEnabled()
+ {
+ initManager(SegmentMetadataCache.UsageMode.ALWAYS);
+
+ manager.startPollingDatabasePeriodically();
+ Assert.assertTrue(manager.isPollingDatabasePeriodically());
+
+ syncSegmentMetadataCache();
+ verifyDatasourceSnapshot();
+
+ // isPolling returns true even after stop since cache is still polling the metadata store
+ manager.stopPollingDatabasePeriodically();
+ Assert.assertTrue(manager.isPollingDatabasePeriodically());
+
+ emitter.verifyNotEmitted("segment/poll/time");
+ emitter.verifyNotEmitted("segment/pollWithSchema/time");
+ emitter.verifyEmitted(Metric.SYNC_DURATION_MILLIS, 2);
+ }
+
+ @Test
+ public void test_manager_pollsSegments_ifCacheIsDisabled()
+ {
+ initManager(SegmentMetadataCache.UsageMode.NEVER);
+
+ manager.startPollingDatabasePeriodically();
+ Assert.assertTrue(manager.isPollingDatabasePeriodically());
+
+ verifyDatasourceSnapshot();
+
+ manager.stopPollingDatabasePeriodically();
+ Assert.assertFalse(manager.isPollingDatabasePeriodically());
+
+ emitter.verifyEmitted("segment/poll/time", 1);
+ emitter.verifyNotEmitted(Metric.SYNC_DURATION_MILLIS);
+ }
+
+ @Test
+ public void test_manager_throwsException_ifBothCacheAndSchemaAreEnabled()
+ {
+ MatcherAssert.assertThat(
+ Assert.assertThrows(
+ IllegalArgumentException.class,
+ () -> new SqlSegmentsMetadataManagerV2(
+ new NoopSegmentMetadataCache() {
+ @Override
+ public boolean isEnabled()
+ {
+ return true;
+ }
+ },
+ segmentSchemaCache,
+ connector,
+ Suppliers.ofInstance(config),
+ derbyConnectorRule.metadataTablesConfigSupplier(),
+ CentralizedDatasourceSchemaConfig.create(true),
+ emitter,
+ jsonMapper
+ )
+ ),
+ ExceptionMatcher.of(IllegalArgumentException.class).expectMessageIs(
+ "Segment metadata incremental cache['druid.manager.segments.useIncrementalCache']"
+ + " and segment schema cache['druid.centralizedDatasourceSchema.enabled']"
+ + " must not be enabled together."
+ )
+ );
+ }
+
+ private void verifyDatasourceSnapshot()
+ {
+ final DataSourcesSnapshot snapshot = manager.getRecentDataSourcesSnapshot();
+ Assert.assertEquals(
+ Set.copyOf(WIKI_SEGMENTS_1X5D),
+ Sets.newHashSet(snapshot.iterateAllUsedSegmentsInSnapshot())
+ );
+ Assert.assertEquals(
+ Set.copyOf(WIKI_SEGMENTS_1X5D),
+ Set.copyOf(snapshot.getDataSource(TestDataSource.WIKI).getSegments())
+ );
+ }
+}
diff --git a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
index b5a2fe6b0d7..48983284071 100644
--- a/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/druid/metadata/segment/cache/HeapMemorySegmentMetadataCacheTest.java
@@ -694,10 +694,8 @@ public class HeapMemorySegmentMetadataCacheTest
private void insertSegmentsInMetadataStore(Set<DataSegmentPlus> segments)
{
- final String table = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
-
IndexerSqlMetadataStorageCoordinatorTestBase
- .insertSegments(segments, derbyConnector, table, TestHelper.JSON_MAPPER);
+ .insertSegments(segments, derbyConnectorRule, TestHelper.JSON_MAPPER);
}
private void updateSegmentInMetadataStore(DataSegmentPlus segment)
diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java
index 74a201eae27..5a0a9833827 100644
--- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java
+++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentDataCacheConcurrencyTest.java
@@ -27,6 +27,7 @@ import com.google.common.collect.Sets;
import org.apache.druid.client.BrokerServerView;
import org.apache.druid.client.CoordinatorSegmentWatcherConfig;
import org.apache.druid.client.CoordinatorServerView;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.DirectDruidClientFactory;
import org.apache.druid.client.DruidServer;
import org.apache.druid.client.InternalQueryConfig;
@@ -41,8 +42,8 @@ import org.apache.druid.java.util.common.NonnullPair;
import org.apache.druid.java.util.common.concurrent.Execs;
import org.apache.druid.java.util.common.concurrent.ScheduledExecutors;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
+import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.query.QueryRunner;
import org.apache.druid.query.TableDataSource;
@@ -77,7 +78,6 @@ import javax.annotation.Nullable;
import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -108,7 +108,7 @@ public class CoordinatorSegmentDataCacheConcurrencyTest extends SegmentMetadataC
private TestSegmentMetadataQueryWalker walker;
private SegmentSchemaCache segmentSchemaCache;
private SegmentSchemaBackFillQueue backFillQueue;
- private SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
+ private SegmentsMetadataManager segmentsMetadataManager;
private Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier;
private final ObjectMapper mapper = TestHelper.makeJsonMapper();
@@ -198,8 +198,9 @@ public class CoordinatorSegmentDataCacheConcurrencyTest extends SegmentMetadataC
}
);
- sqlSegmentsMetadataManager = Mockito.mock(SqlSegmentsMetadataManager.class);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).thenReturn(Collections.emptyList());
+ segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of()));
SegmentsMetadataManagerConfig metadataManagerConfig = Mockito.mock(SegmentsMetadataManagerConfig.class);
Mockito.when(metadataManagerConfig.getPollDuration()).thenReturn(Period.millis(1000));
segmentsMetadataManagerConfigSupplier = Suppliers.ofInstance(metadataManagerConfig);
@@ -242,7 +243,7 @@ public class CoordinatorSegmentDataCacheConcurrencyTest extends SegmentMetadataC
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -358,7 +359,7 @@ public class CoordinatorSegmentDataCacheConcurrencyTest extends SegmentMetadataC
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
diff --git a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
index d56db2ea441..f641cc859ba 100644
--- a/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
+++ b/server/src/test/java/org/apache/druid/segment/metadata/CoordinatorSegmentMetadataCacheTest.java
@@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.DruidServer;
import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.client.InternalQueryConfig;
@@ -42,8 +43,8 @@ import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.metrics.StubServiceEmitter;
import org.apache.druid.metadata.MetadataStorageTablesConfig;
+import org.apache.druid.metadata.SegmentsMetadataManager;
import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.query.DruidMetrics;
import org.apache.druid.query.QueryContexts;
@@ -117,13 +118,12 @@ import java.util.stream.Collectors;
public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetadataCacheTestBase
{
- // Timeout to allow (rapid) debugging, while not blocking tests with errors.
private static final ObjectMapper MAPPER = TestHelper.makeJsonMapper();
private static final SegmentMetadataCacheConfig SEGMENT_CACHE_CONFIG_DEFAULT = SegmentMetadataCacheConfig.create("PT1S");
private CoordinatorSegmentMetadataCache runningSchema;
- private CountDownLatch buildTableLatch = new CountDownLatch(1);
- private CountDownLatch markDataSourceLatch = new CountDownLatch(1);
- private SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
+ private final CountDownLatch buildTableLatch = new CountDownLatch(1);
+ private final CountDownLatch markDataSourceLatch = new CountDownLatch(1);
+ private SegmentsMetadataManager segmentsMetadataManager;
private Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier;
@Before
@@ -131,8 +131,9 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
public void setUp() throws Exception
{
super.setUp();
- sqlSegmentsMetadataManager = Mockito.mock(SqlSegmentsMetadataManager.class);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).thenReturn(Collections.emptyList());
+ segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of()));
SegmentsMetadataManagerConfig metadataManagerConfig = Mockito.mock(SegmentsMetadataManagerConfig.class);
Mockito.when(metadataManagerConfig.getPollDuration()).thenReturn(Period.millis(1000));
segmentsMetadataManagerConfigSupplier = Suppliers.ofInstance(metadataManagerConfig);
@@ -166,7 +167,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -340,7 +341,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -553,7 +554,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -589,8 +590,9 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
{
String datasource = "newSegmentAddTest";
CountDownLatch addSegmentLatch = new CountDownLatch(2);
- SqlSegmentsMetadataManager sqlSegmentsMetadataManager = Mockito.mock(SqlSegmentsMetadataManager.class);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).thenReturn(Collections.emptyList());
+ SegmentsMetadataManager segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of()));
SegmentsMetadataManagerConfig metadataManagerConfig = Mockito.mock(SegmentsMetadataManagerConfig.class);
Mockito.when(metadataManagerConfig.getPollDuration()).thenReturn(Period.millis(1000));
Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier = Suppliers.ofInstance(metadataManagerConfig);
@@ -603,7 +605,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -643,8 +645,9 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
{
String datasource = "newSegmentAddTest";
CountDownLatch addSegmentLatch = new CountDownLatch(1);
- SqlSegmentsMetadataManager sqlSegmentsMetadataManager = Mockito.mock(SqlSegmentsMetadataManager.class);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).thenReturn(Collections.emptyList());
+ SegmentsMetadataManager segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of()));
SegmentsMetadataManagerConfig metadataManagerConfig = Mockito.mock(SegmentsMetadataManagerConfig.class);
Mockito.when(metadataManagerConfig.getPollDuration()).thenReturn(Period.millis(1000));
Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier = Suppliers.ofInstance(metadataManagerConfig);
@@ -657,7 +660,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -694,8 +697,9 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
{
String datasource = "newSegmentAddTest";
CountDownLatch addSegmentLatch = new CountDownLatch(1);
- SqlSegmentsMetadataManager sqlSegmentsMetadataManager = Mockito.mock(SqlSegmentsMetadataManager.class);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments()).thenReturn(Collections.emptyList());
+ SegmentsMetadataManager segmentsMetadataManager = Mockito.mock(SegmentsMetadataManager.class);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of()));
SegmentsMetadataManagerConfig metadataManagerConfig = Mockito.mock(SegmentsMetadataManagerConfig.class);
Mockito.when(metadataManagerConfig.getPollDuration()).thenReturn(Period.millis(1000));
Supplier<SegmentsMetadataManagerConfig> segmentsMetadataManagerConfigSupplier = Suppliers.ofInstance(metadataManagerConfig);
@@ -708,7 +712,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -751,7 +755,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -811,7 +815,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -874,7 +878,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -911,7 +915,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -961,7 +965,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -1035,7 +1039,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
);
@@ -1210,7 +1214,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
emitter,
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
)
{
@@ -1377,7 +1381,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
) {
@Override
@@ -1458,7 +1462,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
) {
@Override
@@ -1640,7 +1644,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
) {
@Override
@@ -1802,6 +1806,8 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
private CoordinatorSegmentMetadataCache setupForColdDatasourceSchemaTest(ServiceEmitter emitter)
{
// foo has both hot and cold segments
+ final String fingerprint1 = "fingerprint-1";
+ final String fingerprint2 = "fingerprint-2";
DataSegment coldSegment =
DataSegment.builder()
.dataSource(DATASOURCE1)
@@ -1822,11 +1828,11 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
.build();
ImmutableMap.Builder<SegmentId, SegmentMetadata> segmentStatsMap = new ImmutableMap.Builder<>();
- segmentStatsMap.put(coldSegment.getId(), new SegmentMetadata(20L, "foo-fingerprint"));
- segmentStatsMap.put(singleColdSegment.getId(), new SegmentMetadata(20L, "cold-fingerprint"));
+ segmentStatsMap.put(coldSegment.getId(), new SegmentMetadata(20L, fingerprint1));
+ segmentStatsMap.put(singleColdSegment.getId(), new SegmentMetadata(20L, fingerprint2));
ImmutableMap.Builder<String, SchemaPayload> schemaPayloadMap = new ImmutableMap.Builder<>();
schemaPayloadMap.put(
- "foo-fingerprint",
+ fingerprint1,
new SchemaPayload(RowSignature.builder()
.add("dim1", ColumnType.STRING)
.add("c1", ColumnType.STRING)
@@ -1834,7 +1840,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
.build())
);
schemaPayloadMap.put(
- "cold-fingerprint",
+ fingerprint2,
new SchemaPayload(
RowSignature.builder()
.add("f1", ColumnType.STRING)
@@ -1847,25 +1853,9 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new SegmentSchemaCache.FinalizedSegmentSchemaInfo(segmentStatsMap.build(), schemaPayloadMap.build())
);
- List<ImmutableDruidDataSource> druidDataSources = new ArrayList<>();
- Map<SegmentId, DataSegment> segmentMap = new HashMap<>();
- segmentMap.put(coldSegment.getId(), coldSegment);
- segmentMap.put(segment1.getId(), segment1);
- segmentMap.put(segment2.getId(), segment2);
- druidDataSources.add(new ImmutableDruidDataSource(
- coldSegment.getDataSource(),
- Collections.emptyMap(),
- segmentMap
- ));
- druidDataSources.add(new ImmutableDruidDataSource(
- singleColdSegment.getDataSource(),
- Collections.emptyMap(),
- Collections.singletonMap(singleColdSegment.getId(), singleColdSegment)
- ));
-
- Mockito.when(
- sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments())
- .thenReturn(druidDataSources);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot()).thenReturn(
+ DataSourcesSnapshot.fromUsedSegments(List.of(segment1, segment2, coldSegment, singleColdSegment))
+ );
CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache(
getQueryLifecycleFactory(walker),
@@ -1876,7 +1866,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
emitter,
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
);
@@ -2043,7 +2033,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
DataSegment coldSegmentGamma =
DataSegment.builder()
.dataSource("gamma")
- .interval(Intervals.of("2000/P2Y"))
+ .interval(Intervals.of("2000/P1Y"))
.version("1")
.shardSpec(new LinearShardSpec(0))
.size(0)
@@ -2084,30 +2074,10 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new SegmentSchemaCache.FinalizedSegmentSchemaInfo(segmentStatsMap.build(), schemaPayloadMap.build())
);
- List<ImmutableDruidDataSource> druidDataSources = new ArrayList<>();
- druidDataSources.add(
- new ImmutableDruidDataSource(
- "alpha",
- Collections.emptyMap(),
- Collections.singletonMap(coldSegmentAlpha.getId(), coldSegmentAlpha)
- )
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot()).thenReturn(
+ DataSourcesSnapshot.fromUsedSegments(List.of(coldSegmentAlpha, hotSegmentGamma, coldSegmentGamma))
);
- Map<SegmentId, DataSegment> gammaSegments = new HashMap<>();
- gammaSegments.put(hotSegmentGamma.getId(), hotSegmentGamma);
- gammaSegments.put(coldSegmentGamma.getId(), coldSegmentGamma);
-
- druidDataSources.add(
- new ImmutableDruidDataSource(
- "gamma",
- Collections.emptyMap(),
- gammaSegments
- )
- );
-
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments())
- .thenReturn(druidDataSources);
-
CoordinatorSegmentMetadataCache schema = new CoordinatorSegmentMetadataCache(
getQueryLifecycleFactory(walker),
serverView,
@@ -2117,7 +2087,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
);
@@ -2153,25 +2123,8 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
Assert.assertEquals(new HashSet<>(Arrays.asList("alpha", "gamma")), schema.getDataSourceInformationMap().keySet());
- druidDataSources.clear();
- druidDataSources.add(
- new ImmutableDruidDataSource(
- "beta",
- Collections.emptyMap(),
- Collections.singletonMap(coldSegmentBeta.getId(), coldSegmentBeta)
- )
- );
-
- druidDataSources.add(
- new ImmutableDruidDataSource(
- "gamma",
- Collections.emptyMap(),
- Collections.singletonMap(hotSegmentGamma.getId(), hotSegmentGamma)
- )
- );
-
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments())
- .thenReturn(druidDataSources);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of(coldSegmentBeta, hotSegmentGamma)));
schema.coldDatasourceSchemaExec();
Assert.assertNotNull(schema.getDatasource("beta"));
@@ -2200,7 +2153,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
) {
@Override
@@ -2249,7 +2202,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
);
@@ -2348,7 +2301,7 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
new NoopServiceEmitter(),
segmentSchemaCache,
backFillQueue,
- sqlSegmentsMetadataManager,
+ segmentsMetadataManager,
segmentsMetadataManagerConfigSupplier
) {
@Override
@@ -2417,8 +2370,8 @@ public class CoordinatorSegmentMetadataCacheTest extends CoordinatorSegmentMetad
segmentMap
);
- Mockito.when(sqlSegmentsMetadataManager.getImmutableDataSourceWithUsedSegments(ArgumentMatchers.anyString()))
- .thenReturn(druidDataSource);
+ Mockito.when(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .thenReturn(DataSourcesSnapshot.fromUsedSegments(List.of(segments.get(0), segments.get(1))));
Set<SegmentId> segmentsToRefresh = segments.stream().map(DataSegment::getId).collect(Collectors.toSet());
segmentsToRefresh.remove(segments.get(1).getId());
diff --git a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
index fd90f26bc7f..fd491ef930f 100644
--- a/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/compaction/CompactionRunSimulatorTest.java
@@ -76,7 +76,7 @@ public class CompactionRunSimulatorTest
DruidCompactionConfig.empty().withDatasourceConfig(
InlineSchemaDataSourceCompactionConfig.builder().forDataSource("wiki").build()
),
- segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments(),
+ segmentsMetadataManager.getRecentDataSourcesSnapshot(),
CompactionEngine.NATIVE
);
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
index 182fce84cb0..f77da3140a4 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
@@ -43,6 +43,7 @@ import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
import org.apache.druid.metadata.MetadataRuleManager;
import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
import org.apache.druid.rpc.indexing.OverlordClient;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
import org.apache.druid.server.DruidNode;
@@ -183,7 +184,8 @@ public class DruidCoordinatorTest
null,
metadataRuleManager,
null,
- null
+ null,
+ NoopSegmentMetadataCache.instance()
);
}
@@ -645,13 +647,9 @@ public class DruidCoordinatorTest
DataSourcesSnapshot dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(
Collections.singleton(dataSegment)
);
- EasyMock
- .expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
- .andReturn(dataSourcesSnapshot)
- .anyTimes();
+ EasyMock.expect(segmentsMetadataManager.getRecentDataSourcesSnapshot())
+ .andReturn(dataSourcesSnapshot).anyTimes();
EasyMock.expect(segmentsMetadataManager.isPollingDatabasePeriodically()).andReturn(true).anyTimes();
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedSegments())
- .andReturn(Collections.singletonList(dataSegment)).anyTimes();
EasyMock.expect(serverInventoryView.isStarted()).andReturn(true).anyTimes();
EasyMock.expect(serverInventoryView.getInventory()).andReturn(Collections.emptyList()).anyTimes();
EasyMock.replay(serverInventoryView, loadQueueTaskMaster, segmentsMetadataManager);
@@ -827,7 +825,7 @@ public class DruidCoordinatorTest
{
DataSourcesSnapshot dataSourcesSnapshot = DataSourcesSnapshot.fromUsedSegments(Collections.emptyList());
EasyMock
- .expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
+ .expect(segmentsMetadataManager.getRecentDataSourcesSnapshot())
.andReturn(dataSourcesSnapshot)
.anyTimes();
EasyMock.replay(segmentsMetadataManager);
@@ -840,24 +838,12 @@ public class DruidCoordinatorTest
private void setupSegmentsMetadataMock(DruidDataSource dataSource)
{
EasyMock.expect(segmentsMetadataManager.isPollingDatabasePeriodically()).andReturn(true).anyTimes();
- EasyMock
- .expect(segmentsMetadataManager.iterateAllUsedSegments())
- .andReturn(dataSource.getSegments())
- .anyTimes();
- EasyMock
- .expect(segmentsMetadataManager.getImmutableDataSourcesWithAllUsedSegments())
- .andReturn(Collections.singleton(dataSource.toImmutableDruidDataSource()))
- .anyTimes();
DataSourcesSnapshot dataSourcesSnapshot =
DataSourcesSnapshot.fromUsedSegments(dataSource.getSegments());
EasyMock
- .expect(segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments())
+ .expect(segmentsMetadataManager.getRecentDataSourcesSnapshot())
.andReturn(dataSourcesSnapshot)
.anyTimes();
- EasyMock
- .expect(segmentsMetadataManager.retrieveAllDataSourceNames())
- .andReturn(Collections.singleton(dataSource.getName()))
- .anyTimes();
EasyMock.replay(segmentsMetadataManager);
EasyMock
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java
index 3ec3958c4e7..5e840d98fb6 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillCompactionConfigTest.java
@@ -24,10 +24,10 @@ import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.druid.common.config.ConfigManager;
import org.apache.druid.common.config.JacksonConfigManager;
+import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.metadata.MetadataStorageConnector;
import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.server.coordinator.CoordinatorConfigManager;
import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -56,7 +56,7 @@ public class KillCompactionConfigTest
private DruidCoordinatorRuntimeParams mockDruidCoordinatorRuntimeParams;
@Mock
- private SqlSegmentsMetadataManager mockSqlSegmentsMetadataManager;
+ private IndexerMetadataStorageCoordinator mockStorageCoordinator;
@Mock
private JacksonConfigManager mockJacksonConfigManager;
@@ -92,11 +92,11 @@ public class KillCompactionConfigTest
= new MetadataCleanupConfig(true, new Duration(Long.MAX_VALUE), null);
killCompactionConfig = new KillCompactionConfig(
config,
- mockSqlSegmentsMetadataManager,
+ mockStorageCoordinator,
coordinatorConfigManager
);
killCompactionConfig.run(mockDruidCoordinatorRuntimeParams);
- Mockito.verifyNoInteractions(mockSqlSegmentsMetadataManager);
+ Mockito.verifyNoInteractions(mockStorageCoordinator);
Mockito.verifyNoInteractions(mockJacksonConfigManager);
Assert.assertEquals(0, runStats.rowCount());
}
@@ -121,11 +121,11 @@ public class KillCompactionConfigTest
= new MetadataCleanupConfig(true, new Duration("PT6S"), null);
killCompactionConfig = new KillCompactionConfig(
config,
- mockSqlSegmentsMetadataManager,
+ mockStorageCoordinator,
coordinatorConfigManager
);
killCompactionConfig.run(mockDruidCoordinatorRuntimeParams);
- Mockito.verifyNoInteractions(mockSqlSegmentsMetadataManager);
+ Mockito.verifyNoInteractions(mockStorageCoordinator);
Assert.assertTrue(runStats.hasStat(Stats.Kill.COMPACTION_CONFIGS));
Assert.assertEquals(0, runStats.get(Stats.Kill.COMPACTION_CONFIGS));
@@ -184,7 +184,7 @@ public class KillCompactionConfigTest
ArgumentMatchers.eq(DruidCompactionConfig.class),
ArgumentMatchers.eq(DruidCompactionConfig.empty()))
).thenReturn(originalCurrentConfig);
- Mockito.when(mockSqlSegmentsMetadataManager.retrieveAllDataSourceNames()).thenReturn(ImmutableSet.of(activeDatasourceName));
+ Mockito.when(mockStorageCoordinator.retrieveAllDatasourceNames()).thenReturn(ImmutableSet.of(activeDatasourceName));
final ArgumentCaptor<byte[]> oldConfigCaptor = ArgumentCaptor.forClass(byte[].class);
final ArgumentCaptor<DruidCompactionConfig> newConfigCaptor = ArgumentCaptor.forClass(DruidCompactionConfig.class);
Mockito.when(mockJacksonConfigManager.set(
@@ -198,7 +198,7 @@ public class KillCompactionConfigTest
= new MetadataCleanupConfig(true, new Duration("PT6S"), null);
killCompactionConfig = new KillCompactionConfig(
config,
- mockSqlSegmentsMetadataManager,
+ mockStorageCoordinator,
coordinatorConfigManager
);
killCompactionConfig.run(mockDruidCoordinatorRuntimeParams);
@@ -231,8 +231,8 @@ public class KillCompactionConfigTest
ArgumentMatchers.any()
);
Mockito.verifyNoMoreInteractions(mockJacksonConfigManager);
- Mockito.verify(mockSqlSegmentsMetadataManager).retrieveAllDataSourceNames();
- Mockito.verifyNoMoreInteractions(mockSqlSegmentsMetadataManager);
+ Mockito.verify(mockStorageCoordinator).retrieveAllDatasourceNames();
+ Mockito.verifyNoMoreInteractions(mockStorageCoordinator);
}
@Test
@@ -265,7 +265,7 @@ public class KillCompactionConfigTest
ArgumentMatchers.eq(DruidCompactionConfig.class),
ArgumentMatchers.eq(DruidCompactionConfig.empty()))
).thenReturn(originalCurrentConfig);
- Mockito.when(mockSqlSegmentsMetadataManager.retrieveAllDataSourceNames()).thenReturn(ImmutableSet.of());
+ Mockito.when(mockStorageCoordinator.retrieveAllDatasourceNames()).thenReturn(ImmutableSet.of());
Mockito.when(mockJacksonConfigManager.set(
ArgumentMatchers.eq(DruidCompactionConfig.CONFIG_KEY),
ArgumentMatchers.any(byte[].class),
@@ -284,7 +284,7 @@ public class KillCompactionConfigTest
= new MetadataCleanupConfig(true, new Duration("PT6S"), null);
killCompactionConfig = new KillCompactionConfig(
config,
- mockSqlSegmentsMetadataManager,
+ mockStorageCoordinator,
coordinatorConfigManager
);
killCompactionConfig.run(mockDruidCoordinatorRuntimeParams);
@@ -314,7 +314,7 @@ public class KillCompactionConfigTest
);
Mockito.verifyNoMoreInteractions(mockJacksonConfigManager);
// Should call retrieveAllDataSourceNames four times due to RetryableException when failed
- Mockito.verify(mockSqlSegmentsMetadataManager, Mockito.times(4)).retrieveAllDataSourceNames();
- Mockito.verifyNoMoreInteractions(mockSqlSegmentsMetadataManager);
+ Mockito.verify(mockStorageCoordinator, Mockito.times(4)).retrieveAllDatasourceNames();
+ Mockito.verifyNoMoreInteractions(mockStorageCoordinator);
}
}
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java
index 1f02a5e9db9..f8ea6730838 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/KillUnusedSegmentsTest.java
@@ -19,7 +19,6 @@
package org.apache.druid.server.coordinator.duty;
-import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.Futures;
@@ -31,15 +30,15 @@ import org.apache.druid.indexer.RunnerTaskState;
import org.apache.druid.indexer.TaskLocation;
import org.apache.druid.indexer.TaskState;
import org.apache.druid.indexer.TaskStatusPlus;
+import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
import org.apache.druid.java.util.common.CloseableIterators;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.common.JodaUtils;
import org.apache.druid.java.util.common.parsers.CloseableIterator;
+import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
import org.apache.druid.metadata.MetadataStorageTablesConfig;
import org.apache.druid.metadata.SQLMetadataConnector;
-import org.apache.druid.metadata.SegmentsMetadataManagerConfig;
-import org.apache.druid.metadata.SqlSegmentsMetadataManager;
import org.apache.druid.metadata.SqlSegmentsMetadataManagerTestBase;
import org.apache.druid.metadata.TestDerbyConnector;
import org.apache.druid.rpc.indexing.NoopOverlordClient;
@@ -52,7 +51,6 @@ import org.apache.druid.server.coordinator.stats.CoordinatorRunStats;
import org.apache.druid.server.coordinator.stats.Dimension;
import org.apache.druid.server.coordinator.stats.RowKey;
import org.apache.druid.server.coordinator.stats.Stats;
-import org.apache.druid.server.metrics.NoopServiceEmitter;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.NoneShardSpec;
import org.joda.time.DateTime;
@@ -65,13 +63,13 @@ import org.junit.Rule;
import org.junit.Test;
import javax.annotation.Nullable;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
public class KillUnusedSegmentsTest
{
@@ -102,8 +100,10 @@ public class KillUnusedSegmentsTest
private KillUnusedSegments killDuty;
@Rule
- public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule();
- private SqlSegmentsMetadataManager sqlSegmentsMetadataManager;
+ public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule
+ = new TestDerbyConnector.DerbyConnectorRule();
+
+ private IndexerMetadataStorageCoordinator storageCoordinator;
private SQLMetadataConnector connector;
private MetadataStorageTablesConfig config;
@@ -111,17 +111,14 @@ public class KillUnusedSegmentsTest
public void setup()
{
connector = derbyConnectorRule.getConnector();
- SegmentsMetadataManagerConfig config = new SegmentsMetadataManagerConfig(Period.millis(1), null);
- sqlSegmentsMetadataManager = new SqlSegmentsMetadataManager(
- TestHelper.makeJsonMapper(),
- Suppliers.ofInstance(config),
- derbyConnectorRule.metadataTablesConfigSupplier(),
+ storageCoordinator = new IndexerSQLMetadataStorageCoordinator(
+ null,
+ TestHelper.JSON_MAPPER,
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
connector,
null,
- CentralizedDatasourceSchemaConfig.create(),
- NoopServiceEmitter.instance()
+ CentralizedDatasourceSchemaConfig.create()
);
- sqlSegmentsMetadataManager.start();
this.config = derbyConnectorRule.metadataTablesConfigSupplier().get();
connector.createSegmentTable();
@@ -1068,12 +1065,7 @@ public class KillUnusedSegmentsTest
private DataSegment createAndAddUsedSegment(final String dataSource, final Interval interval, final String version)
{
final DataSegment segment = createSegment(dataSource, interval, version);
- try {
- SqlSegmentsMetadataManagerTestBase.publishSegment(connector, config, TestHelper.makeJsonMapper(), segment);
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
+ SqlSegmentsMetadataManagerTestBase.publishSegment(connector, config, TestHelper.makeJsonMapper(), segment);
return segment;
}
@@ -1085,8 +1077,14 @@ public class KillUnusedSegmentsTest
)
{
final DataSegment segment = createAndAddUsedSegment(dataSource, interval, version);
- sqlSegmentsMetadataManager.markSegmentsAsUnused(ImmutableSet.of(segment.getId()));
- derbyConnectorRule.segments().updateUsedStatusLastUpdated(segment.getId().toString(), lastUpdatedTime);
+ int numUpdatedSegments = SqlSegmentsMetadataManagerTestBase.markSegmentsAsUnused(
+ Set.of(segment.getId()),
+ derbyConnectorRule.getConnector(),
+ derbyConnectorRule.metadataTablesConfigSupplier().get(),
+ TestHelper.JSON_MAPPER,
+ lastUpdatedTime
+ );
+ Assert.assertEquals(1, numUpdatedSegments);
}
private DataSegment createSegment(final String dataSource, final Interval interval, final String version)
@@ -1106,7 +1104,7 @@ public class KillUnusedSegmentsTest
private void initDuty()
{
- killDuty = new KillUnusedSegments(sqlSegmentsMetadataManager, overlordClient, configBuilder.build());
+ killDuty = new KillUnusedSegments(storageCoordinator, overlordClient, configBuilder.build());
}
private CoordinatorRunStats runDutyAndGetStats()
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkEternityTombstonesAsUnusedTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkEternityTombstonesAsUnusedTest.java
index 37fbe144783..4b412f58864 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkEternityTombstonesAsUnusedTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkEternityTombstonesAsUnusedTest.java
@@ -169,7 +169,7 @@ public class MarkEternityTombstonesAsUnusedTest
final ImmutableList<DataSegment> expectedUsedSegments = ImmutableList.copyOf(allUsedSegments);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
@@ -195,7 +195,7 @@ public class MarkEternityTombstonesAsUnusedTest
final ImmutableList<DataSegment> expectedUsedSegments = ImmutableList.copyOf(allUsedSegments);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds1Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds1Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
@@ -221,7 +221,7 @@ public class MarkEternityTombstonesAsUnusedTest
);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- SegmentTimeline timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ SegmentTimeline timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
@@ -244,7 +244,7 @@ public class MarkEternityTombstonesAsUnusedTest
final ImmutableList<DataSegment> expectedUsedSegments = ImmutableList.of();
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds1Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds1Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
@@ -270,7 +270,7 @@ public class MarkEternityTombstonesAsUnusedTest
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- SegmentTimeline timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ SegmentTimeline timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
@@ -320,7 +320,7 @@ public class MarkEternityTombstonesAsUnusedTest
);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- SegmentTimeline ds1Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ SegmentTimeline ds1Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds1);
Assert.assertTrue(ds1Timeline.isOvershadowed(ds1TombstoneSegmentMinTo2000V1));
@@ -328,7 +328,7 @@ public class MarkEternityTombstonesAsUnusedTest
Assert.assertFalse(ds1Timeline.isOvershadowed(ds1TombstoneSegment2001ToMaxV1));
Assert.assertFalse(ds1Timeline.isOvershadowed(ds1TombstoneSegmentMinTo2000V2));
- SegmentTimeline ds2Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ SegmentTimeline ds2Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds2);
Assert.assertFalse(ds2Timeline.isOvershadowed(ds2TombstoneSegmentMinTo2000V1));
@@ -358,7 +358,7 @@ public class MarkEternityTombstonesAsUnusedTest
);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds2Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds2Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds2);
@@ -391,7 +391,7 @@ public class MarkEternityTombstonesAsUnusedTest
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds2Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds2Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds2);
Assert.assertFalse(ds2Timeline.isOvershadowed(ds2TombstoneSegmentMinTo2000V1));
@@ -416,7 +416,7 @@ public class MarkEternityTombstonesAsUnusedTest
);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds2Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds2Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds2);
@@ -440,7 +440,7 @@ public class MarkEternityTombstonesAsUnusedTest
);
final DruidCoordinatorRuntimeParams params = initializeServerAndGetParams(allUsedSegments);
- final SegmentTimeline ds2Timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ final SegmentTimeline ds2Timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get(ds2);
@@ -465,7 +465,7 @@ public class MarkEternityTombstonesAsUnusedTest
final DruidCoordinatorRuntimeParams params = DruidCoordinatorRuntimeParams
.builder()
.withDataSourcesSnapshot(
- segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ segmentsMetadataManager.getRecentDataSourcesSnapshot()
)
.withDruidCluster(druidCluster)
.withDynamicConfigs(
@@ -488,7 +488,9 @@ public class MarkEternityTombstonesAsUnusedTest
(ds, segmentIds) -> segmentsMetadataManager.markSegmentsAsUnused(segmentIds)
).run(params);
- final Set<DataSegment> actualUsedSegments = Sets.newHashSet(segmentsMetadataManager.iterateAllUsedSegments());
+ final Set<DataSegment> actualUsedSegments = Sets.newHashSet(
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().iterateAllUsedSegmentsInSnapshot()
+ );
Assert.assertEquals(expectedUsedSegments.size(), actualUsedSegments.size());
Assert.assertTrue(actualUsedSegments.containsAll(expectedUsedSegments));
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkOvershadowedSegmentsAsUnusedTest.java b/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkOvershadowedSegmentsAsUnusedTest.java
index a06f1d3c732..f500e5d4b37 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkOvershadowedSegmentsAsUnusedTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/duty/MarkOvershadowedSegmentsAsUnusedTest.java
@@ -92,7 +92,7 @@ public class MarkOvershadowedSegmentsAsUnusedTest
DruidCoordinatorRuntimeParams params = DruidCoordinatorRuntimeParams
.builder()
.withDataSourcesSnapshot(
- segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ segmentsMetadataManager.getRecentDataSourcesSnapshot()
)
.withDruidCluster(druidCluster)
.withDynamicConfigs(
@@ -102,7 +102,7 @@ public class MarkOvershadowedSegmentsAsUnusedTest
.withSegmentAssignerUsing(new SegmentLoadQueueManager(null, null))
.build();
- SegmentTimeline timeline = segmentsMetadataManager.getSnapshotOfDataSourcesWithAllUsedSegments()
+ SegmentTimeline timeline = segmentsMetadataManager.getRecentDataSourcesSnapshot()
.getUsedSegmentsTimelinesPerDataSource()
.get("test");
@@ -115,7 +115,9 @@ public class MarkOvershadowedSegmentsAsUnusedTest
(ds, segmentIds) -> segmentsMetadataManager.markSegmentsAsUnused(segmentIds)
).run(params);
- Set<DataSegment> updatedUsedSegments = Sets.newHashSet(segmentsMetadataManager.iterateAllUsedSegments());
+ Set<DataSegment> updatedUsedSegments = Sets.newHashSet(
+ segmentsMetadataManager.getRecentDataSourcesSnapshot().iterateAllUsedSegmentsInSnapshot()
+ );
Assert.assertEquals(1, updatedUsedSegments.size());
Assert.assertTrue(updatedUsedSegments.contains(segmentV2));
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
index 70237158a00..b8e56419c02 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/CoordinatorSimulationBuilder.java
@@ -38,7 +38,7 @@ import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.java.util.http.client.HttpClient;
import org.apache.druid.java.util.metrics.MetricsVerifier;
import org.apache.druid.java.util.metrics.StubServiceEmitter;
-import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
import org.apache.druid.rpc.indexing.NoopOverlordClient;
import org.apache.druid.rpc.indexing.SegmentUpdateResponse;
import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
@@ -212,7 +212,7 @@ public class CoordinatorSimulationBuilder
env.coordinatorInventoryView,
env.serviceEmitter,
env.executorFactory,
- new SimOverlordClient(env.metadataManager.segments()),
+ new SimOverlordClient(env.segmentManager),
env.loadQueueTaskMaster,
env.loadQueueManager,
new ServiceAnnouncer.Noop(),
@@ -487,7 +487,8 @@ public class CoordinatorSimulationBuilder
null,
ruleManager,
null,
- null
+ null,
+ NoopSegmentMetadataCache.instance()
);
this.configSyncer = EasyMock.niceMock(CoordinatorDynamicConfigSyncer.class);
@@ -636,9 +637,9 @@ public class CoordinatorSimulationBuilder
private static class SimOverlordClient extends NoopOverlordClient
{
- private final SegmentsMetadataManager segmentsMetadataManager;
+ private final TestSegmentsMetadataManager segmentsMetadataManager;
- private SimOverlordClient(SegmentsMetadataManager segmentsMetadataManager)
+ private SimOverlordClient(TestSegmentsMetadataManager segmentsMetadataManager)
{
this.segmentsMetadataManager = segmentsMetadataManager;
}
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
index 3cd7f8c0e2b..66aaaae3a99 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/simulate/TestSegmentsMetadataManager.java
@@ -19,24 +19,16 @@
package org.apache.druid.server.coordinator.simulate;
-import com.google.common.base.Optional;
import com.google.common.base.Predicates;
import com.google.common.collect.Maps;
import org.apache.druid.client.DataSourcesSnapshot;
-import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.metadata.SegmentsMetadataManager;
-import org.apache.druid.metadata.SortOrder;
-import org.apache.druid.server.http.DataSegmentPlus;
import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.Partitions;
import org.apache.druid.timeline.SegmentId;
import org.apache.druid.timeline.SegmentTimeline;
-import org.apache.druid.timeline.VersionedIntervalTimeline;
-import org.joda.time.DateTime;
import org.joda.time.Interval;
import javax.annotation.Nullable;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -55,6 +47,18 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
private volatile DataSourcesSnapshot snapshot;
private volatile boolean pollingStarted;
+ @Override
+ public void start()
+ {
+
+ }
+
+ @Override
+ public void stop()
+ {
+
+ }
+
public boolean addSegment(DataSegment segment)
{
allSegments.put(segment.getId().toString(), segment);
@@ -100,13 +104,11 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
return pollingStarted;
}
- @Override
public int markAsUsedAllNonOvershadowedSegmentsInDataSource(String dataSource)
{
return markNonOvershadowedSegmentsAsUsed(dataSource, Predicates.alwaysTrue());
}
- @Override
public int markAsUsedNonOvershadowedSegmentsInInterval(String dataSource, Interval interval, @Nullable List<String> versions)
{
final Set<String> allowedVersions = versions == null ? null : new HashSet<>(versions);
@@ -117,13 +119,11 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
);
}
- @Override
public int markAsUsedNonOvershadowedSegments(String dataSource, Set<SegmentId> segmentIds)
{
return markNonOvershadowedSegmentsAsUsed(dataSource, segment -> segmentIds.contains(segment.getId()));
}
- @Override
public boolean markSegmentAsUsed(String segmentId)
{
if (!allSegments.containsKey(segmentId)
@@ -135,13 +135,11 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
return true;
}
- @Override
public int markAsUnusedAllSegmentsInDataSource(String dataSource)
{
return markSegmentsAsUnused(segment -> segment.getDataSource().equals(dataSource));
}
- @Override
public int markAsUnusedSegmentsInInterval(String dataSource, Interval interval, @Nullable List<String> versions)
{
final Set<String> eligibleVersions = versions == null ? null : new HashSet<>(versions);
@@ -152,7 +150,6 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
);
}
- @Override
public int markSegmentsAsUnused(Set<SegmentId> segmentIds)
{
return markSegmentsAsUnused(
@@ -160,7 +157,6 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
);
}
- @Override
public boolean markSegmentAsUnused(SegmentId segmentId)
{
boolean updated = usedSegments.remove(segmentId.toString()) != null;
@@ -171,24 +167,8 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
return updated;
}
- @Nullable
- @Override
- public ImmutableDruidDataSource getImmutableDataSourceWithUsedSegments(String dataSource)
- {
- if (snapshot == null) {
- getSnapshotOfDataSourcesWithAllUsedSegments();
- }
- return snapshot.getDataSource(dataSource);
- }
-
@Override
- public Collection<ImmutableDruidDataSource> getImmutableDataSourcesWithAllUsedSegments()
- {
- return getSnapshotOfDataSourcesWithAllUsedSegments().getDataSourcesWithAllUsedSegments();
- }
-
- @Override
- public DataSourcesSnapshot getSnapshotOfDataSourcesWithAllUsedSegments()
+ public DataSourcesSnapshot getRecentDataSourcesSnapshot()
{
if (snapshot == null) {
snapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments.values());
@@ -197,57 +177,17 @@ public class TestSegmentsMetadataManager implements SegmentsMetadataManager
}
@Override
- public Iterable<DataSegment> iterateAllUsedSegments()
- {
- return usedSegments.values();
- }
-
- @Override
- public Optional<Iterable<DataSegment>> iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(
- String datasource,
- Interval interval,
- boolean requiresLatest
- )
+ public DataSourcesSnapshot forceUpdateDataSourcesSnapshot()
{
- VersionedIntervalTimeline<String, DataSegment> usedSegmentsTimeline
- = getSnapshotOfDataSourcesWithAllUsedSegments().getUsedSegmentsTimelinesPerDataSource().get(datasource);
- return Optional.fromNullable(usedSegmentsTimeline)
- .transform(timeline -> timeline.findNonOvershadowedObjectsInInterval(
- interval,
- Partitions.ONLY_COMPLETE
- ));
- }
-
- @Override
- public Iterable<DataSegmentPlus> iterateAllUnusedSegmentsForDatasource(
- String datasource,
- @Nullable Interval interval,
- @Nullable Integer limit,
- @Nullable String lastSegmentId,
- @Nullable SortOrder sortOrder
- )
- {
- return null;
+ snapshot = DataSourcesSnapshot.fromUsedSegments(usedSegments.values());
+ return snapshot;
}
- @Override
public Set<String> retrieveAllDataSourceNames()
{
return allSegments.values().stream().map(DataSegment::getDataSource).collect(Collectors.toSet());
}
- @Override
- public List<Interval> getUnusedSegmentIntervals(
- final String dataSource,
- @Nullable final DateTime minStartTime,
- final DateTime maxEndTime,
- final int limit,
- final DateTime maxUsedStatusLastUpdatedTime
- )
- {
- return null;
- }
-
@Override
public void populateUsedFlagLastUpdatedAsync()
{
diff --git a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
index 67a1db8c31b..0337ae6fdf4 100644
--- a/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/DataSourcesResourceTest.java
@@ -21,7 +21,6 @@ package org.apache.druid.server.http;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
@@ -33,13 +32,16 @@ import it.unimi.dsi.fastutil.objects.Object2LongOpenHashMap;
import org.apache.curator.shaded.com.google.common.base.Charsets;
import org.apache.druid.audit.AuditManager;
import org.apache.druid.client.CoordinatorServerView;
+import org.apache.druid.client.DataSourcesSnapshot;
import org.apache.druid.client.DruidDataSource;
import org.apache.druid.client.DruidServer;
import org.apache.druid.client.ImmutableDruidDataSource;
import org.apache.druid.client.ImmutableSegmentLoadInfo;
import org.apache.druid.client.SegmentLoadInfo;
import org.apache.druid.error.DruidExceptionMatcher;
+import org.apache.druid.error.ErrorResponse;
import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.java.util.common.Intervals;
import org.apache.druid.java.util.http.client.response.StringFullResponseHolder;
import org.apache.druid.metadata.MetadataRuleManager;
@@ -76,7 +78,9 @@ import org.easymock.EasyMock;
import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
import org.jboss.netty.handler.codec.http.HttpResponseStatus;
import org.jboss.netty.handler.codec.http.HttpVersion;
+import org.joda.time.DateTime;
import org.joda.time.Interval;
+import org.joda.time.Period;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -739,15 +743,19 @@ public class DataSourcesResourceTest
.andThrow(new RuntimeException(new HttpResponseException(responseHolder))).once();
EasyMock.replay(overlordClient);
- EasyMock.expect(segmentsMetadataManager.markSegmentAsUsed(segment.getId().toString()))
- .andReturn(true).once();
- EasyMock.replay(segmentsMetadataManager);
-
Response response = dataSourcesResource.markSegmentAsUsed(segment.getDataSource(), segment.getId().toString());
- Assert.assertEquals(200, response.getStatus());
- Assert.assertEquals(new SegmentUpdateResponse(1), response.getEntity());
+ Assert.assertEquals(404, response.getStatus());
+
+ final Object payload = response.getEntity();
+ Assert.assertTrue(payload instanceof ErrorResponse);
- EasyMock.verify(overlordClient, segmentsMetadataManager);
+ final ErrorResponse errorResponse = (ErrorResponse) payload;
+ Assert.assertEquals(
+ "Could not update segments since Overlord is on an older version.",
+ errorResponse.getAsMap().get("errorMessage")
+ );
+
+ EasyMock.verify(overlordClient);
}
@Test
@@ -1433,15 +1441,10 @@ public class DataSourcesResourceTest
@Test
public void testGetDatasourceLoadstatusNoSegmentForInterval()
{
- List<DataSegment> segments = ImmutableList.of();
// Test when datasource fully loaded
EasyMock.expect(
- segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(
- EasyMock.eq(TestDataSource.WIKI),
- EasyMock.anyObject(Interval.class),
- EasyMock.anyBoolean()
- )
- ).andReturn(Optional.of(segments)).once();
+ segmentsMetadataManager.forceUpdateDataSourcesSnapshot()
+ ).andReturn(DataSourcesSnapshot.fromUsedSegments(List.of())).once();
EasyMock.replay(segmentsMetadataManager);
Response response = dataSourcesResource.getDatasourceLoadstatus(TestDataSource.WIKI, true, null, null, null, null);
@@ -1451,9 +1454,10 @@ public class DataSourcesResourceTest
@Test
public void testGetDatasourceLoadstatusDefault()
{
+ final DateTime now = DateTimes.nowUtc();
DataSegment datasource1Segment1 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(5), Period.days(1)),
"",
null,
null,
@@ -1465,7 +1469,7 @@ public class DataSourcesResourceTest
DataSegment datasource1Segment2 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-22/P1D"),
+ new Interval(now.minusDays(4), Period.days(1)),
"",
null,
null,
@@ -1476,7 +1480,7 @@ public class DataSourcesResourceTest
);
DataSegment datasource2Segment1 = new DataSegment(
TestDataSource.KOALA,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(3), Period.days(1)),
"",
null,
null,
@@ -1496,8 +1500,8 @@ public class DataSourcesResourceTest
);
// Test when datasource fully loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once();
EasyMock.replay(segmentsMetadataManager, inventoryView);
@@ -1511,8 +1515,8 @@ public class DataSourcesResourceTest
EasyMock.reset(segmentsMetadataManager, inventoryView);
// Test when datasource half loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once();
EasyMock.replay(segmentsMetadataManager, inventoryView);
@@ -1528,9 +1532,10 @@ public class DataSourcesResourceTest
@Test
public void testGetDatasourceLoadstatusSimple()
{
+ final DateTime now = DateTimes.nowUtc();
DataSegment datasource1Segment1 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(5), Period.days(1)),
"",
null,
null,
@@ -1542,7 +1547,7 @@ public class DataSourcesResourceTest
DataSegment datasource1Segment2 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-22/P1D"),
+ new Interval(now.minusDays(4), Period.days(1)),
"",
null,
null,
@@ -1553,7 +1558,7 @@ public class DataSourcesResourceTest
);
DataSegment datasource2Segment1 = new DataSegment(
TestDataSource.KOALA,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(3), Period.days(1)),
"",
null,
null,
@@ -1573,8 +1578,8 @@ public class DataSourcesResourceTest
);
// Test when datasource fully loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(completedLoadInfoMap).once();
EasyMock.replay(segmentsMetadataManager, inventoryView);
@@ -1588,8 +1593,8 @@ public class DataSourcesResourceTest
EasyMock.reset(segmentsMetadataManager, inventoryView);
// Test when datasource half loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
EasyMock.expect(inventoryView.getLoadInfoForAllSegments()).andReturn(halfLoadedInfoMap).once();
EasyMock.replay(segmentsMetadataManager, inventoryView);
@@ -1605,9 +1610,10 @@ public class DataSourcesResourceTest
@Test
public void testGetDatasourceLoadstatusFull()
{
+ final DateTime now = DateTimes.nowUtc();
DataSegment datasource1Segment1 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(4), Period.days(1)),
"",
null,
null,
@@ -1619,7 +1625,7 @@ public class DataSourcesResourceTest
DataSegment datasource1Segment2 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-22/P1D"),
+ new Interval(now.minusDays(3), Period.days(1)),
"",
null,
null,
@@ -1639,10 +1645,10 @@ public class DataSourcesResourceTest
underReplicationCountsPerDataSourcePerTier.put("tier2", tier2);
// Test when datasource fully loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
DruidCoordinator druidCoordinator = EasyMock.createMock(DruidCoordinator.class);
- EasyMock.expect(druidCoordinator.getTierToDatasourceToUnderReplicatedCount(segments, false))
+ EasyMock.expect(druidCoordinator.getTierToDatasourceToUnderReplicatedCount(EasyMock.anyObject(), EasyMock.eq(false)))
.andReturn(underReplicationCountsPerDataSourcePerTier).once();
EasyMock.replay(segmentsMetadataManager, druidCoordinator);
@@ -1663,9 +1669,10 @@ public class DataSourcesResourceTest
@Test
public void testGetDatasourceLoadstatusFullAndComputeUsingClusterView()
{
+ final DateTime now = DateTimes.nowUtc();
DataSegment datasource1Segment1 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-01/P1D"),
+ new Interval(now.minusDays(5), Period.days(1)),
"",
null,
null,
@@ -1677,7 +1684,7 @@ public class DataSourcesResourceTest
DataSegment datasource1Segment2 = new DataSegment(
TestDataSource.WIKI,
- Intervals.of("2010-01-22/P1D"),
+ new Interval(now.minusDays(4), Period.days(1)),
"",
null,
null,
@@ -1697,10 +1704,10 @@ public class DataSourcesResourceTest
underReplicationCountsPerDataSourcePerTier.put("tier2", tier2);
// Test when datasource fully loaded
- EasyMock.expect(segmentsMetadataManager.iterateAllUsedNonOvershadowedSegmentsForDatasourceInterval(EasyMock.eq(TestDataSource.WIKI), EasyMock.anyObject(Interval.class), EasyMock.anyBoolean()))
- .andReturn(Optional.of(segments)).once();
+ EasyMock.expect(segmentsMetadataManager.forceUpdateDataSourcesSnapshot())
+ .andReturn(DataSourcesSnapshot.fromUsedSegments(segments)).once();
DruidCoordinator druidCoordinator = EasyMock.createMock(DruidCoordinator.class);
- EasyMock.expect(druidCoordinator.getTierToDatasourceToUnderReplicatedCount(segments, true))
+ EasyMock.expect(druidCoordinator.getTierToDatasourceToUnderReplicatedCount(EasyMock.anyObject(), EasyMock.eq(true)))
.andReturn(underReplicationCountsPerDataSourcePerTier).once();
EasyMock.replay(segmentsMetadataManager, druidCoordinator);
diff --git a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java
index 4c6947de51d..36452095c7c 100644
--- a/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java
+++ b/server/src/test/java/org/apache/druid/server/http/MetadataResourceTest.java
@@ -56,7 +56,6 @@ import org.mockito.stubbing.Answer;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.core.Response;
-
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashMap;
@@ -105,12 +104,12 @@ public class MetadataResourceTest
DataSourcesSnapshot dataSourcesSnapshot = Mockito.mock(DataSourcesSnapshot.class);
Mockito.doReturn(dataSourcesSnapshot)
- .when(segmentsMetadataManager).getSnapshotOfDataSourcesWithAllUsedSegments();
+ .when(segmentsMetadataManager).getRecentDataSourcesSnapshot();
Mockito.doReturn(ImmutableList.of(druidDataSource1))
.when(dataSourcesSnapshot).getDataSourcesWithAllUsedSegments();
Mockito.doReturn(druidDataSource1)
- .when(segmentsMetadataManager)
- .getImmutableDataSourceWithUsedSegments(DATASOURCE1);
+ .when(dataSourcesSnapshot)
+ .getDataSource(DATASOURCE1);
coordinator = Mockito.mock(DruidCoordinator.class);
Mockito.doReturn(2).when(coordinator).getReplicationFactor(segments[0].getId());
@@ -132,7 +131,7 @@ public class MetadataResourceTest
.retrieveSegmentForId(DATASOURCE1, segments[5].getId().toString());
Mockito.doAnswer(mockIterateAllUnusedSegmentsForDatasource())
- .when(segmentsMetadataManager)
+ .when(storageCoordinator)
.iterateAllUnusedSegmentsForDatasource(
ArgumentMatchers.any(),
ArgumentMatchers.any(),
diff --git a/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java b/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java
index 23efafe89f8..5e5d0f5ef78 100644
--- a/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java
+++ b/services/src/test/java/org/apache/druid/cli/CliOverlordTest.java
@@ -21,6 +21,8 @@ package org.apache.druid.cli;
import com.google.inject.Injector;
import org.apache.druid.guice.StartupInjectorBuilder;
+import org.apache.druid.metadata.SegmentsMetadataManager;
+import org.apache.druid.metadata.segment.SqlSegmentsMetadataManagerV2;
import org.apache.druid.metadata.segment.cache.HeapMemorySegmentMetadataCache;
import org.apache.druid.metadata.segment.cache.SegmentMetadataCache;
import org.junit.Assert;
@@ -40,5 +42,9 @@ public class CliOverlordTest
final SegmentMetadataCache segmentMetadataCache
= overlordInjector.getInstance(SegmentMetadataCache.class);
Assert.assertTrue(segmentMetadataCache instanceof HeapMemorySegmentMetadataCache);
+
+ final SegmentsMetadataManager segmentsMetadataManager
+ = overlordInjector.getInstance(SegmentsMetadataManager.class);
+ Assert.assertTrue(segmentsMetadataManager instanceof SqlSegmentsMetadataManagerV2);
}
}
diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java
index 738c497caa0..a05bab5b0e1 100644
--- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java
+++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/MetadataSegmentView.java
@@ -52,13 +52,9 @@ import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/**
- * This class polls the Coordinator in background to keep the latest segments.
- * Provides {@link #getSegments()} for others to get the segments.
- *
- * The difference between this class and {@link SegmentsMetadataManager} is that this class resides
- * in Broker's memory, while {@link SegmentsMetadataManager} resides in Coordinator's memory. In
- * fact, this class polls the data from {@link SegmentsMetadataManager} object in the memory of the
- * currently leading Coordinator via HTTP queries.
+ * Builds a view of used segment metadata currently present on the Coordinator
+ * in the {@link SegmentsMetadataManager} object, by polling the REST API
+ * {@code /druid/coordinator/v1/metadata/segments}.
*/
@ManageLifecycle
public class MetadataSegmentView
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org