You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ge...@apache.org on 2024/03/11 14:24:53 UTC
(druid) branch master updated: Add deep storage segment metric (#16072)
This is an automated email from the ASF dual-hosted git repository.
georgew5656 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 94d2a28465c Add deep storage segment metric (#16072)
94d2a28465c is described below
commit 94d2a28465c2b9cb920c54bcfd46ba5acd620597
Author: George Shiqi Wu <ge...@imply.io>
AuthorDate: Mon Mar 11 10:24:46 2024 -0400
Add deep storage segment metric (#16072)
* Add new metric for deepStorage segments
* Add docs
* change metric name
---
docs/operations/metrics.md | 1 +
.../druid/server/coordinator/DruidCoordinator.java | 26 ++++++++++++++++++++++
.../druid/server/coordinator/stats/Stats.java | 2 ++
.../server/coordinator/DruidCoordinatorTest.java | 5 +++++
4 files changed, 34 insertions(+)
diff --git a/docs/operations/metrics.md b/docs/operations/metrics.md
index 6acc9aa8087..01a56cac2d2 100644
--- a/docs/operations/metrics.md
+++ b/docs/operations/metrics.md
@@ -332,6 +332,7 @@ These metrics are for the Druid Coordinator and are reset each time the Coordina
|`segment/unneededEternityTombstone/count`|Number of non-overshadowed eternity tombstones marked as unused.| |Varies|
|`segment/unavailable/count`|Number of unique segments left to load until all used segments are available for queries.|`dataSource`|0|
|`segment/underReplicated/count`|Number of segments, including replicas, left to load until all used segments are available for queries.|`tier`, `dataSource`|0|
+|`segment/availableDeepStorageOnly/count`|Number of unique segments that are only available for querying directly from deep storage.|`dataSource`|Varies|
|`tier/historical/count`|Number of available historical nodes in each tier.|`tier`|Varies|
|`tier/replication/factor`|Configured maximum replication factor in each tier.|`tier`|Varies|
|`tier/required/capacity`|Total capacity in bytes required in each tier.|`tier`|Varies|
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
index 725fbefd6d6..a78e085fc9e 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
@@ -252,6 +252,25 @@ public class DruidCoordinator
return datasourceToUnavailableSegments;
}
+ public Object2IntMap<String> getDatasourceToDeepStorageQueryOnlySegmentCount()
+ {
+ if (segmentReplicationStatus == null) {
+ return Object2IntMaps.emptyMap();
+ }
+
+ final Object2IntOpenHashMap<String> datasourceToDeepStorageOnlySegments = new Object2IntOpenHashMap<>();
+
+ final Iterable<DataSegment> dataSegments = metadataManager.segments().iterateAllUsedSegments();
+ for (DataSegment segment : dataSegments) {
+ SegmentReplicaCount replicaCount = segmentReplicationStatus.getReplicaCountsInCluster(segment.getId());
+ if (replicaCount != null && replicaCount.totalLoaded() == 0 && replicaCount.required() == 0) {
+ datasourceToDeepStorageOnlySegments.addTo(segment.getDataSource(), 1);
+ }
+ }
+
+ return datasourceToDeepStorageOnlySegments;
+ }
+
public Map<String, Double> getDatasourceToLoadStatus()
{
final Map<String, Double> loadStatus = new HashMap<>();
@@ -762,6 +781,13 @@ public class DruidCoordinator
stats.addToSegmentStat(Stats.Segments.UNDER_REPLICATED, tier, dataSource, underReplicatedCount)
)
);
+ getDatasourceToDeepStorageQueryOnlySegmentCount().forEach(
+ (dataSource, numDeepStorageOnly) -> stats.add(
+ Stats.Segments.DEEP_STORAGE_ONLY,
+ RowKey.of(Dimension.DATASOURCE, dataSource),
+ numDeepStorageOnly
+ )
+ );
return params;
}
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java b/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java
index 6ac6633979b..bae0e407037 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/stats/Stats.java
@@ -53,6 +53,8 @@ public class Stats
= CoordinatorStat.toDebugAndEmit("underreplicated", "segment/underReplicated/count");
public static final CoordinatorStat UNAVAILABLE
= CoordinatorStat.toDebugAndEmit("unavailable", "segment/unavailable/count");
+ public static final CoordinatorStat DEEP_STORAGE_ONLY
+ = CoordinatorStat.toDebugAndEmit("deepStorageOnly", "segment/availableDeepStorageOnly/count");
public static final CoordinatorStat UNNEEDED
= CoordinatorStat.toDebugAndEmit("unneeded", "segment/unneeded/count");
public static final CoordinatorStat OVERSHADOWED
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
index d2c70cb4cc4..63ce8f23416 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DruidCoordinatorTest.java
@@ -895,6 +895,11 @@ public class DruidCoordinatorTest extends CuratorTestBase
Assert.assertNotNull(underRepliicationCountsPerDataSourceColdTier);
Assert.assertEquals(0, underRepliicationCountsPerDataSourceColdTier.getLong(dataSource));
+ Object2IntMap<String> numsDeepStorageOnlySegmentsPerDataSource =
+ coordinator.getDatasourceToDeepStorageQueryOnlySegmentCount();
+
+ Assert.assertEquals(1, numsDeepStorageOnlySegmentsPerDataSource.getInt(dataSource));
+
coordinator.stop();
leaderUnannouncerLatch.await();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org