You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by jl...@apache.org on 2021/10/11 20:09:54 UTC
[pinot] 01/01: Clean up controller-table related metrics in
ControllerPeriodicTask
This is an automated email from the ASF dual-hosted git repository.
jlli pushed a commit to branch clean-up-non-leader-metrics
in repository https://gitbox.apache.org/repos/asf/pinot.git
commit ecffaf7b9243dcce7c6a881821ef5ab850c36820
Author: Jack Li(Analytics Engineering) <jl...@jlli-mn1.linkedin.biz>
AuthorDate: Mon Oct 11 13:07:53 2021 -0700
Clean up controller-table related metrics in ControllerPeriodicTask
---
.../pinot/common/metrics/ValidationMetrics.java | 57 ++++++++++++++++++++++
.../controller/helix/SegmentStatusChecker.java | 16 ++++++
.../core/periodictask/ControllerPeriodicTask.java | 14 ++++++
.../validation/OfflineSegmentIntervalChecker.java | 14 ++++++
.../RealtimeSegmentValidationManager.java | 10 ++++
5 files changed, 111 insertions(+)
diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java
index efe3c82..e9208f8 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ValidationMetrics.java
@@ -141,6 +141,16 @@ public class ValidationMetrics {
}
/**
+ * Cleans up the missing segment count gauge.
+ *
+ * @param resource The resource for which the gauge is removed
+ */
+ public void cleanupMissingSegmentCountGauge(final String resource) {
+ final String fullGaugeName = makeGaugeName(resource, "missingSegmentCount");
+ removeGauge(fullGaugeName);
+ }
+
+ /**
* Updates the offline segment delay gauge.
*
* @param resource The resource for which the gauge is updated
@@ -154,6 +164,16 @@ public class ValidationMetrics {
}
/**
+ * Cleans up offline segment delay gauge.
+ *
+ * @param resource The resource for which the gauge is removed
+ */
+ public void cleanupOfflineSegmentDelayGauge(final String resource) {
+ final String fullGaugeNameHours = makeGaugeName(resource, "offlineSegmentDelayHours");
+ removeGauge(fullGaugeNameHours);
+ }
+
+ /**
* Updates the last push time gauge.
*
* @param resource The resource for which the gauge is updated
@@ -167,6 +187,16 @@ public class ValidationMetrics {
}
/**
+ * Cleans up the last push time gauge.
+ *
+ * @param resource The resource for which the gauge is removed
+ */
+ public void cleanupLastPushTimeGauge(final String resource) {
+ final String fullGaugeNameHours = makeGaugeName(resource, "lastPushTimeDelayHours");
+ removeGauge(fullGaugeNameHours);
+ }
+
+ /**
* Updates the total document count gauge.
*
* @param resource The resource for which the gauge is updated
@@ -178,6 +208,16 @@ public class ValidationMetrics {
}
/**
+ * Cleans up the total document count gauge.
+ *
+ * @param resource The resource for which the gauge is removed
+ */
+ public void cleanupTotalDocumentCountGauge(final String resource) {
+ final String fullGaugeName = makeGaugeName(resource, "TotalDocumentCount");
+ removeGauge(fullGaugeName);
+ }
+
+ /**
* Updates the non consuming partition count metric.
*
* @param resource The resource for which the gauge is updated
@@ -199,6 +239,16 @@ public class ValidationMetrics {
makeGauge(fullGaugeName, makeMetricName(fullGaugeName), _storedValueGaugeFactory, segmentCount);
}
+ /**
+ * Cleans up the segment count gauge.
+ *
+ * @param resource The resource for which the gauge is removed
+ */
+ public void cleanupSegmentCountGauge(final String resource) {
+ final String fullGaugeName = makeGaugeName(resource, "SegmentCount");
+ removeGauge(fullGaugeName);
+ }
+
@VisibleForTesting
public static String makeGaugeName(final String resource, final String gaugeName) {
return "pinot.controller." + resource + "." + gaugeName;
@@ -219,6 +269,13 @@ public class ValidationMetrics {
}
}
+ private void removeGauge(final String gaugeName) {
+ PinotMetricName pinotMetricName = makeMetricName(gaugeName);
+ PinotMetricUtils.removeMetric(_metricsRegistry, pinotMetricName);
+ _metricNames.remove(pinotMetricName);
+ _gaugeValues.remove(gaugeName);
+ }
+
/**
* Unregisters all validation metrics.
*/
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
index d7d74f9..93350fe 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/SegmentStatusChecker.java
@@ -240,6 +240,22 @@ public class SegmentStatusChecker extends ControllerPeriodicTask<SegmentStatusCh
}
}
+ @Override
+ protected void nonLeaderCleanup(List<String> tableNamesWithType) {
+ for (String tableNameWithType : tableNamesWithType) {
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.NUMBER_OF_REPLICAS);
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_OF_REPLICAS);
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE);
+
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.IDEALSTATE_ZNODE_SIZE);
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.IDEALSTATE_ZNODE_BYTE_SIZE);
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.SEGMENT_COUNT);
+
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.SEGMENTS_IN_ERROR_STATE);
+ _controllerMetrics.removeTableGauge(tableNameWithType, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE);
+ }
+ }
+
private void setStatusToDefault() {
List<String> allTableNames = _pinotHelixResourceManager.getAllTables();
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
index 9d7a676..439f8be 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/periodictask/ControllerPeriodicTask.java
@@ -65,11 +65,14 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask {
// Process the tables that are managed by this controller
List<String> tablesToProcess = new ArrayList<>();
+ List<String> nonLeaderForTables = new ArrayList<>();
if (propTableNameWithType == null) {
// Table name is not available, so task should run on all tables for which this controller is the lead.
for (String tableNameWithType : _pinotHelixResourceManager.getAllTables()) {
if (_leadControllerManager.isLeaderForTable(tableNameWithType)) {
tablesToProcess.add(tableNameWithType);
+ } else {
+ nonLeaderForTables.add(tableNameWithType);
}
}
} else {
@@ -82,6 +85,9 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask {
if (!tablesToProcess.isEmpty()) {
processTables(tablesToProcess);
}
+ if (!nonLeaderForTables.isEmpty()) {
+ nonLeaderCleanup(nonLeaderForTables);
+ }
} catch (Exception e) {
LOGGER.error("Caught exception while running task: {}", _taskName, e);
_controllerMetrics.addMeteredTableValue(_taskName, ControllerMeter.CONTROLLER_PERIODIC_TASK_ERROR, 1L);
@@ -156,4 +162,12 @@ public abstract class ControllerPeriodicTask<C> extends BasePeriodicTask {
*/
protected void postprocess() {
}
+
+ /**
+ * Can be overridden to perform cleanups for tables that the current controller isn't the leader.
+ *
+ * @param tableNamesWithType the table names that the current controller isn't the leader for
+ */
+ protected void nonLeaderCleanup(List<String> tableNamesWithType) {
+ }
}
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
index 1e11d1e..7a88d36 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/OfflineSegmentIntervalChecker.java
@@ -136,6 +136,20 @@ public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask<Void>
_validationMetrics.updateSegmentCountGauge(offlineTableName, numSegments);
}
+ @Override
+ protected void nonLeaderCleanup(List<String> tableNamesWithType) {
+ for (String tableNameWithType : tableNamesWithType) {
+ TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
+ if (tableType == TableType.OFFLINE) {
+ _validationMetrics.cleanupMissingSegmentCountGauge(tableNameWithType);
+ _validationMetrics.cleanupOfflineSegmentDelayGauge(tableNameWithType);
+ _validationMetrics.cleanupLastPushTimeGauge(tableNameWithType);
+ _validationMetrics.cleanupTotalDocumentCountGauge(tableNameWithType);
+ _validationMetrics.cleanupSegmentCountGauge(tableNameWithType);
+ }
+ }
+ }
+
/**
* Computes the number of missing segments based on the given existing segment intervals and the expected frequency
* of the intervals.
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
index 53291b3..237924a 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/validation/RealtimeSegmentValidationManager.java
@@ -124,6 +124,16 @@ public class RealtimeSegmentValidationManager extends ControllerPeriodicTask<Rea
}
}
+ @Override
+ protected void nonLeaderCleanup(List<String> tableNamesWithType) {
+ for (String tableNameWithType : tableNamesWithType) {
+ TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableNameWithType);
+ if (tableType == TableType.REALTIME) {
+ _validationMetrics.cleanupTotalDocumentCountGauge(tableNameWithType);
+ }
+ }
+ }
+
@VisibleForTesting
static long computeRealtimeTotalDocumentInSegments(List<SegmentZKMetadata> segmentsZKMetadata,
boolean countHLCSegments) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org