[pinot] branch master updated: add segment size metric on segment push (#8387)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git


The following commit(s) were added to refs/heads/master by this push:
     new 6d2f749  add segment size metric on segment push (#8387)
6d2f749 is described below

commit 6d2f74978e9fff85d0786ec3d231d157c01a770f
Author: Johan Adami <47...@users.noreply.github.com>
AuthorDate: Wed Mar 23 12:50:30 2022 -0400

    add segment size metric on segment push (#8387)
    
    Adds an offline segment size metric only when a new segment is pushed. this is stored as a gauge and based on table name. This is useful to build detectors in case:
    - someone tries to push a segment that is too large
    - your segments are organically growing in size and you want to know once you've hit a threshold
---
 .../main/java/org/apache/pinot/common/metrics/ControllerGauge.java   | 4 ++++
 .../api/resources/PinotSegmentUploadDownloadRestletResource.java     | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
index ae33acc..f6af993 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/metrics/ControllerGauge.java
@@ -69,6 +69,10 @@ public enum ControllerGauge implements AbstractMetrics.Gauge {
   // Estimated size of offline table
   OFFLINE_TABLE_ESTIMATED_SIZE("OfflineTableEstimatedSize", false),
 
+  // Size of the last uploaded offline segment
+  LAST_PUSHED_SEGMENT_SIZE("LastPushedSegmentSize", false),
+
+
   // Table quota based on setting in table config
   TABLE_QUOTA("TableQuotaBasedOnTableConfig", false),
 
diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
index 253a3fb..1d9d2e7 100644
--- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
+++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentUploadDownloadRestletResource.java
@@ -61,6 +61,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.pinot.common.metrics.ControllerGauge;
 import org.apache.pinot.common.metrics.ControllerMeter;
 import org.apache.pinot.common.metrics.ControllerMetrics;
 import org.apache.pinot.common.restlet.resources.StartReplaceSegmentsRequest;
@@ -304,6 +305,10 @@ public class PinotSegmentUploadDownloadRestletResource {
       completeZkOperations(enableParallelPushProtection, headers, finalSegmentFile, tableNameWithType, segmentMetadata,
           segmentName, zkDownloadUri, moveSegmentToFinalLocation, crypterClassName, allowRefresh);
 
+      // We only set this gauge after the segment is successfully pushed.
+      _controllerMetrics.setValueOfTableGauge(tableNameWithType, ControllerGauge.LAST_PUSHED_SEGMENT_SIZE,
+          FileUtils.sizeOfDirectory(tempSegmentDir));
+
       return new SuccessResponse("Successfully uploaded segment: " + segmentName + " of table: " + tableNameWithType);
     } catch (WebApplicationException e) {
       throw e;

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org