You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2024/06/18 08:42:57 UTC

(celeborn) branch main updated: [CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new c7b1b8d61 [CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys
c7b1b8d61 is described below

commit c7b1b8d61ee1b1e6198b7acc93c781fb8c34226c
Author: SteNicholas <pr...@163.com>
AuthorDate: Tue Jun 18 16:31:57 2024 +0800

    [CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys
    
    ### What changes were proposed in this pull request?
    
    Introduce `CleanTaskQueueSize` and `CleanExpiredShuffleKeysTime` to record situation of cleaning up expired shuffle keys.
    
    ### Why are the changes needed?
    
    There is a backlog of task queue for cleaning up shuffle data of expired shuffle keys in the production environment. It's recommended to introduce `CleanTaskQueueSize` and `CleanExpiredShuffleKeysTime` to record the progress of cleaning up expired shuffle keys.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    [Celeborn Grafana Dashboard](https://stenicholas.grafana.net/public-dashboards/4b5a0b79a35e4ddbb18ddccfe2ec06d7)
    
    Closes #2557 from SteNicholas/CELEBORN-1459.
    
    Authored-by: SteNicholas <pr...@163.com>
    Signed-off-by: mingji <fe...@alibaba-inc.com>
---
 METRICS.md                                         |   2 +
 assets/grafana/celeborn-dashboard.json             | 308 +++++++++++++++++++--
 docs/monitoring.md                                 |   3 +
 .../celeborn/service/deploy/worker/Worker.scala    |   8 +-
 .../service/deploy/worker/WorkerSource.scala       |   6 +
 5 files changed, 303 insertions(+), 24 deletions(-)

diff --git a/METRICS.md b/METRICS.md
index 6c9dc05db..dea75b17f 100644
--- a/METRICS.md
+++ b/METRICS.md
@@ -143,6 +143,8 @@ Here is an example of Grafana dashboard importing.
 |        ReadBufferAllocatedCount        |      worker       |                                    This value means count of allocated read buffer.                                    |
 |        ActiveCreditStreamCount         |      worker       |                       This value means active count of stream for map partition reading streams.                       |
 |        ActiveMapPartitionCount         |      worker       |                            This value means count of active map partition reading streams.                             |
+|           CleanTaskQueueSize           |      worker       |                          This value means count of task for cleaning up expired shuffle keys.                          |
+|      CleanExpiredShuffleKeysTime       |      worker       |                      CleanExpiredShuffleKeys means clean up shuffle data of expired shuffle keys.                      |
 |           DeviceOSFreeBytes            |      worker       |                             This value means actual usable space of OS for device monitor.                             |
 |           DeviceOSTotalBytes           |      worker       |                             This value means total usable space of OS for device monitor.                              |
 |         PotentialConsumeSpeed          |      worker       |                        This value means speed of potential consumption for congestion control.                         |
diff --git a/assets/grafana/celeborn-dashboard.json b/assets/grafana/celeborn-dashboard.json
index bc9471ff9..70b26ce3f 100644
--- a/assets/grafana/celeborn-dashboard.json
+++ b/assets/grafana/celeborn-dashboard.json
@@ -2536,6 +2536,270 @@
           ],
           "title": "metrics_ActiveSlotsCount_Value",
           "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 54
+          },
+          "id": 49,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "expr": "metrics_CleanTaskQueueSize_Value",
+              "legendFormat": "${baseLegend}",
+              "range": true,
+              "refId": "A"
+            }
+          ],
+          "title": "metrics_CleanTaskQueueSize_Value",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "ms"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 62
+          },
+          "id": 51,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "expr": "metrics_CleanExpiredShuffleKeysTime_Mean",
+              "legendFormat": "${baseLegend}",
+              "refId": "A"
+            }
+          ],
+          "title": "metrics_CleanExpiredShuffleKeysTime_Mean",
+          "type": "timeseries"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "fieldConfig": {
+            "defaults": {
+              "color": {
+                "mode": "palette-classic"
+              },
+              "custom": {
+                "axisLabel": "",
+                "axisPlacement": "auto",
+                "barAlignment": 0,
+                "drawStyle": "line",
+                "fillOpacity": 0,
+                "gradientMode": "none",
+                "hideFrom": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": false
+                },
+                "lineInterpolation": "linear",
+                "lineWidth": 1,
+                "pointSize": 5,
+                "scaleDistribution": {
+                  "type": "linear"
+                },
+                "showPoints": "auto",
+                "spanNulls": false,
+                "stacking": {
+                  "group": "A",
+                  "mode": "none"
+                },
+                "thresholdsStyle": {
+                  "mode": "off"
+                }
+              },
+              "mappings": [],
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green"
+                  },
+                  {
+                    "color": "red",
+                    "value": 80
+                  }
+                ]
+              },
+              "unit": "ms"
+            },
+            "overrides": []
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 62
+          },
+          "id": 52,
+          "options": {
+            "legend": {
+              "calcs": [],
+              "displayMode": "list",
+              "placement": "bottom",
+              "showLegend": true
+            },
+            "tooltip": {
+              "mode": "single",
+              "sort": "none"
+            }
+          },
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "expr": "metrics_CleanExpiredShuffleKeysTime_Max",
+              "legendFormat": "${baseLegend}",
+              "refId": "A"
+            }
+          ],
+          "title": "metrics_CleanExpiredShuffleKeysTime_Max",
+          "type": "timeseries"
         }
       ],
       "title": "Worker",
@@ -2547,7 +2811,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 43
+        "y": 71
       },
       "id": 134,
       "panels": [
@@ -2612,7 +2876,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 137
+            "y": 72
           },
           "id": 68,
           "options": {
@@ -2702,7 +2966,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 137
+            "y": 72
           },
           "id": 70,
           "options": {
@@ -2792,7 +3056,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 146
+            "y": 81
           },
           "id": 72,
           "options": {
@@ -2882,7 +3146,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 146
+            "y": 81
           },
           "id": 74,
           "options": {
@@ -2971,7 +3235,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 155
+            "y": 90
           },
           "id": 83,
           "options": {
@@ -3062,7 +3326,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 155
+            "y": 90
           },
           "id": 76,
           "options": {
@@ -3153,7 +3417,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 163
+            "y": 98
           },
           "id": 128,
           "options": {
@@ -3244,7 +3508,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 163
+            "y": 98
           },
           "id": 129,
           "options": {
@@ -3335,7 +3599,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 171
+            "y": 106
           },
           "id": 130,
           "options": {
@@ -3426,7 +3690,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 171
+            "y": 106
           },
           "id": 132,
           "options": {
@@ -3608,7 +3872,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 179
+            "y": 114
           },
           "id": 133,
           "options": {
@@ -3699,7 +3963,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 158
+            "y": 122
           },
           "id": 79,
           "options": {
@@ -3740,7 +4004,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 44
+        "y": 72
       },
       "id": 12,
       "panels": [
@@ -4569,7 +4833,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 45
+        "y": 73
       },
       "id": 10,
       "panels": [
@@ -5123,7 +5387,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 46
+        "y": 74
       },
       "id": 8,
       "panels": [
@@ -6522,7 +6786,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 47
+        "y": 75
       },
       "id": 50,
       "panels": [
@@ -7078,7 +7342,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 48
+        "y": 76
       },
       "id": 157,
       "panels": [
@@ -7371,7 +7635,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 49
+        "y": 77
       },
       "id": 137,
       "panels": [
@@ -8762,7 +9026,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 50
+        "y": 78
       },
       "id": 110,
       "panels": [
@@ -8958,7 +9222,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 51
+        "y": 79
       },
       "id": 123,
       "panels": [
@@ -9436,7 +9700,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 52
+        "y": 80
       },
       "id": 172,
       "panels": [
diff --git a/docs/monitoring.md b/docs/monitoring.md
index b4eba8686..a9d4e7a43 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -229,6 +229,9 @@ These metrics are exposed by Celeborn worker.
     - ActiveCreditStreamCount
         - Active stream count for map partition reading streams.
     - ActiveMapPartitionCount
+    - CleanTaskQueueSize
+    - CleanExpiredShuffleKeysTime
+        - The time for a worker to clean up shuffle data of expired shuffle keys.
     - DeviceOSFreeBytes
     - DeviceOSTotalBytes
     - DeviceCelebornFreeBytes
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index c379b812e..31464c6f5 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -20,7 +20,7 @@ package org.apache.celeborn.service.deploy.worker
 import java.io.File
 import java.lang.{Long => JLong}
 import java.util
-import java.util.{HashMap => JHashMap, HashSet => JHashSet, Locale, Map => JMap}
+import java.util.{HashMap => JHashMap, HashSet => JHashSet, Locale, Map => JMap, UUID}
 import java.util.concurrent._
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicIntegerArray}
 
@@ -728,7 +728,11 @@ private[celeborn] class Worker(
       threadPool.execute(new Runnable {
         override def run(): Unit = {
           removeAppActiveConnection(expiredApplicationIds)
-          storageManager.cleanupExpiredShuffleKey(expiredShuffleKeys)
+          workerSource.sample(
+            WorkerSource.CLEAN_EXPIRED_SHUFFLE_KEYS_TIME,
+            s"cleanExpiredShuffleKeys-${UUID.randomUUID()}") {
+            storageManager.cleanupExpiredShuffleKey(expiredShuffleKeys)
+          }
         }
       })
     }
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
index eb74d6f12..efd596002 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
@@ -78,6 +78,8 @@ class WorkerSource(conf: CelebornConf) extends AbstractSource(conf, MetricsSyste
   addTimer(TAKE_BUFFER_TIME)
   addTimer(SORT_TIME)
 
+  addTimer(CLEAN_EXPIRED_SHUFFLE_KEYS_TIME)
+
   def getCounterCount(metricsName: String): Long = {
     val metricNameWithLabel = metricNameWithCustomizedLabels(metricsName, Map.empty)
     namedCounters.get(metricNameWithLabel).counter.getCount
@@ -211,4 +213,8 @@ object WorkerSource {
 
   // decommission
   val IS_DECOMMISSIONING_WORKER = "IsDecommissioningWorker"
+
+  // clean
+  val CLEAN_TASK_QUEUE_SIZE = "CleanTaskQueueSize"
+  val CLEAN_EXPIRED_SHUFFLE_KEYS_TIME = "CleanExpiredShuffleKeysTime"
 }