You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2024/06/18 08:57:04 UTC
(celeborn) branch branch-0.5 updated: [CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys
This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch branch-0.5
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.5 by this push:
new abb74a987 [CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys
abb74a987 is described below
commit abb74a987cd7ca202977697d58cf20cd8989417d
Author: SteNicholas <pr...@163.com>
AuthorDate: Tue Jun 18 16:31:57 2024 +0800
[CELEBORN-1459] Introduce CleanTaskQueueSize and CleanExpiredShuffleKeysTime to record situation of cleaning up expired shuffle keys
Introduce `CleanTaskQueueSize` and `CleanExpiredShuffleKeysTime` to record situation of cleaning up expired shuffle keys.
There is a backlog of task queue for cleaning up shuffle data of expired shuffle keys in the production environment. It's recommended to introduce `CleanTaskQueueSize` and `CleanExpiredShuffleKeysTime` to record the progress of cleaning up expired shuffle keys.
No.
[Celeborn Grafana Dashboard](https://stenicholas.grafana.net/public-dashboards/4b5a0b79a35e4ddbb18ddccfe2ec06d7)
Closes #2557 from SteNicholas/CELEBORN-1459.
Authored-by: SteNicholas <pr...@163.com>
Signed-off-by: mingji <fe...@alibaba-inc.com>
(cherry picked from commit c7b1b8d61ee1b1e6198b7acc93c781fb8c34226c)
Signed-off-by: mingji <fe...@alibaba-inc.com>
---
METRICS.md | 2 +
assets/grafana/celeborn-dashboard.json | 484 ++++++++++++++++++++-
docs/monitoring.md | 3 +
.../celeborn/service/deploy/worker/Worker.scala | 8 +-
.../service/deploy/worker/WorkerSource.scala | 9 +
5 files changed, 482 insertions(+), 24 deletions(-)
diff --git a/METRICS.md b/METRICS.md
index f6fed40e9..629696ab8 100644
--- a/METRICS.md
+++ b/METRICS.md
@@ -142,6 +142,8 @@ Here is an example of Grafana dashboard importing.
| ReadBufferAllocatedCount | worker | This value means count of allocated read buffer. |
| ActiveCreditStreamCount | worker | This value means active count of stream for map partition reading streams. |
| ActiveMapPartitionCount | worker | This value means count of active map partition reading streams. |
+| CleanTaskQueueSize | worker | This value means count of task for cleaning up expired shuffle keys. |
+| CleanExpiredShuffleKeysTime | worker | CleanExpiredShuffleKeys means clean up shuffle data of expired shuffle keys. |
| DeviceOSFreeBytes | worker | This value means actual usable space of OS for device monitor. |
| DeviceOSTotalBytes | worker | This value means total usable space of OS for device monitor. |
| PotentialConsumeSpeed | worker | This value means speed of potential consumption for congestion control. |
diff --git a/assets/grafana/celeborn-dashboard.json b/assets/grafana/celeborn-dashboard.json
index 45f433310..3f160ccda 100644
--- a/assets/grafana/celeborn-dashboard.json
+++ b/assets/grafana/celeborn-dashboard.json
@@ -2380,6 +2380,446 @@
],
"title": "metrics_ActiveConnectionCount_Count",
"type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 46
+ },
+ "id": 190,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "metrics_IsDecommissioningWorker_Value",
+ "legendFormat": "${baseLegend}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_IsDecommissioningWorker_Value",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 54
+ },
+ "id": 48,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "metrics_ActiveSlotsCount_Value",
+ "legendFormat": "${baseLegend}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_ActiveSlotsCount_Value",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 54
+ },
+ "id": 49,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "metrics_CleanTaskQueueSize_Value",
+ "legendFormat": "${baseLegend}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_CleanTaskQueueSize_Value",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 62
+ },
+ "id": 51,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "metrics_CleanExpiredShuffleKeysTime_Mean",
+ "legendFormat": "${baseLegend}",
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_CleanExpiredShuffleKeysTime_Mean",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 62
+ },
+ "id": 52,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${DS_PROMETHEUS}"
+ },
+ "expr": "metrics_CleanExpiredShuffleKeysTime_Max",
+ "legendFormat": "${baseLegend}",
+ "refId": "A"
+ }
+ ],
+ "title": "metrics_CleanExpiredShuffleKeysTime_Max",
+ "type": "timeseries"
}
],
"title": "Worker",
@@ -2391,7 +2831,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 43
+ "y": 71
},
"id": 134,
"panels": [
@@ -2456,7 +2896,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 137
+ "y": 72
},
"id": 68,
"options": {
@@ -2546,7 +2986,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 137
+ "y": 72
},
"id": 70,
"options": {
@@ -2636,7 +3076,7 @@
"h": 9,
"w": 12,
"x": 0,
- "y": 146
+ "y": 81
},
"id": 72,
"options": {
@@ -2726,7 +3166,7 @@
"h": 9,
"w": 12,
"x": 12,
- "y": 146
+ "y": 81
},
"id": 74,
"options": {
@@ -2815,7 +3255,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 155
+ "y": 90
},
"id": 83,
"options": {
@@ -2906,7 +3346,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 155
+ "y": 90
},
"id": 76,
"options": {
@@ -2997,7 +3437,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 163
+ "y": 98
},
"id": 128,
"options": {
@@ -3088,7 +3528,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 163
+ "y": 98
},
"id": 129,
"options": {
@@ -3179,7 +3619,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 171
+ "y": 106
},
"id": 130,
"options": {
@@ -3270,7 +3710,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 171
+ "y": 106
},
"id": 132,
"options": {
@@ -3452,7 +3892,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 179
+ "y": 114
},
"id": 133,
"options": {
@@ -3543,7 +3983,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 158
+ "y": 122
},
"id": 79,
"options": {
@@ -3584,7 +4024,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 44
+ "y": 72
},
"id": 12,
"panels": [
@@ -4413,7 +4853,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 45
+ "y": 73
},
"id": 10,
"panels": [
@@ -4967,7 +5407,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 46
+ "y": 74
},
"id": 8,
"panels": [
@@ -6366,7 +6806,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 47
+ "y": 75
},
"id": 50,
"panels": [
@@ -6922,7 +7362,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 48
+ "y": 76
},
"id": 157,
"panels": [
@@ -7215,7 +7655,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 49
+ "y": 77
},
"id": 137,
"panels": [
@@ -8606,7 +9046,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 50
+ "y": 78
},
"id": 110,
"panels": [
@@ -8802,7 +9242,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 51
+ "y": 79
},
"id": 123,
"panels": [
@@ -9280,7 +9720,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 52
+ "y": 80
},
"id": 172,
"panels": [
diff --git a/docs/monitoring.md b/docs/monitoring.md
index cc3badbb4..0e6056b84 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -228,6 +228,9 @@ These metrics are exposed by Celeborn worker.
- ActiveCreditStreamCount
- Active stream count for map partition reading streams.
- ActiveMapPartitionCount
+ - CleanTaskQueueSize
+ - CleanExpiredShuffleKeysTime
+ - The time for a worker to clean up shuffle data of expired shuffle keys.
- DeviceOSFreeBytes
- DeviceOSTotalBytes
- DeviceCelebornFreeBytes
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 16f4e4fb3..e3987a195 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -20,7 +20,7 @@ package org.apache.celeborn.service.deploy.worker
import java.io.File
import java.lang.{Long => JLong}
import java.util
-import java.util.{HashMap => JHashMap, HashSet => JHashSet, Locale, Map => JMap}
+import java.util.{HashMap => JHashMap, HashSet => JHashSet, Locale, Map => JMap, UUID}
import java.util.concurrent._
import java.util.concurrent.atomic.{AtomicBoolean, AtomicIntegerArray}
@@ -716,7 +716,11 @@ private[celeborn] class Worker(
threadPool.execute(new Runnable {
override def run(): Unit = {
removeAppActiveConnection(expiredApplicationIds)
- storageManager.cleanupExpiredShuffleKey(expiredShuffleKeys)
+ workerSource.sample(
+ WorkerSource.CLEAN_EXPIRED_SHUFFLE_KEYS_TIME,
+ s"cleanExpiredShuffleKeys-${UUID.randomUUID()}") {
+ storageManager.cleanupExpiredShuffleKey(expiredShuffleKeys)
+ }
}
})
}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
index 5fbc4dde7..efd596002 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerSource.scala
@@ -78,6 +78,8 @@ class WorkerSource(conf: CelebornConf) extends AbstractSource(conf, MetricsSyste
addTimer(TAKE_BUFFER_TIME)
addTimer(SORT_TIME)
+ addTimer(CLEAN_EXPIRED_SHUFFLE_KEYS_TIME)
+
def getCounterCount(metricsName: String): Long = {
val metricNameWithLabel = metricNameWithCustomizedLabels(metricsName, Map.empty)
namedCounters.get(metricNameWithLabel).counter.getCount
@@ -208,4 +210,11 @@ object WorkerSource {
// active shuffle
val ACTIVE_SHUFFLE_SIZE = "ActiveShuffleSize"
val ACTIVE_SHUFFLE_FILE_COUNT = "ActiveShuffleFileCount"
+
+ // decommission
+ val IS_DECOMMISSIONING_WORKER = "IsDecommissioningWorker"
+
+ // clean
+ val CLEAN_TASK_QUEUE_SIZE = "CleanTaskQueueSize"
+ val CLEAN_EXPIRED_SHUFFLE_KEYS_TIME = "CleanExpiredShuffleKeysTime"
}