You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2022/11/22 12:06:41 UTC
[incubator-celeborn] branch main updated: [CELEBORN-4] Add metrics about top disk used apps. (#985)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new ee243f28 [CELEBORN-4] Add metrics about top disk used apps. (#985)
ee243f28 is described below
commit ee243f286deaa2d3c0dbf8413aa6115478a00dba
Author: Ethan Feng <et...@apache.org>
AuthorDate: Tue Nov 22 20:06:36 2022 +0800
[CELEBORN-4] Add metrics about top disk used apps. (#985)
---
common/src/main/proto/TransportMessages.proto | 3 +-
.../org/apache/celeborn/common/CelebornConf.scala | 30 +++++
.../common/protocol/message/ControlMessages.scala | 22 ++-
docs/configuration/master.md | 3 +
.../master/clustermeta/AbstractMetaManager.java | 18 +++
.../master/clustermeta/IMetadataHandler.java | 1 +
.../clustermeta/SingleMasterMetaManager.java | 13 +-
.../master/clustermeta/ha/HAMasterMetaManager.java | 4 +
.../deploy/master/clustermeta/ha/MetaHandler.java | 29 ++--
master/src/main/proto/Resource.proto | 1 +
.../celeborn/service/deploy/master/Master.scala | 16 ++-
.../deploy/master/metrics/AppDiskUsageMetric.scala | 150 +++++++++++++++++++++
.../clustermeta/DefaultMetaSystemSuiteJ.java | 9 +-
.../ha/RatisMasterStatusSystemSuiteJ.java | 3 +
.../deploy/master/AppDiskUsageMetricSuite.scala | 104 ++++++++++++++
.../celeborn/server/common/HttpService.scala | 2 +
.../server/common/http/HttpRequestHandler.scala | 2 +
.../celeborn/service/deploy/worker/Worker.scala | 25 +++-
.../deploy/worker/storage/StorageManager.scala | 12 ++
19 files changed, 413 insertions(+), 34 deletions(-)
diff --git a/common/src/main/proto/TransportMessages.proto b/common/src/main/proto/TransportMessages.proto
index 96ef54d9..a8c4f766 100644
--- a/common/src/main/proto/TransportMessages.proto
+++ b/common/src/main/proto/TransportMessages.proto
@@ -138,9 +138,10 @@ message PbHeartbeatFromWorker {
int32 fetchPort = 4;
int32 replicatePort = 5;
repeated PbDiskInfo disks = 6;
- repeated string shuffleKeys = 7;
+ repeated string activeShuffleKeys = 7;
string requestId = 8;
map<string, PbResourceConsumption> userResourceConsumption = 9;
+ map<string, int64> estimatedAppDiskUsage = 10;
}
message PbHeartbeatResponse {
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index df0c1933..ffcea627 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -479,6 +479,9 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY)
def estimatedPartitionSizeForEstimationUpdateInterval: Long =
get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL)
+ def metricsAppTopDiskUsageCount: Int = get(METRICS_APP_TOP_DISK_USAGE_COUNT)
+ def metricsAppTopDiskUsageWindowSize: Int = get(METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE)
+ def metricsAppTopDiskUsageInterval: Long = get(METRICS_APP_TOP_DISK_USAGE_INTERVAL)
// //////////////////////////////////////////////////////
// Worker //
@@ -1957,6 +1960,33 @@ object CelebornConf extends Logging {
.timeConf(TimeUnit.MILLISECONDS)
.createWithDefaultString("10min")
+ val METRICS_APP_TOP_DISK_USAGE_COUNT: ConfigEntry[Int] =
+ buildConf("celeborn.metrics.app.topDiskUsage.count")
+ .withAlternative("rss.metrics.app.topDiskUsage.count")
+ .categories("master")
+ .doc("Size for top items about top disk usage applications list.")
+ .version("0.2.0")
+ .intConf
+ .createWithDefault(50)
+
+ val METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE: ConfigEntry[Int] =
+ buildConf("celeborn.metrics.app.topDiskUsage.windowSize")
+ .withAlternative("rss.metrics.app.topDiskUsage.windowSize")
+ .categories("master")
+ .doc("Window size about top disk usage application list.")
+ .version("0.2.0")
+ .intConf
+ .createWithDefault(24)
+
+ val METRICS_APP_TOP_DISK_USAGE_INTERVAL: ConfigEntry[Long] =
+ buildConf("celeborn.metrics.app.topDiskUsage.interval")
+ .withAlternative("rss.metrics.app.topDiskUsage.interval")
+ .categories("master")
+ .doc("Time length for a window about top disk usage application list.")
+ .version("0.2.0")
+ .timeConf(TimeUnit.SECONDS)
+ .createWithDefaultString("10min")
+
val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
buildConf("celeborn.push.stageEnd.timeout")
.withAlternative("rss.stage.end.timeout")
diff --git a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
index b92a996c..7eff974b 100644
--- a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
@@ -109,7 +109,8 @@ object ControlMessages extends Logging {
replicatePort: Int,
disks: Seq[DiskInfo],
userResourceConsumption: util.Map[UserIdentifier, ResourceConsumption],
- shuffleKeys: util.HashSet[String],
+ activeShuffleKeys: util.Set[String],
+ estimatedAppDiskUsage: util.HashMap[String, java.lang.Long],
override var requestId: String = ZERO_UUID) extends MasterRequestMessage
case class HeartbeatResponse(
@@ -434,7 +435,8 @@ object ControlMessages extends Logging {
replicatePort,
disks,
userResourceConsumption,
- shuffleKeys,
+ activeShuffleKeys,
+ estimatedAppDiskUsage,
requestId) =>
val pbDisks = disks.map(PbSerDeUtils.toPbDiskInfo).asJava
val pbUserResourceConsumption =
@@ -447,7 +449,8 @@ object ControlMessages extends Logging {
.addAllDisks(pbDisks)
.putAllUserResourceConsumption(pbUserResourceConsumption)
.setReplicatePort(replicatePort)
- .addAllShuffleKeys(shuffleKeys)
+ .addAllActiveShuffleKeys(activeShuffleKeys)
+ .putAllEstimatedAppDiskUsage(estimatedAppDiskUsage)
.setRequestId(requestId)
.build().toByteArray
new TransportMessage(MessageType.HEARTBEAT_FROM_WORKER, payload)
@@ -786,12 +789,16 @@ object ControlMessages extends Logging {
case HEARTBEAT_FROM_WORKER =>
val pbHeartbeatFromWorker = PbHeartbeatFromWorker.parseFrom(message.getPayload)
- val shuffleKeys = new util.HashSet[String]()
+ val estimatedAppDiskUsage = new util.HashMap[String, java.lang.Long]()
val userResourceConsumption = PbSerDeUtils.fromPbUserResourceConsumption(
pbHeartbeatFromWorker.getUserResourceConsumptionMap)
val pbDisks = pbHeartbeatFromWorker.getDisksList.asScala.map(PbSerDeUtils.fromPbDiskInfo)
- if (pbHeartbeatFromWorker.getShuffleKeysCount > 0) {
- shuffleKeys.addAll(pbHeartbeatFromWorker.getShuffleKeysList)
+ if (!pbHeartbeatFromWorker.getEstimatedAppDiskUsageMap.isEmpty) {
+ estimatedAppDiskUsage.putAll(pbHeartbeatFromWorker.getEstimatedAppDiskUsageMap)
+ }
+ val activeShuffleKeys = new util.HashSet[String]()
+ if (!pbHeartbeatFromWorker.getActiveShuffleKeysList.isEmpty) {
+ activeShuffleKeys.addAll(pbHeartbeatFromWorker.getActiveShuffleKeysList)
}
HeartbeatFromWorker(
pbHeartbeatFromWorker.getHost,
@@ -801,7 +808,8 @@ object ControlMessages extends Logging {
pbHeartbeatFromWorker.getReplicatePort,
pbDisks,
userResourceConsumption,
- shuffleKeys,
+ activeShuffleKeys,
+ estimatedAppDiskUsage,
pbHeartbeatFromWorker.getRequestId)
case HEARTBEAT_RESPONSE =>
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index 78ca595b..59adaa0c 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -30,6 +30,9 @@ license: |
| celeborn.master.metrics.prometheus.host | 0.0.0.0 | Master's Prometheus host. | 0.2.0 |
| celeborn.master.metrics.prometheus.port | 9098 | Master's Prometheus port. | 0.2.0 |
| celeborn.master.port | 9097 | Port for master to bind. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.count | 50 | Size for top items about top disk usage applications list. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.interval | 10min | Time length for a window about top disk usage application list. | 0.2.0 |
+| celeborn.metrics.app.topDiskUsage.windowSize | 24 | Window size about top disk usage application list. | 0.2.0 |
| celeborn.metrics.capacity | 4096 | The maximum number of metrics which a source can use to generate output strings. | 0.2.0 |
| celeborn.metrics.collectPerfCritical.enabled | false | It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them. | 0.2.0 |
| celeborn.metrics.enabled | true | When true, enable metrics system. | 0.2.0 |
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
index 4f612fea..744052d4 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
@@ -27,6 +27,7 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.atomic.LongAdder;
import org.slf4j.Logger;
@@ -40,6 +41,8 @@ import org.apache.celeborn.common.quota.ResourceConsumption;
import org.apache.celeborn.common.rpc.RpcAddress;
import org.apache.celeborn.common.rpc.RpcEnv;
import org.apache.celeborn.common.util.Utils;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageSnapShot;
public abstract class AbstractMetaManager implements IMetadataHandler {
private static final Logger LOG = LoggerFactory.getLogger(AbstractMetaManager.class);
@@ -61,6 +64,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
public long estimatedPartitionSize;
public final LongAdder partitionTotalWritten = new LongAdder();
public final LongAdder partitionTotalFileCount = new LongAdder();
+ public AppDiskUsageMetric appDiskUsageMetric = null;
public void updateRequestSlotsMeta(
String shuffleKey, String hostName, Map<String, Map<String, Integer>> workerWithAllocations) {
@@ -165,6 +169,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
int replicatePort,
Map<String, DiskInfo> disks,
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+ Map<String, Long> estimatedAppDiskUsage,
long time) {
WorkerInfo worker =
new WorkerInfo(
@@ -188,6 +193,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
info.lastHeartbeat_$eq(time);
});
}
+ appDiskUsageMetric.update(estimatedAppDiskUsage);
if (!blacklist.contains(worker) && disks.isEmpty()) {
LOG.debug("Worker: {} num total slots is 0, add to blacklist", worker);
blacklist.add(worker);
@@ -276,6 +282,11 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
}
});
+ out.writeLong(partitionTotalWritten.sum());
+ out.writeLong(partitionTotalFileCount.sum());
+ out.writeObject(appDiskUsageMetric.snapShots());
+ out.writeObject(appDiskUsageMetric.currentSnapShot());
+
out.flush();
}
@@ -331,6 +342,13 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
for (int i = 0; i < workersSize; ++i) {
workers.add((WorkerInfo) in.readObject());
}
+ partitionTotalWritten.reset();
+ partitionTotalWritten.add(in.readLong());
+ partitionTotalFileCount.reset();
+ partitionTotalFileCount.add(in.readLong());
+ appDiskUsageMetric.restoreFromSnapshot((AppDiskUsageSnapShot[]) in.readObject());
+ appDiskUsageMetric.currentSnapShot_$eq(
+ (AtomicReference<AppDiskUsageSnapShot>) in.readObject());
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
index c09f3e9c..84b65465 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
@@ -59,6 +59,7 @@ public interface IMetadataHandler {
int replicatePort,
Map<String, DiskInfo> disks,
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+ Map<String, Long> estimatedAppDiskUsage,
long time,
String requestId);
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
index 497e8183..2e81777f 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
@@ -29,6 +29,7 @@ import org.apache.celeborn.common.meta.DiskInfo;
import org.apache.celeborn.common.meta.WorkerInfo;
import org.apache.celeborn.common.quota.ResourceConsumption;
import org.apache.celeborn.common.rpc.RpcEnv;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
public class SingleMasterMetaManager extends AbstractMetaManager {
private static final Logger LOG = LoggerFactory.getLogger(SingleMasterMetaManager.class);
@@ -38,6 +39,7 @@ public class SingleMasterMetaManager extends AbstractMetaManager {
this.conf = conf;
this.initialEstimatedPartitionSize = conf.initialEstimatedPartitionSize();
this.estimatedPartitionSize = initialEstimatedPartitionSize;
+ this.appDiskUsageMetric = new AppDiskUsageMetric(conf);
}
@Override
@@ -95,10 +97,19 @@ public class SingleMasterMetaManager extends AbstractMetaManager {
int replicatePort,
Map<String, DiskInfo> disks,
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+ Map<String, Long> estimatedAppDiskUsage,
long time,
String requestId) {
updateWorkerHeartbeatMeta(
- host, rpcPort, pushPort, fetchPort, replicatePort, disks, userResourceConsumption, time);
+ host,
+ rpcPort,
+ pushPort,
+ fetchPort,
+ replicatePort,
+ disks,
+ userResourceConsumption,
+ estimatedAppDiskUsage,
+ time);
}
@Override
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
index 5af627b1..0476df87 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
@@ -37,6 +37,7 @@ import org.apache.celeborn.service.deploy.master.clustermeta.MetaUtil;
import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos;
import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos.ResourceRequest;
import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos.Type;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
public class HAMasterMetaManager extends AbstractMetaManager {
private static final Logger LOG = LoggerFactory.getLogger(HAMasterMetaManager.class);
@@ -48,6 +49,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
this.conf = conf;
this.initialEstimatedPartitionSize = conf.initialEstimatedPartitionSize();
this.estimatedPartitionSize = initialEstimatedPartitionSize;
+ this.appDiskUsageMetric = new AppDiskUsageMetric(conf);
}
public HARaftServer getRatisServer() {
@@ -220,6 +222,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
int replicatePort,
Map<String, DiskInfo> disks,
Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+ Map<String, Long> estimatedAppDiskUsage,
long time,
String requestId) {
try {
@@ -237,6 +240,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
.putAllDisks(MetaUtil.toPbDiskInfos(disks))
.putAllUserResourceConsumption(
MetaUtil.toPbUserResourceConsumption(userResourceConsumption))
+ .putAllEstimatedAppDiskUsage(estimatedAppDiskUsage)
.setTime(time)
.build())
.build());
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
index 4ae4a3e2..839cefa5 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
@@ -19,10 +19,7 @@ package org.apache.celeborn.service.deploy.master.clustermeta.ha;
import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
import java.util.stream.Collectors;
import org.slf4j.Logger;
@@ -100,10 +97,11 @@ public class MetaHandler {
int pushPort;
int fetchPort;
int replicatePort;
- Map<String, DiskInfo> disks;
+ Map<String, DiskInfo> diskInfos;
Map<UserIdentifier, ResourceConsumption> userResourceConsumption;
List<Map<String, Integer>> slots = new ArrayList<>();
Map<String, Map<String, Integer>> workerAllocations = new HashMap<>();
+ Map<String, Long> estimatedAppDiskUsage = new HashMap<>();
switch (cmdType) {
case RequestSlots:
shuffleKey = request.getRequestSlotsRequest().getShuffleKey();
@@ -177,10 +175,12 @@ public class MetaHandler {
rpcPort = request.getWorkerHeartbeatRequest().getRpcPort();
pushPort = request.getWorkerHeartbeatRequest().getPushPort();
fetchPort = request.getWorkerHeartbeatRequest().getFetchPort();
- disks = MetaUtil.fromPbDiskInfos(request.getWorkerHeartbeatRequest().getDisksMap());
+ diskInfos = MetaUtil.fromPbDiskInfos(request.getWorkerHeartbeatRequest().getDisksMap());
userResourceConsumption =
MetaUtil.fromPbUserResourceConsumption(
request.getWorkerHeartbeatRequest().getUserResourceConsumptionMap());
+ estimatedAppDiskUsage.putAll(
+ request.getWorkerHeartbeatRequest().getEstimatedAppDiskUsageMap());
replicatePort = request.getWorkerHeartbeatRequest().getReplicatePort();
LOG.debug(
"Handle worker heartbeat for {} {} {} {} {} {} {}",
@@ -189,7 +189,7 @@ public class MetaHandler {
pushPort,
fetchPort,
replicatePort,
- disks,
+ diskInfos,
userResourceConsumption);
time = request.getWorkerHeartbeatRequest().getTime();
metaSystem.updateWorkerHeartbeatMeta(
@@ -198,8 +198,9 @@ public class MetaHandler {
pushPort,
fetchPort,
replicatePort,
- disks,
+ diskInfos,
userResourceConsumption,
+ estimatedAppDiskUsage,
time);
break;
@@ -209,7 +210,7 @@ public class MetaHandler {
pushPort = request.getRegisterWorkerRequest().getPushPort();
fetchPort = request.getRegisterWorkerRequest().getFetchPort();
replicatePort = request.getRegisterWorkerRequest().getReplicatePort();
- disks = MetaUtil.fromPbDiskInfos(request.getRegisterWorkerRequest().getDisksMap());
+ diskInfos = MetaUtil.fromPbDiskInfos(request.getRegisterWorkerRequest().getDisksMap());
userResourceConsumption =
MetaUtil.fromPbUserResourceConsumption(
request.getRegisterWorkerRequest().getUserResourceConsumptionMap());
@@ -220,10 +221,16 @@ public class MetaHandler {
pushPort,
fetchPort,
replicatePort,
- disks,
+ diskInfos,
userResourceConsumption);
metaSystem.updateRegisterWorkerMeta(
- host, rpcPort, pushPort, fetchPort, replicatePort, disks, userResourceConsumption);
+ host,
+ rpcPort,
+ pushPort,
+ fetchPort,
+ replicatePort,
+ diskInfos,
+ userResourceConsumption);
break;
case ReportWorkerUnavailable:
diff --git a/master/src/main/proto/Resource.proto b/master/src/main/proto/Resource.proto
index 88031761..d0733218 100644
--- a/master/src/main/proto/Resource.proto
+++ b/master/src/main/proto/Resource.proto
@@ -115,6 +115,7 @@ message WorkerHeartbeatRequest {
map<string, DiskInfo> disks = 6;
required int64 time = 7;
map<string, ResourceConsumption> userResourceConsumption = 8;
+ map<string, int64> estimatedAppDiskUsage = 9;
}
message RegisterWorkerRequest {
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 7de2fe53..a930c98f 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -278,7 +278,8 @@ private[celeborn] class Master(
replicatePort,
disks,
userResourceConsumption,
- shuffleKeys,
+ activeShuffleKey,
+ estimatedAppDiskUsage,
requestId) =>
logDebug(s"Received heartbeat from" +
s" worker $host:$rpcPort:$pushPort:$fetchPort with $disks.")
@@ -293,7 +294,8 @@ private[celeborn] class Master(
replicatePort,
disks,
userResourceConsumption,
- shuffleKeys,
+ activeShuffleKey,
+ estimatedAppDiskUsage,
requestId))
case GetWorkerInfos =>
@@ -356,7 +358,8 @@ private[celeborn] class Master(
replicatePort: Int,
disks: Seq[DiskInfo],
userResourceConsumption: util.Map[UserIdentifier, ResourceConsumption],
- shuffleKeys: util.HashSet[String],
+ activeShuffleKeys: util.Set[String],
+ estimatedAppDiskUsage: util.HashMap[String, java.lang.Long],
requestId: String): Unit = {
val targetWorker = new WorkerInfo(host, rpcPort, pushPort, fetchPort, replicatePort)
val registered = workersSnapShot.asScala.contains(targetWorker)
@@ -372,12 +375,13 @@ private[celeborn] class Master(
replicatePort,
disks.map { disk => disk.mountPoint -> disk }.toMap.asJava,
userResourceConsumption,
+ estimatedAppDiskUsage,
System.currentTimeMillis(),
requestId)
}
val expiredShuffleKeys = new util.HashSet[String]
- shuffleKeys.asScala.foreach { shuffleKey =>
+ activeShuffleKeys.asScala.foreach { shuffleKey =>
if (!statusSystem.registeredShuffle.contains(shuffleKey)) {
logWarning(s"Shuffle $shuffleKey expired on $host:$rpcPort:$pushPort:$fetchPort.")
expiredShuffleKeys.add(shuffleKey)
@@ -693,6 +697,10 @@ private[celeborn] class Master(
statusSystem.registeredShuffle.asScala.mkString("\n")
}
+ override def listTopDiskUseApps: String = {
+ statusSystem.appDiskUsageMetric.summary
+ }
+
private def requestGetWorkerInfos(endpoint: RpcEndpointRef): GetWorkerInfosResponse = {
try {
endpoint.askSync[GetWorkerInfosResponse](GetWorkerInfos)
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala
new file mode 100644
index 00000000..7674f712
--- /dev/null
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.service.deploy.master.metrics
+
+import java.time.LocalDateTime
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
+
+import scala.collection.JavaConverters.mapAsScalaMapConverter
+
+import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.internal.Logging
+import org.apache.celeborn.common.util.{ThreadUtils, Utils}
+
+case class AppDiskUsage(var appId: String, var estimatedUsage: Long) {
+ override def toString: String =
+ s"Application ${appId} used approximate ${Utils.bytesToString(estimatedUsage)} "
+}
+
+class AppDiskUsageSnapShot(val topItemCount: Int) extends Logging with Serializable {
+ val topNItems = new Array[AppDiskUsage](topItemCount)
+ val startSnapShotTime = LocalDateTime.now()
+ var endSnapShotTime: LocalDateTime = _
+
+ def commit(): Unit = {
+ endSnapShotTime = LocalDateTime.now()
+ }
+
+ def updateAppDiskUsage(appId: String, usage: Long): Unit = {
+ val dropIndex = topNItems.indexWhere(usage => usage != null && usage.appId == appId)
+ if (dropIndex != -1) {
+ drop(dropIndex)
+ }
+ val insertIndex = findInsertPosition(usage)
+ if (insertIndex != -1) {
+ shift(insertIndex)
+ topNItems(insertIndex) = AppDiskUsage(appId, usage)
+ }
+ }
+
+ def shift(index: Int): Unit = {
+ for (i <- topItemCount - 1 until index by -1) {
+ topNItems(i) = topNItems(i - 1)
+ }
+ }
+
+ def drop(index: Int): Unit = {
+ for (i <- index until topItemCount - 1) {
+ topNItems(i) = topNItems(i + 1)
+ }
+ topNItems(topItemCount - 1) = null
+ }
+
+ def findInsertPosition(usage: Long): Int = {
+ if (topNItems(0) == null) {
+ return 0
+ }
+ for (i <- 0 until topItemCount) {
+ if (topNItems(i) == null || topNItems(i).estimatedUsage < usage) {
+ return i
+ }
+ }
+ -1
+ }
+
+ override def toString(): String = s"Snapshot " +
+ s"start ${startSnapShotTime} end ${endSnapShotTime}" +
+ s" ${topNItems.filter(_ != null).mkString(",")}"
+}
+
+// This metric collects approximate value because worker won't report all app disk usage value for reducing memory pressure. .
+class AppDiskUsageMetric(conf: CelebornConf) extends Logging {
+ val usageCount = conf.metricsAppTopDiskUsageCount
+ val snapshotCount = conf.metricsAppTopDiskUsageWindowSize
+ val interval = conf.metricsAppTopDiskUsageInterval
+ val snapShots = new Array[AppDiskUsageSnapShot](snapshotCount)
+ val logExecutor =
+ ThreadUtils.newDaemonSingleThreadScheduledExecutor("App_disk_usage_log_thread")
+ val updateExecutor =
+ ThreadUtils.newDaemonSingleThreadExecutor("App_disk_usage_metric_thread")
+ var currentSnapShot: AtomicReference[AppDiskUsageSnapShot] =
+ new AtomicReference[AppDiskUsageSnapShot]()
+
+ def update(appDiskUsage: java.util.Map[String, java.lang.Long]): Unit = {
+ updateExecutor.submit(new Runnable {
+ override def run(): Unit = {
+ if (currentSnapShot.get() != null) {
+ appDiskUsage.asScala.foreach { case (key, usage) =>
+ currentSnapShot.get().updateAppDiskUsage(key, usage)
+ }
+ }
+ }
+ })
+ }
+
+ logExecutor.scheduleAtFixedRate(
+ new Runnable {
+ override def run(): Unit = {
+ if (currentSnapShot.get() != null) {
+ currentSnapShot.get().commit()
+ }
+ currentSnapShot.set(getNewSnapShot())
+ logInfo(s"App Disk Usage Top${usageCount} Report ${summary()}")
+ }
+ },
+ 60,
+ interval,
+ TimeUnit.SECONDS)
+
+ def getNewSnapShot(): AppDiskUsageSnapShot = {
+ for (i <- snapshotCount - 1 until 0 by -1) {
+ snapShots(i) = snapShots(i - 1)
+ }
+ snapShots(0) = new AppDiskUsageSnapShot(usageCount)
+ snapShots(0)
+ }
+
+ def summary(): String = {
+ val stringBuilder = new StringBuilder()
+ for (i <- 0 until snapshotCount) {
+ if (snapShots(i) != null) {
+ stringBuilder.append(snapShots(i))
+ stringBuilder.append(" \n")
+ }
+ }
+ stringBuilder.toString()
+ }
+
+ def restoreFromSnapshot(array: Array[AppDiskUsageSnapShot]): Unit = {
+ for (i <- 0 until (snapshotCount)) {
+ snapShots(i) = array(i)
+ }
+ }
+
+}
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
index c37ce34f..6a4b8193 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
@@ -21,11 +21,7 @@ import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.atomic.AtomicLong;
import org.junit.After;
@@ -518,6 +514,7 @@ public class DefaultMetaSystemSuiteJ {
REPLICATEPORT1,
new HashMap<>(),
userResourceConsumption1,
+ new HashMap<>(),
1,
getNewReqeustId());
@@ -531,6 +528,7 @@ public class DefaultMetaSystemSuiteJ {
REPLICATEPORT2,
new HashMap<>(),
userResourceConsumption2,
+ new HashMap<>(),
1,
getNewReqeustId());
@@ -544,6 +542,7 @@ public class DefaultMetaSystemSuiteJ {
REPLICATEPORT3,
disks1,
userResourceConsumption1,
+ new HashMap<>(),
1,
getNewReqeustId());
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index c55f0467..7414cdd7 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -695,6 +695,7 @@ public class RatisMasterStatusSystemSuiteJ {
REPLICATEPORT1,
new HashMap<>(),
userResourceConsumption1,
+ new HashMap<>(),
1,
getNewReqeustId());
Thread.sleep(3000L);
@@ -711,6 +712,7 @@ public class RatisMasterStatusSystemSuiteJ {
REPLICATEPORT2,
new HashMap<>(),
userResourceConsumption2,
+ new HashMap<>(),
1,
getNewReqeustId());
Thread.sleep(3000L);
@@ -728,6 +730,7 @@ public class RatisMasterStatusSystemSuiteJ {
REPLICATEPORT1,
disks1,
userResourceConsumption1,
+ new HashMap<>(),
1,
getNewReqeustId());
Thread.sleep(3000L);
diff --git a/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
new file mode 100644
index 00000000..91dec16c
--- /dev/null
+++ b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.service.deploy.master
+
+import java.util
+
+import scala.util.Random
+
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.internal.Logging
+import org.apache.celeborn.common.meta.WorkerInfo
+import org.apache.celeborn.service.deploy.master.metrics.{AppDiskUsageMetric, AppDiskUsageSnapShot}
+
+class AppDiskUsageMetricSuite extends AnyFunSuite
+ with BeforeAndAfterAll
+ with BeforeAndAfterEach
+ with Logging {
+ val WORKER1 = new WorkerInfo("host1", 111, 112, 113, 114)
+ val WORKER2 = new WorkerInfo("host2", 211, 212, 213, 214)
+ val WORKER3 = new WorkerInfo("host3", 311, 312, 313, 314)
+
+ test("test snapshot ordering") {
+ val snapShot = new AppDiskUsageSnapShot(50)
+ val rand = new Random()
+ for (i <- 1 to 60) {
+ snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1)
+ }
+ println(snapShot.toString)
+ }
+
+ test("test snapshot ordering with duplicate entries") {
+ val snapShot = new AppDiskUsageSnapShot(50)
+ val rand = new Random()
+ for (i <- 1 to 60) {
+ snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1)
+ }
+ for (i <- 1 to 15) {
+ snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1000000000)
+ }
+
+ println(snapShot.toString)
+ }
+
+ test("test app usage snapshot") {
+ Thread.sleep(5000)
+
+ val conf = new CelebornConf()
+ conf.set("rss.metrics.app.topDiskUsage.windowSize", "5")
+ conf.set("rss.metrids.app.topDiskUsage.interval", "2s")
+ val usageMetric = new AppDiskUsageMetric(conf)
+
+ val map1 = new util.HashMap[String, java.lang.Long]()
+ map1.put("app1", 2874371)
+ map1.put("app2", 43452)
+ map1.put("app3", 2134526)
+ map1.put("app4", 23465463)
+ map1.put("app5", 132456)
+ map1.put("app6", 6535635)
+ usageMetric.update(map1)
+ println(usageMetric.summary())
+ Thread.sleep(2000)
+
+ map1.clear()
+ map1.put("app1", 374524)
+ map1.put("app2", 5234665)
+ map1.put("app3", 24453)
+ map1.put("app4", 2345637)
+ map1.put("app5", 4534)
+ map1.put("app6", 5357)
+ usageMetric.update(map1)
+ println(usageMetric.summary())
+ Thread.sleep(2000)
+
+ map1.clear()
+ map1.put("app1", 12343)
+ map1.put("app2", 3456565)
+ map1.put("app3", 4345)
+ map1.put("app4", 35245268)
+ map1.put("app5", 45367)
+ map1.put("app6", 64345)
+ usageMetric.update(map1)
+ println(usageMetric.summary())
+ Thread.sleep(2500)
+ println(usageMetric.summary())
+ }
+}
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
index b0e6fe05..fece4879 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
@@ -35,6 +35,8 @@ abstract class HttpService extends Service with Logging {
def getShuffleList: String
+ def listTopDiskUseApps: String
+
def startHttpServer(): Unit = {
val handlers =
if (metricsSystem.running) {
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
index 1a480a14..aa6355e2 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
@@ -72,6 +72,8 @@ class HttpRequestHandler(
service.getApplicationList
case "/shuffles" =>
service.getShuffleList
+ case "/listTopDiskUsedApps" =>
+ service.listTopDiskUseApps
case _ => INVALID
}
}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index d324ebfc..a0067a3a 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -18,11 +18,13 @@
package org.apache.celeborn.service.deploy.worker
import java.io.IOException
-import java.util.{HashSet => JHashSet}
+import java.lang.{Long => JLong}
+import java.util.{HashMap => JHashMap, HashSet => JHashSet}
import java.util.concurrent._
import java.util.concurrent.atomic.AtomicBoolean
import scala.collection.JavaConverters._
+import scala.collection.mutable
import com.google.common.annotations.VisibleForTesting
import io.netty.util.HashedWheelTimer
@@ -219,9 +221,13 @@ private[celeborn] class Worker(
_ => memoryTracker.getPausePushDataAndReplicateCounter)
private def heartBeatToMaster(): Unit = {
- val shuffleKeys = new JHashSet[String]
- shuffleKeys.addAll(partitionLocationInfo.shuffleKeySet)
- shuffleKeys.addAll(storageManager.shuffleKeySet())
+ val activeShuffleKeys = new JHashSet[String]()
+ val estimatedAppDiskUsage = new JHashMap[String, JLong]()
+ activeShuffleKeys.addAll(partitionLocationInfo.shuffleKeySet)
+ activeShuffleKeys.addAll(storageManager.shuffleKeySet())
+ storageManager.topAppDiskUsage.asScala.foreach { case (shuffleId, usage) =>
+ estimatedAppDiskUsage.put(shuffleId, usage)
+ }
// During shutdown, return an empty diskInfo list to mark this worker as unavailable,
// and avoid remove this from master's blacklist.
val diskInfos =
@@ -245,7 +251,8 @@ private[celeborn] class Worker(
replicatePort,
diskInfos,
resourceConsumption,
- shuffleKeys),
+ activeShuffleKeys,
+ estimatedAppDiskUsage),
classOf[HeartbeatResponse])
if (response.registered) {
response.expiredShuffleKeys.asScala.foreach(shuffleKey => workerInfo.releaseSlots(shuffleKey))
@@ -432,6 +439,14 @@ private[celeborn] class Worker(
storageManager.shuffleKeySet().asScala.mkString("\n")
}
+ override def listTopDiskUseApps: String = {
+ val stringBuilder = new StringBuilder()
+ storageManager.topAppDiskUsage.asScala.foreach { case (appId, usage) =>
+ stringBuilder.append(s"application ${appId} used ${Utils.bytesToString(usage)} ")
+ }
+ stringBuilder.toString()
+ }
+
@VisibleForTesting
def isRegistered(): Boolean = {
registered.get()
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index f7afd5c6..266ad164 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -371,6 +371,18 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
hashSet
}
+ def topAppDiskUsage: util.Map[String, Long] = {
+ fileInfos.asScala.map { keyedWriters =>
+ {
+ keyedWriters._1 -> keyedWriters._2.values().asScala.map(_.getFileLength).sum
+ }
+ }.toList.map { case (shuffleKey, usage) =>
+ shuffleKey.split("-")(0) -> usage
+ }.groupBy(_._1).map { case (key, values) =>
+ key -> values.map(_._2).sum
+ }.toSeq.sortBy(_._2).reverse.take(conf.metricsAppTopDiskUsageCount * 2).toMap.asJava
+ }
+
def cleanupExpiredShuffleKey(expiredShuffleKeys: util.HashSet[String]): Unit = {
expiredShuffleKeys.asScala.foreach { shuffleKey =>
logInfo(s"Cleanup expired shuffle $shuffleKey.")