You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2022/11/22 12:06:41 UTC

[incubator-celeborn] branch main updated: [CELEBORN-4] Add metrics about top disk used apps. (#985)

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/main by this push:
     new ee243f28 [CELEBORN-4] Add metrics about top disk used apps. (#985)
ee243f28 is described below

commit ee243f286deaa2d3c0dbf8413aa6115478a00dba
Author: Ethan Feng <et...@apache.org>
AuthorDate: Tue Nov 22 20:06:36 2022 +0800

    [CELEBORN-4] Add metrics about top disk used apps. (#985)
---
 common/src/main/proto/TransportMessages.proto      |   3 +-
 .../org/apache/celeborn/common/CelebornConf.scala  |  30 +++++
 .../common/protocol/message/ControlMessages.scala  |  22 ++-
 docs/configuration/master.md                       |   3 +
 .../master/clustermeta/AbstractMetaManager.java    |  18 +++
 .../master/clustermeta/IMetadataHandler.java       |   1 +
 .../clustermeta/SingleMasterMetaManager.java       |  13 +-
 .../master/clustermeta/ha/HAMasterMetaManager.java |   4 +
 .../deploy/master/clustermeta/ha/MetaHandler.java  |  29 ++--
 master/src/main/proto/Resource.proto               |   1 +
 .../celeborn/service/deploy/master/Master.scala    |  16 ++-
 .../deploy/master/metrics/AppDiskUsageMetric.scala | 150 +++++++++++++++++++++
 .../clustermeta/DefaultMetaSystemSuiteJ.java       |   9 +-
 .../ha/RatisMasterStatusSystemSuiteJ.java          |   3 +
 .../deploy/master/AppDiskUsageMetricSuite.scala    | 104 ++++++++++++++
 .../celeborn/server/common/HttpService.scala       |   2 +
 .../server/common/http/HttpRequestHandler.scala    |   2 +
 .../celeborn/service/deploy/worker/Worker.scala    |  25 +++-
 .../deploy/worker/storage/StorageManager.scala     |  12 ++
 19 files changed, 413 insertions(+), 34 deletions(-)

diff --git a/common/src/main/proto/TransportMessages.proto b/common/src/main/proto/TransportMessages.proto
index 96ef54d9..a8c4f766 100644
--- a/common/src/main/proto/TransportMessages.proto
+++ b/common/src/main/proto/TransportMessages.proto
@@ -138,9 +138,10 @@ message PbHeartbeatFromWorker {
   int32 fetchPort = 4;
   int32 replicatePort = 5;
   repeated PbDiskInfo disks = 6;
-  repeated string shuffleKeys = 7;
+  repeated string activeShuffleKeys = 7;
   string requestId = 8;
   map<string, PbResourceConsumption> userResourceConsumption = 9;
+  map<string, int64> estimatedAppDiskUsage = 10;
 }
 
 message PbHeartbeatResponse {
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index df0c1933..ffcea627 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -479,6 +479,9 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
     get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INITIAL_DELAY)
   def estimatedPartitionSizeForEstimationUpdateInterval: Long =
     get(SHUFFLE_ESTIMATED_PARTITION_SIZE_UPDATE_INTERVAL)
+  def metricsAppTopDiskUsageCount: Int = get(METRICS_APP_TOP_DISK_USAGE_COUNT)
+  def metricsAppTopDiskUsageWindowSize: Int = get(METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE)
+  def metricsAppTopDiskUsageInterval: Long = get(METRICS_APP_TOP_DISK_USAGE_INTERVAL)
 
   // //////////////////////////////////////////////////////
   //                      Worker                         //
@@ -1957,6 +1960,33 @@ object CelebornConf extends Logging {
       .timeConf(TimeUnit.MILLISECONDS)
       .createWithDefaultString("10min")
 
+  val METRICS_APP_TOP_DISK_USAGE_COUNT: ConfigEntry[Int] =
+    buildConf("celeborn.metrics.app.topDiskUsage.count")
+      .withAlternative("rss.metrics.app.topDiskUsage.count")
+      .categories("master")
+      .doc("Size for top items about top disk usage applications list.")
+      .version("0.2.0")
+      .intConf
+      .createWithDefault(50)
+
+  val METRICS_APP_TOP_DISK_USAGE_WINDOW_SIZE: ConfigEntry[Int] =
+    buildConf("celeborn.metrics.app.topDiskUsage.windowSize")
+      .withAlternative("rss.metrics.app.topDiskUsage.windowSize")
+      .categories("master")
+      .doc("Window size about top disk usage application list.")
+      .version("0.2.0")
+      .intConf
+      .createWithDefault(24)
+
+  val METRICS_APP_TOP_DISK_USAGE_INTERVAL: ConfigEntry[Long] =
+    buildConf("celeborn.metrics.app.topDiskUsage.interval")
+      .withAlternative("rss.metrics.app.topDiskUsage.interval")
+      .categories("master")
+      .doc("Time length for a window about top disk usage application list.")
+      .version("0.2.0")
+      .timeConf(TimeUnit.SECONDS)
+      .createWithDefaultString("10min")
+
   val PUSH_STAGE_END_TIMEOUT: ConfigEntry[Long] =
     buildConf("celeborn.push.stageEnd.timeout")
       .withAlternative("rss.stage.end.timeout")
diff --git a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
index b92a996c..7eff974b 100644
--- a/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/protocol/message/ControlMessages.scala
@@ -109,7 +109,8 @@ object ControlMessages extends Logging {
       replicatePort: Int,
       disks: Seq[DiskInfo],
       userResourceConsumption: util.Map[UserIdentifier, ResourceConsumption],
-      shuffleKeys: util.HashSet[String],
+      activeShuffleKeys: util.Set[String],
+      estimatedAppDiskUsage: util.HashMap[String, java.lang.Long],
       override var requestId: String = ZERO_UUID) extends MasterRequestMessage
 
   case class HeartbeatResponse(
@@ -434,7 +435,8 @@ object ControlMessages extends Logging {
           replicatePort,
           disks,
           userResourceConsumption,
-          shuffleKeys,
+          activeShuffleKeys,
+          estimatedAppDiskUsage,
           requestId) =>
       val pbDisks = disks.map(PbSerDeUtils.toPbDiskInfo).asJava
       val pbUserResourceConsumption =
@@ -447,7 +449,8 @@ object ControlMessages extends Logging {
         .addAllDisks(pbDisks)
         .putAllUserResourceConsumption(pbUserResourceConsumption)
         .setReplicatePort(replicatePort)
-        .addAllShuffleKeys(shuffleKeys)
+        .addAllActiveShuffleKeys(activeShuffleKeys)
+        .putAllEstimatedAppDiskUsage(estimatedAppDiskUsage)
         .setRequestId(requestId)
         .build().toByteArray
       new TransportMessage(MessageType.HEARTBEAT_FROM_WORKER, payload)
@@ -786,12 +789,16 @@ object ControlMessages extends Logging {
 
       case HEARTBEAT_FROM_WORKER =>
         val pbHeartbeatFromWorker = PbHeartbeatFromWorker.parseFrom(message.getPayload)
-        val shuffleKeys = new util.HashSet[String]()
+        val estimatedAppDiskUsage = new util.HashMap[String, java.lang.Long]()
         val userResourceConsumption = PbSerDeUtils.fromPbUserResourceConsumption(
           pbHeartbeatFromWorker.getUserResourceConsumptionMap)
         val pbDisks = pbHeartbeatFromWorker.getDisksList.asScala.map(PbSerDeUtils.fromPbDiskInfo)
-        if (pbHeartbeatFromWorker.getShuffleKeysCount > 0) {
-          shuffleKeys.addAll(pbHeartbeatFromWorker.getShuffleKeysList)
+        if (!pbHeartbeatFromWorker.getEstimatedAppDiskUsageMap.isEmpty) {
+          estimatedAppDiskUsage.putAll(pbHeartbeatFromWorker.getEstimatedAppDiskUsageMap)
+        }
+        val activeShuffleKeys = new util.HashSet[String]()
+        if (!pbHeartbeatFromWorker.getActiveShuffleKeysList.isEmpty) {
+          activeShuffleKeys.addAll(pbHeartbeatFromWorker.getActiveShuffleKeysList)
         }
         HeartbeatFromWorker(
           pbHeartbeatFromWorker.getHost,
@@ -801,7 +808,8 @@ object ControlMessages extends Logging {
           pbHeartbeatFromWorker.getReplicatePort,
           pbDisks,
           userResourceConsumption,
-          shuffleKeys,
+          activeShuffleKeys,
+          estimatedAppDiskUsage,
           pbHeartbeatFromWorker.getRequestId)
 
       case HEARTBEAT_RESPONSE =>
diff --git a/docs/configuration/master.md b/docs/configuration/master.md
index 78ca595b..59adaa0c 100644
--- a/docs/configuration/master.md
+++ b/docs/configuration/master.md
@@ -30,6 +30,9 @@ license: |
 | celeborn.master.metrics.prometheus.host | 0.0.0.0 | Master's Prometheus host. | 0.2.0 | 
 | celeborn.master.metrics.prometheus.port | 9098 | Master's Prometheus port. | 0.2.0 | 
 | celeborn.master.port | 9097 | Port for master to bind. | 0.2.0 | 
+| celeborn.metrics.app.topDiskUsage.count | 50 | Size for top items about top disk usage applications list. | 0.2.0 | 
+| celeborn.metrics.app.topDiskUsage.interval | 10min | Time length for a window about top disk usage application list. | 0.2.0 | 
+| celeborn.metrics.app.topDiskUsage.windowSize | 24 | Window size about top disk usage application list. | 0.2.0 | 
 | celeborn.metrics.capacity | 4096 | The maximum number of metrics which a source can use to generate output strings. | 0.2.0 | 
 | celeborn.metrics.collectPerfCritical.enabled | false | It controls whether to collect metrics which may affect performance. When enable, Celeborn collects them. | 0.2.0 | 
 | celeborn.metrics.enabled | true | When true, enable metrics system. | 0.2.0 | 
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
index 4f612fea..744052d4 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
@@ -27,6 +27,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.concurrent.atomic.LongAdder;
 
 import org.slf4j.Logger;
@@ -40,6 +41,8 @@ import org.apache.celeborn.common.quota.ResourceConsumption;
 import org.apache.celeborn.common.rpc.RpcAddress;
 import org.apache.celeborn.common.rpc.RpcEnv;
 import org.apache.celeborn.common.util.Utils;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageSnapShot;
 
 public abstract class AbstractMetaManager implements IMetadataHandler {
   private static final Logger LOG = LoggerFactory.getLogger(AbstractMetaManager.class);
@@ -61,6 +64,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
   public long estimatedPartitionSize;
   public final LongAdder partitionTotalWritten = new LongAdder();
   public final LongAdder partitionTotalFileCount = new LongAdder();
+  public AppDiskUsageMetric appDiskUsageMetric = null;
 
   public void updateRequestSlotsMeta(
       String shuffleKey, String hostName, Map<String, Map<String, Integer>> workerWithAllocations) {
@@ -165,6 +169,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
       int replicatePort,
       Map<String, DiskInfo> disks,
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+      Map<String, Long> estimatedAppDiskUsage,
       long time) {
     WorkerInfo worker =
         new WorkerInfo(
@@ -188,6 +193,7 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
             info.lastHeartbeat_$eq(time);
           });
     }
+    appDiskUsageMetric.update(estimatedAppDiskUsage);
     if (!blacklist.contains(worker) && disks.isEmpty()) {
       LOG.debug("Worker: {} num total slots is 0, add to blacklist", worker);
       blacklist.add(worker);
@@ -276,6 +282,11 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
           }
         });
 
+    out.writeLong(partitionTotalWritten.sum());
+    out.writeLong(partitionTotalFileCount.sum());
+    out.writeObject(appDiskUsageMetric.snapShots());
+    out.writeObject(appDiskUsageMetric.currentSnapShot());
+
     out.flush();
   }
 
@@ -331,6 +342,13 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
       for (int i = 0; i < workersSize; ++i) {
         workers.add((WorkerInfo) in.readObject());
       }
+      partitionTotalWritten.reset();
+      partitionTotalWritten.add(in.readLong());
+      partitionTotalFileCount.reset();
+      partitionTotalFileCount.add(in.readLong());
+      appDiskUsageMetric.restoreFromSnapshot((AppDiskUsageSnapShot[]) in.readObject());
+      appDiskUsageMetric.currentSnapShot_$eq(
+          (AtomicReference<AppDiskUsageSnapShot>) in.readObject());
     } catch (ClassNotFoundException e) {
       throw new IOException(e);
     }
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
index c09f3e9c..84b65465 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/IMetadataHandler.java
@@ -59,6 +59,7 @@ public interface IMetadataHandler {
       int replicatePort,
       Map<String, DiskInfo> disks,
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+      Map<String, Long> estimatedAppDiskUsage,
       long time,
       String requestId);
 
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
index 497e8183..2e81777f 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/SingleMasterMetaManager.java
@@ -29,6 +29,7 @@ import org.apache.celeborn.common.meta.DiskInfo;
 import org.apache.celeborn.common.meta.WorkerInfo;
 import org.apache.celeborn.common.quota.ResourceConsumption;
 import org.apache.celeborn.common.rpc.RpcEnv;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
 
 public class SingleMasterMetaManager extends AbstractMetaManager {
   private static final Logger LOG = LoggerFactory.getLogger(SingleMasterMetaManager.class);
@@ -38,6 +39,7 @@ public class SingleMasterMetaManager extends AbstractMetaManager {
     this.conf = conf;
     this.initialEstimatedPartitionSize = conf.initialEstimatedPartitionSize();
     this.estimatedPartitionSize = initialEstimatedPartitionSize;
+    this.appDiskUsageMetric = new AppDiskUsageMetric(conf);
   }
 
   @Override
@@ -95,10 +97,19 @@ public class SingleMasterMetaManager extends AbstractMetaManager {
       int replicatePort,
       Map<String, DiskInfo> disks,
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+      Map<String, Long> estimatedAppDiskUsage,
       long time,
       String requestId) {
     updateWorkerHeartbeatMeta(
-        host, rpcPort, pushPort, fetchPort, replicatePort, disks, userResourceConsumption, time);
+        host,
+        rpcPort,
+        pushPort,
+        fetchPort,
+        replicatePort,
+        disks,
+        userResourceConsumption,
+        estimatedAppDiskUsage,
+        time);
   }
 
   @Override
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
index 5af627b1..0476df87 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
@@ -37,6 +37,7 @@ import org.apache.celeborn.service.deploy.master.clustermeta.MetaUtil;
 import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos;
 import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos.ResourceRequest;
 import org.apache.celeborn.service.deploy.master.clustermeta.ResourceProtos.Type;
+import org.apache.celeborn.service.deploy.master.metrics.AppDiskUsageMetric;
 
 public class HAMasterMetaManager extends AbstractMetaManager {
   private static final Logger LOG = LoggerFactory.getLogger(HAMasterMetaManager.class);
@@ -48,6 +49,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
     this.conf = conf;
     this.initialEstimatedPartitionSize = conf.initialEstimatedPartitionSize();
     this.estimatedPartitionSize = initialEstimatedPartitionSize;
+    this.appDiskUsageMetric = new AppDiskUsageMetric(conf);
   }
 
   public HARaftServer getRatisServer() {
@@ -220,6 +222,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
       int replicatePort,
       Map<String, DiskInfo> disks,
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption,
+      Map<String, Long> estimatedAppDiskUsage,
       long time,
       String requestId) {
     try {
@@ -237,6 +240,7 @@ public class HAMasterMetaManager extends AbstractMetaManager {
                       .putAllDisks(MetaUtil.toPbDiskInfos(disks))
                       .putAllUserResourceConsumption(
                           MetaUtil.toPbUserResourceConsumption(userResourceConsumption))
+                      .putAllEstimatedAppDiskUsage(estimatedAppDiskUsage)
                       .setTime(time)
                       .build())
               .build());
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
index 4ae4a3e2..839cefa5 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
@@ -19,10 +19,7 @@ package org.apache.celeborn.service.deploy.master.clustermeta.ha;
 
 import java.io.File;
 import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import org.slf4j.Logger;
@@ -100,10 +97,11 @@ public class MetaHandler {
       int pushPort;
       int fetchPort;
       int replicatePort;
-      Map<String, DiskInfo> disks;
+      Map<String, DiskInfo> diskInfos;
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption;
       List<Map<String, Integer>> slots = new ArrayList<>();
       Map<String, Map<String, Integer>> workerAllocations = new HashMap<>();
+      Map<String, Long> estimatedAppDiskUsage = new HashMap<>();
       switch (cmdType) {
         case RequestSlots:
           shuffleKey = request.getRequestSlotsRequest().getShuffleKey();
@@ -177,10 +175,12 @@ public class MetaHandler {
           rpcPort = request.getWorkerHeartbeatRequest().getRpcPort();
           pushPort = request.getWorkerHeartbeatRequest().getPushPort();
           fetchPort = request.getWorkerHeartbeatRequest().getFetchPort();
-          disks = MetaUtil.fromPbDiskInfos(request.getWorkerHeartbeatRequest().getDisksMap());
+          diskInfos = MetaUtil.fromPbDiskInfos(request.getWorkerHeartbeatRequest().getDisksMap());
           userResourceConsumption =
               MetaUtil.fromPbUserResourceConsumption(
                   request.getWorkerHeartbeatRequest().getUserResourceConsumptionMap());
+          estimatedAppDiskUsage.putAll(
+              request.getWorkerHeartbeatRequest().getEstimatedAppDiskUsageMap());
           replicatePort = request.getWorkerHeartbeatRequest().getReplicatePort();
           LOG.debug(
               "Handle worker heartbeat for {} {} {} {} {} {} {}",
@@ -189,7 +189,7 @@ public class MetaHandler {
               pushPort,
               fetchPort,
               replicatePort,
-              disks,
+              diskInfos,
               userResourceConsumption);
           time = request.getWorkerHeartbeatRequest().getTime();
           metaSystem.updateWorkerHeartbeatMeta(
@@ -198,8 +198,9 @@ public class MetaHandler {
               pushPort,
               fetchPort,
               replicatePort,
-              disks,
+              diskInfos,
               userResourceConsumption,
+              estimatedAppDiskUsage,
               time);
           break;
 
@@ -209,7 +210,7 @@ public class MetaHandler {
           pushPort = request.getRegisterWorkerRequest().getPushPort();
           fetchPort = request.getRegisterWorkerRequest().getFetchPort();
           replicatePort = request.getRegisterWorkerRequest().getReplicatePort();
-          disks = MetaUtil.fromPbDiskInfos(request.getRegisterWorkerRequest().getDisksMap());
+          diskInfos = MetaUtil.fromPbDiskInfos(request.getRegisterWorkerRequest().getDisksMap());
           userResourceConsumption =
               MetaUtil.fromPbUserResourceConsumption(
                   request.getRegisterWorkerRequest().getUserResourceConsumptionMap());
@@ -220,10 +221,16 @@ public class MetaHandler {
               pushPort,
               fetchPort,
               replicatePort,
-              disks,
+              diskInfos,
               userResourceConsumption);
           metaSystem.updateRegisterWorkerMeta(
-              host, rpcPort, pushPort, fetchPort, replicatePort, disks, userResourceConsumption);
+              host,
+              rpcPort,
+              pushPort,
+              fetchPort,
+              replicatePort,
+              diskInfos,
+              userResourceConsumption);
           break;
 
         case ReportWorkerUnavailable:
diff --git a/master/src/main/proto/Resource.proto b/master/src/main/proto/Resource.proto
index 88031761..d0733218 100644
--- a/master/src/main/proto/Resource.proto
+++ b/master/src/main/proto/Resource.proto
@@ -115,6 +115,7 @@ message WorkerHeartbeatRequest {
   map<string, DiskInfo> disks = 6;
   required int64 time = 7;
   map<string, ResourceConsumption> userResourceConsumption = 8;
+  map<string, int64> estimatedAppDiskUsage = 9;
 }
 
 message RegisterWorkerRequest {
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 7de2fe53..a930c98f 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -278,7 +278,8 @@ private[celeborn] class Master(
           replicatePort,
           disks,
           userResourceConsumption,
-          shuffleKeys,
+          activeShuffleKey,
+          estimatedAppDiskUsage,
           requestId) =>
       logDebug(s"Received heartbeat from" +
         s" worker $host:$rpcPort:$pushPort:$fetchPort with $disks.")
@@ -293,7 +294,8 @@ private[celeborn] class Master(
           replicatePort,
           disks,
           userResourceConsumption,
-          shuffleKeys,
+          activeShuffleKey,
+          estimatedAppDiskUsage,
           requestId))
 
     case GetWorkerInfos =>
@@ -356,7 +358,8 @@ private[celeborn] class Master(
       replicatePort: Int,
       disks: Seq[DiskInfo],
       userResourceConsumption: util.Map[UserIdentifier, ResourceConsumption],
-      shuffleKeys: util.HashSet[String],
+      activeShuffleKeys: util.Set[String],
+      estimatedAppDiskUsage: util.HashMap[String, java.lang.Long],
       requestId: String): Unit = {
     val targetWorker = new WorkerInfo(host, rpcPort, pushPort, fetchPort, replicatePort)
     val registered = workersSnapShot.asScala.contains(targetWorker)
@@ -372,12 +375,13 @@ private[celeborn] class Master(
         replicatePort,
         disks.map { disk => disk.mountPoint -> disk }.toMap.asJava,
         userResourceConsumption,
+        estimatedAppDiskUsage,
         System.currentTimeMillis(),
         requestId)
     }
 
     val expiredShuffleKeys = new util.HashSet[String]
-    shuffleKeys.asScala.foreach { shuffleKey =>
+    activeShuffleKeys.asScala.foreach { shuffleKey =>
       if (!statusSystem.registeredShuffle.contains(shuffleKey)) {
         logWarning(s"Shuffle $shuffleKey expired on $host:$rpcPort:$pushPort:$fetchPort.")
         expiredShuffleKeys.add(shuffleKey)
@@ -693,6 +697,10 @@ private[celeborn] class Master(
     statusSystem.registeredShuffle.asScala.mkString("\n")
   }
 
+  override def listTopDiskUseApps: String = {
+    statusSystem.appDiskUsageMetric.summary
+  }
+
   private def requestGetWorkerInfos(endpoint: RpcEndpointRef): GetWorkerInfosResponse = {
     try {
       endpoint.askSync[GetWorkerInfosResponse](GetWorkerInfos)
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala
new file mode 100644
index 00000000..7674f712
--- /dev/null
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/metrics/AppDiskUsageMetric.scala
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.service.deploy.master.metrics
+
+import java.time.LocalDateTime
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
+
+import scala.collection.JavaConverters.mapAsScalaMapConverter
+
+import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.internal.Logging
+import org.apache.celeborn.common.util.{ThreadUtils, Utils}
+
+case class AppDiskUsage(var appId: String, var estimatedUsage: Long) {
+  override def toString: String =
+    s"Application ${appId} used approximate ${Utils.bytesToString(estimatedUsage)} "
+}
+
+class AppDiskUsageSnapShot(val topItemCount: Int) extends Logging with Serializable {
+  val topNItems = new Array[AppDiskUsage](topItemCount)
+  val startSnapShotTime = LocalDateTime.now()
+  var endSnapShotTime: LocalDateTime = _
+
+  def commit(): Unit = {
+    endSnapShotTime = LocalDateTime.now()
+  }
+
+  def updateAppDiskUsage(appId: String, usage: Long): Unit = {
+    val dropIndex = topNItems.indexWhere(usage => usage != null && usage.appId == appId)
+    if (dropIndex != -1) {
+      drop(dropIndex)
+    }
+    val insertIndex = findInsertPosition(usage)
+    if (insertIndex != -1) {
+      shift(insertIndex)
+      topNItems(insertIndex) = AppDiskUsage(appId, usage)
+    }
+  }
+
+  def shift(index: Int): Unit = {
+    for (i <- topItemCount - 1 until index by -1) {
+      topNItems(i) = topNItems(i - 1)
+    }
+  }
+
+  def drop(index: Int): Unit = {
+    for (i <- index until topItemCount - 1) {
+      topNItems(i) = topNItems(i + 1)
+    }
+    topNItems(topItemCount - 1) = null
+  }
+
+  def findInsertPosition(usage: Long): Int = {
+    if (topNItems(0) == null) {
+      return 0
+    }
+    for (i <- 0 until topItemCount) {
+      if (topNItems(i) == null || topNItems(i).estimatedUsage < usage) {
+        return i
+      }
+    }
+    -1
+  }
+
+  override def toString(): String = s"Snapshot " +
+    s"start ${startSnapShotTime} end ${endSnapShotTime}" +
+    s" ${topNItems.filter(_ != null).mkString(",")}"
+}
+
+// This metric collects approximate value because worker won't report all app disk usage value for reducing memory pressure. .
+class AppDiskUsageMetric(conf: CelebornConf) extends Logging {
+  val usageCount = conf.metricsAppTopDiskUsageCount
+  val snapshotCount = conf.metricsAppTopDiskUsageWindowSize
+  val interval = conf.metricsAppTopDiskUsageInterval
+  val snapShots = new Array[AppDiskUsageSnapShot](snapshotCount)
+  val logExecutor =
+    ThreadUtils.newDaemonSingleThreadScheduledExecutor("App_disk_usage_log_thread")
+  val updateExecutor =
+    ThreadUtils.newDaemonSingleThreadExecutor("App_disk_usage_metric_thread")
+  var currentSnapShot: AtomicReference[AppDiskUsageSnapShot] =
+    new AtomicReference[AppDiskUsageSnapShot]()
+
+  def update(appDiskUsage: java.util.Map[String, java.lang.Long]): Unit = {
+    updateExecutor.submit(new Runnable {
+      override def run(): Unit = {
+        if (currentSnapShot.get() != null) {
+          appDiskUsage.asScala.foreach { case (key, usage) =>
+            currentSnapShot.get().updateAppDiskUsage(key, usage)
+          }
+        }
+      }
+    })
+  }
+
+  logExecutor.scheduleAtFixedRate(
+    new Runnable {
+      override def run(): Unit = {
+        if (currentSnapShot.get() != null) {
+          currentSnapShot.get().commit()
+        }
+        currentSnapShot.set(getNewSnapShot())
+        logInfo(s"App Disk Usage Top${usageCount} Report ${summary()}")
+      }
+    },
+    60,
+    interval,
+    TimeUnit.SECONDS)
+
+  def getNewSnapShot(): AppDiskUsageSnapShot = {
+    for (i <- snapshotCount - 1 until 0 by -1) {
+      snapShots(i) = snapShots(i - 1)
+    }
+    snapShots(0) = new AppDiskUsageSnapShot(usageCount)
+    snapShots(0)
+  }
+
+  def summary(): String = {
+    val stringBuilder = new StringBuilder()
+    for (i <- 0 until snapshotCount) {
+      if (snapShots(i) != null) {
+        stringBuilder.append(snapShots(i))
+        stringBuilder.append("    \n")
+      }
+    }
+    stringBuilder.toString()
+  }
+
+  def restoreFromSnapshot(array: Array[AppDiskUsageSnapShot]): Unit = {
+    for (i <- 0 until (snapshotCount)) {
+      snapShots(i) = array(i)
+    }
+  }
+
+}
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
index c37ce34f..6a4b8193 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
@@ -21,11 +21,7 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 
 import org.junit.After;
@@ -518,6 +514,7 @@ public class DefaultMetaSystemSuiteJ {
         REPLICATEPORT1,
         new HashMap<>(),
         userResourceConsumption1,
+        new HashMap<>(),
         1,
         getNewReqeustId());
 
@@ -531,6 +528,7 @@ public class DefaultMetaSystemSuiteJ {
         REPLICATEPORT2,
         new HashMap<>(),
         userResourceConsumption2,
+        new HashMap<>(),
         1,
         getNewReqeustId());
 
@@ -544,6 +542,7 @@ public class DefaultMetaSystemSuiteJ {
         REPLICATEPORT3,
         disks1,
         userResourceConsumption1,
+        new HashMap<>(),
         1,
         getNewReqeustId());
 
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index c55f0467..7414cdd7 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -695,6 +695,7 @@ public class RatisMasterStatusSystemSuiteJ {
         REPLICATEPORT1,
         new HashMap<>(),
         userResourceConsumption1,
+        new HashMap<>(),
         1,
         getNewReqeustId());
     Thread.sleep(3000L);
@@ -711,6 +712,7 @@ public class RatisMasterStatusSystemSuiteJ {
         REPLICATEPORT2,
         new HashMap<>(),
         userResourceConsumption2,
+        new HashMap<>(),
         1,
         getNewReqeustId());
     Thread.sleep(3000L);
@@ -728,6 +730,7 @@ public class RatisMasterStatusSystemSuiteJ {
         REPLICATEPORT1,
         disks1,
         userResourceConsumption1,
+        new HashMap<>(),
         1,
         getNewReqeustId());
     Thread.sleep(3000L);
diff --git a/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
new file mode 100644
index 00000000..91dec16c
--- /dev/null
+++ b/master/src/test/scala/org/apache/celeborn/service/deploy/master/AppDiskUsageMetricSuite.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.celeborn.service.deploy.master
+
+import java.util
+
+import scala.util.Random
+
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach}
+import org.scalatest.funsuite.AnyFunSuite
+
+import org.apache.celeborn.common.CelebornConf
+import org.apache.celeborn.common.internal.Logging
+import org.apache.celeborn.common.meta.WorkerInfo
+import org.apache.celeborn.service.deploy.master.metrics.{AppDiskUsageMetric, AppDiskUsageSnapShot}
+
+class AppDiskUsageMetricSuite extends AnyFunSuite
+  with BeforeAndAfterAll
+  with BeforeAndAfterEach
+  with Logging {
+  val WORKER1 = new WorkerInfo("host1", 111, 112, 113, 114)
+  val WORKER2 = new WorkerInfo("host2", 211, 212, 213, 214)
+  val WORKER3 = new WorkerInfo("host3", 311, 312, 313, 314)
+
+  test("test snapshot ordering") {
+    val snapShot = new AppDiskUsageSnapShot(50)
+    val rand = new Random()
+    for (i <- 1 to 60) {
+      snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1)
+    }
+    println(snapShot.toString)
+  }
+
+  test("test snapshot ordering with duplicate entries") {
+    val snapShot = new AppDiskUsageSnapShot(50)
+    val rand = new Random()
+    for (i <- 1 to 60) {
+      snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1)
+    }
+    for (i <- 1 to 15) {
+      snapShot.updateAppDiskUsage(s"app-${i}", rand.nextInt(100000000) + 1000000000)
+    }
+
+    println(snapShot.toString)
+  }
+
+  test("test app usage snapshot") {
+    Thread.sleep(5000)
+
+    val conf = new CelebornConf()
+    conf.set("rss.metrics.app.topDiskUsage.windowSize", "5")
+    conf.set("rss.metrids.app.topDiskUsage.interval", "2s")
+    val usageMetric = new AppDiskUsageMetric(conf)
+
+    val map1 = new util.HashMap[String, java.lang.Long]()
+    map1.put("app1", 2874371)
+    map1.put("app2", 43452)
+    map1.put("app3", 2134526)
+    map1.put("app4", 23465463)
+    map1.put("app5", 132456)
+    map1.put("app6", 6535635)
+    usageMetric.update(map1)
+    println(usageMetric.summary())
+    Thread.sleep(2000)
+
+    map1.clear()
+    map1.put("app1", 374524)
+    map1.put("app2", 5234665)
+    map1.put("app3", 24453)
+    map1.put("app4", 2345637)
+    map1.put("app5", 4534)
+    map1.put("app6", 5357)
+    usageMetric.update(map1)
+    println(usageMetric.summary())
+    Thread.sleep(2000)
+
+    map1.clear()
+    map1.put("app1", 12343)
+    map1.put("app2", 3456565)
+    map1.put("app3", 4345)
+    map1.put("app4", 35245268)
+    map1.put("app5", 45367)
+    map1.put("app6", 64345)
+    usageMetric.update(map1)
+    println(usageMetric.summary())
+    Thread.sleep(2500)
+    println(usageMetric.summary())
+  }
+}
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
index b0e6fe05..fece4879 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
@@ -35,6 +35,8 @@ abstract class HttpService extends Service with Logging {
 
   def getShuffleList: String
 
+  def listTopDiskUseApps: String
+
   def startHttpServer(): Unit = {
     val handlers =
       if (metricsSystem.running) {
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
index 1a480a14..aa6355e2 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
@@ -72,6 +72,8 @@ class HttpRequestHandler(
         service.getApplicationList
       case "/shuffles" =>
         service.getShuffleList
+      case "/listTopDiskUsedApps" =>
+        service.listTopDiskUseApps
       case _ => INVALID
     }
   }
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index d324ebfc..a0067a3a 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -18,11 +18,13 @@
 package org.apache.celeborn.service.deploy.worker
 
 import java.io.IOException
-import java.util.{HashSet => JHashSet}
+import java.lang.{Long => JLong}
+import java.util.{HashMap => JHashMap, HashSet => JHashSet}
 import java.util.concurrent._
 import java.util.concurrent.atomic.AtomicBoolean
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 import com.google.common.annotations.VisibleForTesting
 import io.netty.util.HashedWheelTimer
@@ -219,9 +221,13 @@ private[celeborn] class Worker(
     _ => memoryTracker.getPausePushDataAndReplicateCounter)
 
   private def heartBeatToMaster(): Unit = {
-    val shuffleKeys = new JHashSet[String]
-    shuffleKeys.addAll(partitionLocationInfo.shuffleKeySet)
-    shuffleKeys.addAll(storageManager.shuffleKeySet())
+    val activeShuffleKeys = new JHashSet[String]()
+    val estimatedAppDiskUsage = new JHashMap[String, JLong]()
+    activeShuffleKeys.addAll(partitionLocationInfo.shuffleKeySet)
+    activeShuffleKeys.addAll(storageManager.shuffleKeySet())
+    storageManager.topAppDiskUsage.asScala.foreach { case (shuffleId, usage) =>
+      estimatedAppDiskUsage.put(shuffleId, usage)
+    }
     // During shutdown, return an empty diskInfo list to mark this worker as unavailable,
     // and avoid remove this from master's blacklist.
     val diskInfos =
@@ -245,7 +251,8 @@ private[celeborn] class Worker(
         replicatePort,
         diskInfos,
         resourceConsumption,
-        shuffleKeys),
+        activeShuffleKeys,
+        estimatedAppDiskUsage),
       classOf[HeartbeatResponse])
     if (response.registered) {
       response.expiredShuffleKeys.asScala.foreach(shuffleKey => workerInfo.releaseSlots(shuffleKey))
@@ -432,6 +439,14 @@ private[celeborn] class Worker(
     storageManager.shuffleKeySet().asScala.mkString("\n")
   }
 
+  override def listTopDiskUseApps: String = {
+    val stringBuilder = new StringBuilder()
+    storageManager.topAppDiskUsage.asScala.foreach { case (appId, usage) =>
+      stringBuilder.append(s"application ${appId} used ${Utils.bytesToString(usage)} ")
+    }
+    stringBuilder.toString()
+  }
+
   @VisibleForTesting
   def isRegistered(): Boolean = {
     registered.get()
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index f7afd5c6..266ad164 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -371,6 +371,18 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
     hashSet
   }
 
+  def topAppDiskUsage: util.Map[String, Long] = {
+    fileInfos.asScala.map { keyedWriters =>
+      {
+        keyedWriters._1 -> keyedWriters._2.values().asScala.map(_.getFileLength).sum
+      }
+    }.toList.map { case (shuffleKey, usage) =>
+      shuffleKey.split("-")(0) -> usage
+    }.groupBy(_._1).map { case (key, values) =>
+      key -> values.map(_._2).sum
+    }.toSeq.sortBy(_._2).reverse.take(conf.metricsAppTopDiskUsageCount * 2).toMap.asJava
+  }
+
   def cleanupExpiredShuffleKey(expiredShuffleKeys: util.HashSet[String]): Unit = {
     expiredShuffleKeys.asScala.foreach { shuffleKey =>
       logInfo(s"Cleanup expired shuffle $shuffleKey.")