You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/07/14 12:41:16 UTC

[incubator-celeborn] branch branch-0.3 updated: [CELEBORN-791] Remove slots allocation simulation in master and use active slots sent from worker's heartbeat

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/branch-0.3 by this push:
     new ae9380a9e [CELEBORN-791] Remove slots allocation simulation in master and use active slots sent from worker's heartbeat
ae9380a9e is described below

commit ae9380a9e8c1fb6698e3c5d49f2bce761f33f6fe
Author: mingji <fe...@alibaba-inc.com>
AuthorDate: Fri Jul 14 20:40:55 2023 +0800

    [CELEBORN-791] Remove slots allocation simulation in master and use active slots sent from worker's heartbeat
    
    ### What changes were proposed in this pull request?
    Master won't simulate slots allocations and use active slots sent from worker.
    
    ### Why are the changes needed?
    I have observed that a new worker might allocate more slots than other workers when using the round-robin slot allocation algorithm.
    There is a logic error in processing heartbeat from worker. It will update disk info's active slots to max(current disk info active slots, disk info sent from worker active slots). If I registered a huge shuffle, master will allocate more slots than a disk's max slots and mark them as unknown disk slots but worker will count the unknown disk slots as active slots and report it to the master. Then the slots release logic can not distinguish unknown slots from a number so the release wi [...]
    Due to the gap between work and master, so I think it's OK to remove slots allocation simulation from worker and use active slots from worker.
    
    Before this patch:
    <img width="928" alt="截屏2023-07-12 16 51 15" src="https://github.com/apache/incubator-celeborn/assets/4150993/9c8a46d9-26a8-42f5-a956-938273277c9b">
    
    After this patch:
    <img width="509" alt="截屏2023-07-12 16 25 52" src="https://github.com/apache/incubator-celeborn/assets/4150993/c49b3d91-14ea-4eb8-9b71-9aab73541faf">
    
    ### Does this PR introduce _any_ user-facing change?
    NO.
    
    ### How was this patch tested?
    UT and cluster.
    
    Closes #1710 from FMX/CELEBORN-791.
    
    Authored-by: mingji <fe...@alibaba-inc.com>
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
    (cherry picked from commit a4687716d2f1fcc4bf487ced70c0fb6b5de037f0)
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
 .../apache/celeborn/common/meta/WorkerInfo.scala   | 28 +++++++---------------
 .../master/clustermeta/AbstractMetaManager.java    | 13 ----------
 .../master/clustermeta/ha/HAMasterMetaManager.java |  7 ------
 .../deploy/master/clustermeta/ha/MetaHandler.java  | 16 +------------
 master/src/main/proto/Resource.proto               |  4 ++++
 .../celeborn/service/deploy/master/Master.scala    |  4 +---
 .../deploy/master/SlotsAllocatorSuiteJ.java        | 12 ++++++----
 .../clustermeta/DefaultMetaSystemSuiteJ.java       |  6 ++---
 .../ha/RatisMasterStatusSystemSuiteJ.java          | 16 ++++++++-----
 9 files changed, 36 insertions(+), 70 deletions(-)

diff --git a/common/src/main/scala/org/apache/celeborn/common/meta/WorkerInfo.scala b/common/src/main/scala/org/apache/celeborn/common/meta/WorkerInfo.scala
index 92f6c91ed..be161e004 100644
--- a/common/src/main/scala/org/apache/celeborn/common/meta/WorkerInfo.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/meta/WorkerInfo.scala
@@ -37,7 +37,6 @@ class WorkerInfo(
     _diskInfos: util.Map[String, DiskInfo],
     _userResourceConsumption: util.Map[UserIdentifier, ResourceConsumption]) extends Serializable
   with Logging {
-  var unknownDiskSlots = new java.util.HashMap[String, Integer]()
   var networkLocation = "/default-rack"
   var lastHeartbeat: Long = 0
   val diskInfos =
@@ -72,8 +71,8 @@ class WorkerInfo(
   }
 
   def usedSlots(): Long = this.synchronized {
-    diskInfos.asScala.map(_._2.activeSlots).sum +
-      unknownDiskSlots.values().asScala.map(_.intValue()).sum
+    // HDFS or OSS do not treat as a used slot
+    diskInfos.asScala.map(_._2.activeSlots).sum
   }
 
   def allocateSlots(shuffleKey: String, slotsPerDisk: util.Map[String, Integer]): Unit =
@@ -83,11 +82,6 @@ class WorkerInfo(
       slotsPerDisk.asScala.foreach { case (disk, slots) =>
         if (!diskInfos.containsKey(disk)) {
           logDebug(s"Unknown disk $disk")
-          if (unknownDiskSlots.containsKey(shuffleKey)) {
-            unknownDiskSlots.put(shuffleKey, slots + unknownDiskSlots.get(shuffleKey))
-          } else {
-            unknownDiskSlots.put(shuffleKey, slots)
-          }
         } else {
           diskInfos.get(disk).allocateSlots(shuffleKey, slots)
         }
@@ -107,17 +101,12 @@ class WorkerInfo(
     slots.asScala.foreach { case (disk, slot) =>
       if (diskInfos.containsKey(disk)) {
         diskInfos.get(disk).releaseSlots(shuffleKey, slot)
-      } else {
-        if (unknownDiskSlots.containsKey(shuffleKey)) {
-          unknownDiskSlots.put(shuffleKey, unknownDiskSlots.get(shuffleKey) - slot)
-        }
       }
     }
   }
 
   def releaseSlots(shuffleKey: String): Unit = this.synchronized {
     diskInfos.asScala.foreach(_._2.releaseSlots(shuffleKey))
-    unknownDiskSlots.remove(shuffleKey)
   }
 
   def getShuffleKeySet: util.HashSet[String] = this.synchronized {
@@ -185,17 +174,18 @@ class WorkerInfo(
       val mountPoint: String = newDisk.mountPoint
       val curDisk = diskInfos.get(mountPoint)
       if (curDisk != null) {
-        curDisk.actualUsableSpace_$eq(newDisk.actualUsableSpace)
-        curDisk.activeSlots_$eq(Math.max(curDisk.activeSlots, newDisk.activeSlots))
-        curDisk.avgFlushTime_$eq(newDisk.avgFlushTime)
-        curDisk.avgFetchTime_$eq(newDisk.avgFetchTime)
+        curDisk.actualUsableSpace = newDisk.actualUsableSpace
+        // Update master's diskinfo activeslots to worker's value
+        curDisk.activeSlots = newDisk.activeSlots
+        curDisk.avgFlushTime = newDisk.avgFlushTime
+        curDisk.avgFetchTime = newDisk.avgFetchTime
         if (estimatedPartitionSize.nonEmpty) {
-          curDisk.maxSlots_$eq(curDisk.actualUsableSpace / estimatedPartitionSize.get)
+          curDisk.maxSlots = curDisk.actualUsableSpace / estimatedPartitionSize.get
         }
         curDisk.setStatus(newDisk.status)
       } else {
         if (estimatedPartitionSize.nonEmpty) {
-          newDisk.maxSlots_$eq(newDisk.actualUsableSpace / estimatedPartitionSize.get)
+          newDisk.maxSlots = newDisk.actualUsableSpace / estimatedPartitionSize.get
         }
         diskInfos.put(mountPoint, newDisk)
       }
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
index d0ef66612..60e75f8c3 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/AbstractMetaManager.java
@@ -93,16 +93,6 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
     if (hostName != null) {
       hostnameSet.add(hostName);
     }
-    if (!workerWithAllocations.isEmpty()) {
-      synchronized (workers) {
-        for (WorkerInfo workerInfo : workers) {
-          String workerUniqueId = workerInfo.toUniqueId();
-          if (workerWithAllocations.containsKey(workerUniqueId)) {
-            workerInfo.allocateSlots(shuffleKey, workerWithAllocations.get(workerUniqueId));
-          }
-        }
-      }
-    }
   }
 
   public void updateReleaseSlotsMeta(String shuffleKey) {
@@ -139,9 +129,6 @@ public abstract class AbstractMetaManager implements IMetadataHandler {
   }
 
   public void updateAppLostMeta(String appId) {
-    registeredShuffle.stream()
-        .filter(shuffle -> shuffle.startsWith(appId))
-        .forEach(this::updateReleaseSlotsMeta);
     registeredShuffle.removeIf(shuffleKey -> shuffleKey.startsWith(appId));
     appHeartbeatTime.remove(appId);
   }
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
index 2a0cecf61..352259a2b 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/HAMasterMetaManager.java
@@ -73,13 +73,6 @@ public class HAMasterMetaManager extends AbstractMetaManager {
           ResourceProtos.RequestSlotsRequest.newBuilder()
               .setShuffleKey(shuffleKey)
               .setHostName(hostName);
-      for (String workerUniqueId : workerToAllocatedSlots.keySet()) {
-        builder.putWorkerAllocations(
-            workerUniqueId,
-            ResourceProtos.SlotInfo.newBuilder()
-                .putAllSlot(workerToAllocatedSlots.get(workerUniqueId))
-                .build());
-      }
       ratisServer.submitRequest(
           ResourceRequest.newBuilder()
               .setCmdType(Type.RequestSlots)
diff --git a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
index 2ac57523d..d6ab8309c 100644
--- a/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
+++ b/master/src/main/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/MetaHandler.java
@@ -98,30 +98,16 @@ public class MetaHandler {
       Map<String, DiskInfo> diskInfos;
       Map<UserIdentifier, ResourceConsumption> userResourceConsumption;
       List<Map<String, Integer>> slots = new ArrayList<>();
-      Map<String, Map<String, Integer>> workerAllocations = new HashMap<>();
       Map<String, Long> estimatedAppDiskUsage = new HashMap<>();
       switch (cmdType) {
         case RequestSlots:
           shuffleKey = request.getRequestSlotsRequest().getShuffleKey();
-          request
-              .getRequestSlotsRequest()
-              .getWorkerAllocationsMap()
-              .forEach((k, v) -> workerAllocations.put(k, new HashMap<>(v.getSlotMap())));
           LOG.debug("Handle request slots for {}", shuffleKey);
           metaSystem.updateRequestSlotsMeta(
-              shuffleKey, request.getRequestSlotsRequest().getHostName(), workerAllocations);
+              shuffleKey, request.getRequestSlotsRequest().getHostName(), new HashMap<>());
           break;
 
         case ReleaseSlots:
-          for (ResourceProtos.SlotInfo pbSlotInfo :
-              request.getReleaseSlotsRequest().getSlotsList()) {
-            slots.add(pbSlotInfo.getSlotMap());
-          }
-
-          shuffleKey = request.getReleaseSlotsRequest().getShuffleKey();
-          LOG.debug("Handle release slots for {}", shuffleKey);
-          metaSystem.updateReleaseSlotsMeta(
-              shuffleKey, request.getReleaseSlotsRequest().getWorkerIdsList(), slots);
           break;
 
         case UnRegisterShuffle:
diff --git a/master/src/main/proto/Resource.proto b/master/src/main/proto/Resource.proto
index b639fc98d..8f6f62f25 100644
--- a/master/src/main/proto/Resource.proto
+++ b/master/src/main/proto/Resource.proto
@@ -23,7 +23,10 @@ package deploy.master.meta;
 enum Type {
   UnRegisterShuffle = 12;
   RequestSlots = 13;
+
+  //deprecated
   ReleaseSlots = 14;
+
   AppHeartbeat = 15;
   AppLost = 16;
   WorkerLost = 17;
@@ -40,6 +43,7 @@ message ResourceRequest {
   optional string requestId = 2 [default=""];
 
   optional RequestSlotsRequest requestSlotsRequest = 10;
+  // deprecated
   optional ReleaseSlotsRequest releaseSlotsRequest = 11;
   optional UnregisterShuffleRequest unregisterShuffleRequest = 12;
   optional AppHeartbeatRequest appHeartbeatRequest = 13;
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index ca5016d54..0b3f99e58 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -637,9 +637,7 @@ private[celeborn] class Master(
       workerIds: util.List[String],
       slots: util.List[util.Map[String, Integer]],
       requestId: String): Unit = {
-    val shuffleKey = Utils.makeShuffleKey(applicationId, shuffleId)
-    statusSystem.handleReleaseSlots(shuffleKey, workerIds, slots, requestId)
-    logInfo(s"Release all slots of $shuffleKey")
+    // For compatibility, ignore this message
     context.reply(ReleaseSlotsResponse(StatusCode.SUCCESS))
   }
 
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/SlotsAllocatorSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/SlotsAllocatorSuiteJ.java
index 48ed49752..ccdeb5b2d 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/SlotsAllocatorSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/SlotsAllocatorSuiteJ.java
@@ -261,19 +261,23 @@ public class SlotsAllocatorSuiteJ {
       }
       Map<WorkerInfo, Map<String, Integer>> workerToAllocatedSlots =
           SlotsAllocator.slotsToDiskAllocations(slots);
+      int unknownDiskSlots = 0;
       for (Map.Entry<WorkerInfo, Map<String, Integer>> entry : workerToAllocatedSlots.entrySet()) {
         WorkerInfo worker = entry.getKey();
         Map<String, Integer> allocationMap = entry.getValue();
         worker.allocateSlots(shuffleKey, allocationMap);
+        if (allocationMap.containsKey("UNKNOWN_DISK")) {
+          unknownDiskSlots += allocationMap.get("UNKNOWN_DISK");
+        }
       }
-      int usedTotalSlots = 0;
+      int allocateToDiskSlots = 0;
       for (WorkerInfo worker : workers) {
-        usedTotalSlots += worker.usedSlots();
+        allocateToDiskSlots += worker.usedSlots();
       }
       if (shouldReplicate) {
-        Assert.assertEquals(partitionIds.size() * 2, usedTotalSlots);
+        Assert.assertEquals(partitionIds.size() * 2, unknownDiskSlots + allocateToDiskSlots);
       } else {
-        Assert.assertEquals(partitionIds.size(), usedTotalSlots);
+        Assert.assertEquals(partitionIds.size(), unknownDiskSlots + allocateToDiskSlots);
       }
     } else {
       assert slots.isEmpty()
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
index 6d0ce6e21..665af980d 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/DefaultMetaSystemSuiteJ.java
@@ -247,8 +247,8 @@ public class DefaultMetaSystemSuiteJ {
 
     statusSystem.handleRequestSlots(SHUFFLEKEY1, HOSTNAME1, workersToAllocate, getNewReqeustId());
 
-    assert (workerInfo1.usedSlots() == 5);
-    assert (workerInfo2.usedSlots() == 5);
+    assert (workerInfo1.usedSlots() == 0);
+    assert (workerInfo2.usedSlots() == 0);
     assert (workerInfo3.usedSlots() == 0);
   }
 
@@ -319,7 +319,7 @@ public class DefaultMetaSystemSuiteJ {
     statusSystem.handleReleaseSlots(SHUFFLEKEY1, workerIds, workerSlots, getNewReqeustId());
 
     Assert.assertEquals(
-        2,
+        0,
         statusSystem.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME1))
             .findFirst()
diff --git a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
index 9ff0d3563..3684e095b 100644
--- a/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
+++ b/master/src/test/java/org/apache/celeborn/service/deploy/master/clustermeta/ha/RatisMasterStatusSystemSuiteJ.java
@@ -398,22 +398,24 @@ public class RatisMasterStatusSystemSuiteJ {
 
     statusSystem.handleRequestSlots(SHUFFLEKEY1, HOSTNAME1, workersToAllocate, getNewReqeustId());
 
+    // Do not update diskinfo's activeslots
+
     Assert.assertEquals(
-        15,
+        0,
         statusSystem.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME1))
             .findFirst()
             .get()
             .usedSlots());
     Assert.assertEquals(
-        25,
+        0,
         statusSystem.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME2))
             .findFirst()
             .get()
             .usedSlots());
     Assert.assertEquals(
-        35,
+        0,
         statusSystem.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME3))
             .findFirst()
@@ -495,22 +497,24 @@ public class RatisMasterStatusSystemSuiteJ {
     statusSystem.handleReleaseSlots(SHUFFLEKEY1, workerIds, workerSlots, getNewReqeustId());
     Thread.sleep(3000L);
 
+    // Do not update diskinfo's activeslots
+
     Assert.assertEquals(
-        2,
+        0,
         STATUSSYSTEM1.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME1))
             .findFirst()
             .get()
             .usedSlots());
     Assert.assertEquals(
-        2,
+        0,
         STATUSSYSTEM2.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME1))
             .findFirst()
             .get()
             .usedSlots());
     Assert.assertEquals(
-        2,
+        0,
         STATUSSYSTEM3.workers.stream()
             .filter(w -> w.host().equals(HOSTNAME1))
             .findFirst()