You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2024/07/18 01:36:19 UTC

(celeborn) branch branch-0.5 updated: [CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure

This is an automated email from the ASF dual-hosted git repository.

ethanfeng pushed a commit to branch branch-0.5
in repository https://gitbox.apache.org/repos/asf/celeborn.git


The following commit(s) were added to refs/heads/branch-0.5 by this push:
     new 8c54b3968 [CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure
8c54b3968 is described below

commit 8c54b3968426d6f3668f320062b35e4283816b72
Author: wangshengjie3 <so...@gmail.com>
AuthorDate: Thu Jul 18 09:35:53 2024 +0800

    [CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure
    
    ### What changes were proposed in this pull request?
    Fix revive logic bug to avoid data correctness issue
    
    ### Why are the changes needed?
    Current logic will revive same partition-epoch multi times
    
    ### Does this PR introduce _any_ user-facing change?
    no
    
    ### How was this patch tested?
    UT
    
    Closes #2532 from wangshengjie123/fix-revive.
    
    Lead-authored-by: wangshengjie3 <so...@gmail.com>
    Co-authored-by: wangshengjie <wa...@xiaomi.com>
    Co-authored-by: CodingCat <zh...@gmail.com>
    Signed-off-by: mingji <fe...@alibaba-inc.com>
    (cherry picked from commit 32509d596258c6f3ffcaedb005d4b42cb1b60937)
    Signed-off-by: mingji <fe...@alibaba-inc.com>
---
 .../celeborn/client/ChangePartitionManager.scala   | 79 ++++++++++++----------
 .../org/apache/celeborn/common/CelebornConf.scala  | 10 +++
 docs/configuration/client.md                       |  1 +
 3 files changed, 55 insertions(+), 35 deletions(-)

diff --git a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
index 6909d5262..d8cfa655d 100644
--- a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
@@ -47,8 +47,14 @@ class ChangePartitionManager(
   // shuffleId -> (partitionId -> set of ChangePartition)
   private val changePartitionRequests =
     JavaUtils.newConcurrentHashMap[Int, ConcurrentHashMap[Integer, JSet[ChangePartitionRequest]]]()
+
+  // shuffleId -> locks
+  private val locks = JavaUtils.newConcurrentHashMap[Int, Array[AnyRef]]()
+  private val lockBucketSize = conf.batchHandleChangePartitionBuckets
+
   // shuffleId -> set of partition id
-  private val inBatchPartitions = JavaUtils.newConcurrentHashMap[Int, JSet[Integer]]()
+  private val inBatchPartitions =
+    JavaUtils.newConcurrentHashMap[Int, ConcurrentHashMap.KeySetView[Int, java.lang.Boolean]]()
 
   private val batchHandleChangePartitionEnabled = conf.batchHandleChangePartitionEnabled
   private val batchHandleChangePartitionExecutors = ThreadUtils.newDaemonCachedThreadPool(
@@ -79,14 +85,20 @@ class ChangePartitionManager(
                 batchHandleChangePartitionExecutors.submit {
                   new Runnable {
                     override def run(): Unit = {
-                      val distinctPartitions = requests.synchronized {
-                        // For each partition only need handle one request
-                        requests.asScala.filter { case (partitionId, _) =>
-                          !inBatchPartitions.get(shuffleId).contains(partitionId)
-                        }.map { case (partitionId, request) =>
-                          inBatchPartitions.get(shuffleId).add(partitionId)
-                          request.asScala.toArray.maxBy(_.epoch)
-                        }.toArray
+                      val distinctPartitions = {
+                        val requestSet = inBatchPartitions.get(shuffleId)
+                        val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+                        requests.asScala.map { case (partitionId, request) =>
+                          locksForShuffle(partitionId % locksForShuffle.length).synchronized {
+                            if (!requestSet.contains(partitionId) && requests.containsKey(
+                                partitionId)) {
+                              requestSet.add(partitionId)
+                              Some(request.asScala.toArray.maxBy(_.epoch))
+                            } else {
+                              None
+                            }
+                          }
+                        }.filter(_.isDefined).map(_.get).toArray
                       }
                       if (distinctPartitions.nonEmpty) {
                         handleRequestPartitions(
@@ -123,8 +135,16 @@ class ChangePartitionManager(
         JavaUtils.newConcurrentHashMap()
     }
 
-  private val inBatchShuffleIdRegisterFunc = new util.function.Function[Int, util.Set[Integer]]() {
-    override def apply(s: Int): util.Set[Integer] = new util.HashSet[Integer]()
+  private val inBatchShuffleIdRegisterFunc =
+    new util.function.Function[Int, ConcurrentHashMap.KeySetView[Int, java.lang.Boolean]]() {
+      override def apply(s: Int): ConcurrentHashMap.KeySetView[Int, java.lang.Boolean] =
+        ConcurrentHashMap.newKeySet[Int]()
+    }
+
+  private val locksRegisterFunc = new util.function.Function[Int, Array[AnyRef]] {
+    override def apply(t: Int): Array[AnyRef] = {
+      Array.fill(lockBucketSize)(new AnyRef())
+    }
   }
 
   def handleRequestPartitionLocation(
@@ -151,36 +171,22 @@ class ChangePartitionManager(
       oldPartition,
       cause)
 
-    // If new slot for the partition has been allocated, reply and return.
-    // Else register and allocate for it.
-    getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { latestLoc =>
-      context.reply(
-        partitionId,
-        StatusCode.SUCCESS,
-        Some(latestLoc),
-        lifecycleManager.workerStatusTracker.workerAvailable(oldPartition))
-      logDebug(s"New partition found, old partition $partitionId-$oldEpoch return it." +
-        s" shuffleId: $shuffleId $latestLoc")
-      return
-    }
-
-    requests.synchronized {
+    val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+    locksForShuffle(partitionId % locksForShuffle.length).synchronized {
       if (requests.containsKey(partitionId)) {
+        logDebug(s"[handleRequestPartitionLocation] For shuffle: $shuffleId, request for same " +
+          s"partition: $partitionId-$oldEpoch exists, register context.")
         requests.get(partitionId).add(changePartition)
-        logTrace(s"[handleRequestPartitionLocation] For $shuffleId, request for same partition" +
-          s"$partitionId-$oldEpoch exists, register context.")
         return
       } else {
-        // If new slot for the partition has been allocated, reply and return.
-        // Else register and allocate for it.
         getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { latestLoc =>
           context.reply(
             partitionId,
             StatusCode.SUCCESS,
             Some(latestLoc),
             lifecycleManager.workerStatusTracker.workerAvailable(oldPartition))
-          logDebug(s"New partition found, old partition $partitionId-$oldEpoch return it." +
-            s" shuffleId: $shuffleId $latestLoc")
+          logDebug(s"[handleRequestPartitionLocation]: For shuffle: $shuffleId," +
+            s" old partition: $partitionId-$oldEpoch, new partition: $latestLoc found, return it")
           return
         }
         val set = new util.HashSet[ChangePartitionRequest]()
@@ -229,8 +235,9 @@ class ChangePartitionManager(
 
     // remove together to reduce lock time
     def replySuccess(locations: Array[PartitionLocation]): Unit = {
-      requestsMap.synchronized {
-        locations.map { location =>
+      val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+      locations.map { location =>
+        locksForShuffle(location.getId % locksForShuffle.length).synchronized {
           if (batchHandleChangePartitionEnabled) {
             inBatchPartitions.get(shuffleId).remove(location.getId)
           }
@@ -250,8 +257,9 @@ class ChangePartitionManager(
 
     // remove together to reduce lock time
     def replyFailure(status: StatusCode): Unit = {
-      requestsMap.synchronized {
-        changePartitions.map { changePartition =>
+      changePartitions.map { changePartition =>
+        val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+        locksForShuffle(changePartition.partitionId % locksForShuffle.length).synchronized {
           if (batchHandleChangePartitionEnabled) {
             inBatchPartitions.get(shuffleId).remove(changePartition.partitionId)
           }
@@ -338,5 +346,6 @@ class ChangePartitionManager(
   def removeExpiredShuffle(shuffleId: Int): Unit = {
     changePartitionRequests.remove(shuffleId)
     inBatchPartitions.remove(shuffleId)
+    locks.remove(shuffleId)
   }
 }
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index a303833ce..4085bc7f8 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -1025,6 +1025,8 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
     PartitionSplitMode.valueOf(get(SHUFFLE_PARTITION_SPLIT_MODE))
   def shufflePartitionSplitThreshold: Long = get(SHUFFLE_PARTITION_SPLIT_THRESHOLD)
   def batchHandleChangePartitionEnabled: Boolean = get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_ENABLED)
+  def batchHandleChangePartitionBuckets: Int =
+    get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_BUCKETS)
   def batchHandleChangePartitionNumThreads: Int = get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_THREADS)
   def batchHandleChangePartitionRequestInterval: Long =
     get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_INTERVAL)
@@ -4127,6 +4129,14 @@ object CelebornConf extends Logging {
       .booleanConf
       .createWithDefault(true)
 
+  val CLIENT_BATCH_HANDLE_CHANGE_PARTITION_BUCKETS: ConfigEntry[Int] =
+    buildConf("celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets")
+      .categories("client")
+      .doc("Max number of change partition requests which can be concurrently processed ")
+      .version("0.5.0")
+      .intConf
+      .createWithDefault(256)
+
   val CLIENT_BATCH_HANDLE_CHANGE_PARTITION_THREADS: ConfigEntry[Int] =
     buildConf("celeborn.client.shuffle.batchHandleChangePartition.threads")
       .withAlternative("celeborn.shuffle.batchHandleChangePartition.threads")
diff --git a/docs/configuration/client.md b/docs/configuration/client.md
index 6d290080a..0fe90d0ec 100644
--- a/docs/configuration/client.md
+++ b/docs/configuration/client.md
@@ -82,6 +82,7 @@ license: |
 | celeborn.client.rpc.reserveSlots.askTimeout | &lt;value of celeborn.rpc.askTimeout&gt; | false | Timeout for LifecycleManager request reserve slots. | 0.3.0 |  | 
 | celeborn.client.rpc.shared.threads | 16 | false | Number of shared rpc threads in LifecycleManager. | 0.3.2 |  | 
 | celeborn.client.shuffle.batchHandleChangePartition.interval | 100ms | false | Interval for LifecycleManager to schedule handling change partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.interval | 
+| celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets | 256 | false | Max number of change partition requests which can be concurrently processed  | 0.5.0 |  | 
 | celeborn.client.shuffle.batchHandleChangePartition.threads | 8 | false | Threads number for LifecycleManager to handle change partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.threads | 
 | celeborn.client.shuffle.batchHandleCommitPartition.interval | 5s | false | Interval for LifecycleManager to schedule handling commit partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.interval | 
 | celeborn.client.shuffle.batchHandleCommitPartition.threads | 8 | false | Threads number for LifecycleManager to handle commit partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.threads |