You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by et...@apache.org on 2024/07/18 01:36:19 UTC
(celeborn) branch branch-0.5 updated: [CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure
This is an automated email from the ASF dual-hosted git repository.
ethanfeng pushed a commit to branch branch-0.5
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/branch-0.5 by this push:
new 8c54b3968 [CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure
8c54b3968 is described below
commit 8c54b3968426d6f3668f320062b35e4283816b72
Author: wangshengjie3 <so...@gmail.com>
AuthorDate: Thu Jul 18 09:35:53 2024 +0800
[CELEBORN-1439] Fix revive logic bug which will casue data correctness issue and job failiure
### What changes were proposed in this pull request?
Fix revive logic bug to avoid data correctness issue
### Why are the changes needed?
Current logic will revive same partition-epoch multi times
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
UT
Closes #2532 from wangshengjie123/fix-revive.
Lead-authored-by: wangshengjie3 <so...@gmail.com>
Co-authored-by: wangshengjie <wa...@xiaomi.com>
Co-authored-by: CodingCat <zh...@gmail.com>
Signed-off-by: mingji <fe...@alibaba-inc.com>
(cherry picked from commit 32509d596258c6f3ffcaedb005d4b42cb1b60937)
Signed-off-by: mingji <fe...@alibaba-inc.com>
---
.../celeborn/client/ChangePartitionManager.scala | 79 ++++++++++++----------
.../org/apache/celeborn/common/CelebornConf.scala | 10 +++
docs/configuration/client.md | 1 +
3 files changed, 55 insertions(+), 35 deletions(-)
diff --git a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
index 6909d5262..d8cfa655d 100644
--- a/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
+++ b/client/src/main/scala/org/apache/celeborn/client/ChangePartitionManager.scala
@@ -47,8 +47,14 @@ class ChangePartitionManager(
// shuffleId -> (partitionId -> set of ChangePartition)
private val changePartitionRequests =
JavaUtils.newConcurrentHashMap[Int, ConcurrentHashMap[Integer, JSet[ChangePartitionRequest]]]()
+
+ // shuffleId -> locks
+ private val locks = JavaUtils.newConcurrentHashMap[Int, Array[AnyRef]]()
+ private val lockBucketSize = conf.batchHandleChangePartitionBuckets
+
// shuffleId -> set of partition id
- private val inBatchPartitions = JavaUtils.newConcurrentHashMap[Int, JSet[Integer]]()
+ private val inBatchPartitions =
+ JavaUtils.newConcurrentHashMap[Int, ConcurrentHashMap.KeySetView[Int, java.lang.Boolean]]()
private val batchHandleChangePartitionEnabled = conf.batchHandleChangePartitionEnabled
private val batchHandleChangePartitionExecutors = ThreadUtils.newDaemonCachedThreadPool(
@@ -79,14 +85,20 @@ class ChangePartitionManager(
batchHandleChangePartitionExecutors.submit {
new Runnable {
override def run(): Unit = {
- val distinctPartitions = requests.synchronized {
- // For each partition only need handle one request
- requests.asScala.filter { case (partitionId, _) =>
- !inBatchPartitions.get(shuffleId).contains(partitionId)
- }.map { case (partitionId, request) =>
- inBatchPartitions.get(shuffleId).add(partitionId)
- request.asScala.toArray.maxBy(_.epoch)
- }.toArray
+ val distinctPartitions = {
+ val requestSet = inBatchPartitions.get(shuffleId)
+ val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+ requests.asScala.map { case (partitionId, request) =>
+ locksForShuffle(partitionId % locksForShuffle.length).synchronized {
+ if (!requestSet.contains(partitionId) && requests.containsKey(
+ partitionId)) {
+ requestSet.add(partitionId)
+ Some(request.asScala.toArray.maxBy(_.epoch))
+ } else {
+ None
+ }
+ }
+ }.filter(_.isDefined).map(_.get).toArray
}
if (distinctPartitions.nonEmpty) {
handleRequestPartitions(
@@ -123,8 +135,16 @@ class ChangePartitionManager(
JavaUtils.newConcurrentHashMap()
}
- private val inBatchShuffleIdRegisterFunc = new util.function.Function[Int, util.Set[Integer]]() {
- override def apply(s: Int): util.Set[Integer] = new util.HashSet[Integer]()
+ private val inBatchShuffleIdRegisterFunc =
+ new util.function.Function[Int, ConcurrentHashMap.KeySetView[Int, java.lang.Boolean]]() {
+ override def apply(s: Int): ConcurrentHashMap.KeySetView[Int, java.lang.Boolean] =
+ ConcurrentHashMap.newKeySet[Int]()
+ }
+
+ private val locksRegisterFunc = new util.function.Function[Int, Array[AnyRef]] {
+ override def apply(t: Int): Array[AnyRef] = {
+ Array.fill(lockBucketSize)(new AnyRef())
+ }
}
def handleRequestPartitionLocation(
@@ -151,36 +171,22 @@ class ChangePartitionManager(
oldPartition,
cause)
- // If new slot for the partition has been allocated, reply and return.
- // Else register and allocate for it.
- getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { latestLoc =>
- context.reply(
- partitionId,
- StatusCode.SUCCESS,
- Some(latestLoc),
- lifecycleManager.workerStatusTracker.workerAvailable(oldPartition))
- logDebug(s"New partition found, old partition $partitionId-$oldEpoch return it." +
- s" shuffleId: $shuffleId $latestLoc")
- return
- }
-
- requests.synchronized {
+ val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+ locksForShuffle(partitionId % locksForShuffle.length).synchronized {
if (requests.containsKey(partitionId)) {
+ logDebug(s"[handleRequestPartitionLocation] For shuffle: $shuffleId, request for same " +
+ s"partition: $partitionId-$oldEpoch exists, register context.")
requests.get(partitionId).add(changePartition)
- logTrace(s"[handleRequestPartitionLocation] For $shuffleId, request for same partition" +
- s"$partitionId-$oldEpoch exists, register context.")
return
} else {
- // If new slot for the partition has been allocated, reply and return.
- // Else register and allocate for it.
getLatestPartition(shuffleId, partitionId, oldEpoch).foreach { latestLoc =>
context.reply(
partitionId,
StatusCode.SUCCESS,
Some(latestLoc),
lifecycleManager.workerStatusTracker.workerAvailable(oldPartition))
- logDebug(s"New partition found, old partition $partitionId-$oldEpoch return it." +
- s" shuffleId: $shuffleId $latestLoc")
+ logDebug(s"[handleRequestPartitionLocation]: For shuffle: $shuffleId," +
+ s" old partition: $partitionId-$oldEpoch, new partition: $latestLoc found, return it")
return
}
val set = new util.HashSet[ChangePartitionRequest]()
@@ -229,8 +235,9 @@ class ChangePartitionManager(
// remove together to reduce lock time
def replySuccess(locations: Array[PartitionLocation]): Unit = {
- requestsMap.synchronized {
- locations.map { location =>
+ val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+ locations.map { location =>
+ locksForShuffle(location.getId % locksForShuffle.length).synchronized {
if (batchHandleChangePartitionEnabled) {
inBatchPartitions.get(shuffleId).remove(location.getId)
}
@@ -250,8 +257,9 @@ class ChangePartitionManager(
// remove together to reduce lock time
def replyFailure(status: StatusCode): Unit = {
- requestsMap.synchronized {
- changePartitions.map { changePartition =>
+ changePartitions.map { changePartition =>
+ val locksForShuffle = locks.computeIfAbsent(shuffleId, locksRegisterFunc)
+ locksForShuffle(changePartition.partitionId % locksForShuffle.length).synchronized {
if (batchHandleChangePartitionEnabled) {
inBatchPartitions.get(shuffleId).remove(changePartition.partitionId)
}
@@ -338,5 +346,6 @@ class ChangePartitionManager(
def removeExpiredShuffle(shuffleId: Int): Unit = {
changePartitionRequests.remove(shuffleId)
inBatchPartitions.remove(shuffleId)
+ locks.remove(shuffleId)
}
}
diff --git a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
index a303833ce..4085bc7f8 100644
--- a/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/CelebornConf.scala
@@ -1025,6 +1025,8 @@ class CelebornConf(loadDefaults: Boolean) extends Cloneable with Logging with Se
PartitionSplitMode.valueOf(get(SHUFFLE_PARTITION_SPLIT_MODE))
def shufflePartitionSplitThreshold: Long = get(SHUFFLE_PARTITION_SPLIT_THRESHOLD)
def batchHandleChangePartitionEnabled: Boolean = get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_ENABLED)
+ def batchHandleChangePartitionBuckets: Int =
+ get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_BUCKETS)
def batchHandleChangePartitionNumThreads: Int = get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_THREADS)
def batchHandleChangePartitionRequestInterval: Long =
get(CLIENT_BATCH_HANDLE_CHANGE_PARTITION_INTERVAL)
@@ -4127,6 +4129,14 @@ object CelebornConf extends Logging {
.booleanConf
.createWithDefault(true)
+ val CLIENT_BATCH_HANDLE_CHANGE_PARTITION_BUCKETS: ConfigEntry[Int] =
+ buildConf("celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets")
+ .categories("client")
+ .doc("Max number of change partition requests which can be concurrently processed ")
+ .version("0.5.0")
+ .intConf
+ .createWithDefault(256)
+
val CLIENT_BATCH_HANDLE_CHANGE_PARTITION_THREADS: ConfigEntry[Int] =
buildConf("celeborn.client.shuffle.batchHandleChangePartition.threads")
.withAlternative("celeborn.shuffle.batchHandleChangePartition.threads")
diff --git a/docs/configuration/client.md b/docs/configuration/client.md
index 6d290080a..0fe90d0ec 100644
--- a/docs/configuration/client.md
+++ b/docs/configuration/client.md
@@ -82,6 +82,7 @@ license: |
| celeborn.client.rpc.reserveSlots.askTimeout | <value of celeborn.rpc.askTimeout> | false | Timeout for LifecycleManager request reserve slots. | 0.3.0 | |
| celeborn.client.rpc.shared.threads | 16 | false | Number of shared rpc threads in LifecycleManager. | 0.3.2 | |
| celeborn.client.shuffle.batchHandleChangePartition.interval | 100ms | false | Interval for LifecycleManager to schedule handling change partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.interval |
+| celeborn.client.shuffle.batchHandleChangePartition.partitionBuckets | 256 | false | Max number of change partition requests which can be concurrently processed | 0.5.0 | |
| celeborn.client.shuffle.batchHandleChangePartition.threads | 8 | false | Threads number for LifecycleManager to handle change partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleChangePartition.threads |
| celeborn.client.shuffle.batchHandleCommitPartition.interval | 5s | false | Interval for LifecycleManager to schedule handling commit partition requests in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.interval |
| celeborn.client.shuffle.batchHandleCommitPartition.threads | 8 | false | Threads number for LifecycleManager to handle commit partition request in batch. | 0.3.0 | celeborn.shuffle.batchHandleCommitPartition.threads |