You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by re...@apache.org on 2024/03/14 05:51:22 UTC
(incubator-celeborn) branch main updated: [CELEBORN-863][FOLLOWUP] Fix persisted committed file infos lost
This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 5043f46aa [CELEBORN-863][FOLLOWUP] Fix persisted committed file infos lost
5043f46aa is described below
commit 5043f46aa737a64248230dd5301661657c0541fc
Author: mingji <fe...@alibaba-inc.com>
AuthorDate: Thu Mar 14 13:51:10 2024 +0800
[CELEBORN-863][FOLLOWUP] Fix persisted committed file infos lost
### What changes were proposed in this pull request?
To fix a bug that might cause persisted committed file info lost.
### Why are the changes needed?
A worker starts will clean its persisted committed file info and won't put back if this worker restart again, the committed file infos will lost.
### Does this PR introduce _any_ user-facing change?
NO.
### How was this patch tested?
GA.
Closes #2390 from FMX/b863-1.
Authored-by: mingji <fe...@alibaba-inc.com>
Signed-off-by: Shuang <lv...@alibaba-inc.com>
---
.../org/apache/celeborn/common/util/PbSerDeUtils.scala | 4 ++--
.../org/apache/celeborn/common/util/PbSerDeUtilsTest.scala | 2 +-
.../service/deploy/worker/storage/StorageManager.scala | 14 +++++++-------
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala b/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
index 87d18f28a..1ca5759d2 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/PbSerDeUtils.scala
@@ -149,10 +149,10 @@ object PbSerDeUtils {
fileInfoMap
}
- def toPbFileInfoMap(fileInfoMap: ConcurrentHashMap[String, FileInfo]): Array[Byte] = {
+ def toPbFileInfoMap(fileInfoMap: ConcurrentHashMap[String, DiskFileInfo]): Array[Byte] = {
val pbFileInfoMap = JavaUtils.newConcurrentHashMap[String, PbFileInfo]()
fileInfoMap.entrySet().asScala.foreach { entry =>
- pbFileInfoMap.put(entry.getKey, toPbFileInfo(entry.getValue.asInstanceOf[DiskFileInfo]))
+ pbFileInfoMap.put(entry.getKey, toPbFileInfo(entry.getValue))
}
PbFileInfoMap.newBuilder.putAllValues(pbFileInfoMap).build.toByteArray
}
diff --git a/common/src/test/scala/org/apache/celeborn/common/util/PbSerDeUtilsTest.scala b/common/src/test/scala/org/apache/celeborn/common/util/PbSerDeUtilsTest.scala
index 0d8063d63..3f4c7b2f4 100644
--- a/common/src/test/scala/org/apache/celeborn/common/util/PbSerDeUtilsTest.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/util/PbSerDeUtilsTest.scala
@@ -69,7 +69,7 @@ class PbSerDeUtilsTest extends CelebornFunSuite {
new ReduceFileMeta(chunkOffsets2),
file2.getAbsolutePath,
6000L)
- val fileInfoMap = JavaUtils.newConcurrentHashMap[String, FileInfo]()
+ val fileInfoMap = JavaUtils.newConcurrentHashMap[String, DiskFileInfo]()
fileInfoMap.put("file1", fileInfo1)
fileInfoMap.put("file2", fileInfo2)
val cache = JavaUtils.newConcurrentHashMap[String, UserIdentifier]()
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
index ea84272bf..194e13378 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/storage/StorageManager.scala
@@ -37,7 +37,7 @@ import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.exception.CelebornException
import org.apache.celeborn.common.identity.UserIdentifier
import org.apache.celeborn.common.internal.Logging
-import org.apache.celeborn.common.meta.{DeviceInfo, DiskFileInfo, DiskInfo, DiskStatus, FileInfo, MapFileMeta, ReduceFileMeta, TimeWindow}
+import org.apache.celeborn.common.meta.{DeviceInfo, DiskFileInfo, DiskInfo, DiskStatus, MapFileMeta, ReduceFileMeta, TimeWindow}
import org.apache.celeborn.common.metrics.source.{AbstractSource, ThreadPoolSource}
import org.apache.celeborn.common.network.util.{NettyUtils, TransportConf}
import org.apache.celeborn.common.protocol.{PartitionLocation, PartitionSplitMode, PartitionType, StorageInfo}
@@ -203,7 +203,8 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
conf.workerGracefulShutdownSaveCommittedFileInfoSync
private val saveCommittedFileInfoInterval =
conf.workerGracefulShutdownSaveCommittedFileInfoInterval
- private var committedFileInfos: ConcurrentHashMap[String, ConcurrentHashMap[String, FileInfo]] = _
+ private val committedFileInfos =
+ JavaUtils.newConcurrentHashMap[String, ConcurrentHashMap[String, DiskFileInfo]]()
// ShuffleClient can fetch data from a restarted worker only
// when the worker's fetching port is stable.
val workerGracefulShutdown = conf.workerGracefulShutdown
@@ -219,8 +220,6 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
logError("Init level DB failed:", e)
this.db = null
}
- committedFileInfos =
- JavaUtils.newConcurrentHashMap[String, ConcurrentHashMap[String, FileInfo]]()
saveCommittedFileInfosExecutor =
ThreadUtils.newDaemonSingleThreadScheduledExecutor(
"worker-storage-manager-committed-fileinfo-saver")
@@ -266,6 +265,7 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
val files = PbSerDeUtils.fromPbFileInfoMap(entry.getValue, cache)
logDebug(s"Reload DB: $shuffleKey -> $files")
diskFileInfos.put(shuffleKey, files)
+ committedFileInfos.put(shuffleKey, files)
db.delete(entry.getKey)
} catch {
case exception: Exception =>
@@ -301,8 +301,8 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
private def getNextIndex() = counter.getAndUpdate(counterOperator)
private val newMapFunc =
- new java.util.function.Function[String, ConcurrentHashMap[String, FileInfo]]() {
- override def apply(key: String): ConcurrentHashMap[String, FileInfo] =
+ new java.util.function.Function[String, ConcurrentHashMap[String, DiskFileInfo]]() {
+ override def apply(key: String): ConcurrentHashMap[String, DiskFileInfo] =
JavaUtils.newConcurrentHashMap()
}
@@ -778,7 +778,7 @@ final private[worker] class StorageManager(conf: CelebornConf, workerSource: Abs
def notifyFileInfoCommitted(
shuffleKey: String,
fileName: String,
- fileInfo: FileInfo): Unit = {
+ fileInfo: DiskFileInfo): Unit = {
committedFileInfos.computeIfAbsent(shuffleKey, newMapFunc).put(fileName, fileInfo)
}