You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2021/09/23 08:53:51 UTC
[spark] branch branch-3.1 updated: [SPARK-36782][CORE][FOLLOW-UP]
Only handle shuffle block in separate thread pool
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8e36217 [SPARK-36782][CORE][FOLLOW-UP] Only handle shuffle block in separate thread pool
8e36217 is described below
commit 8e36217793f5120c4fbe59b48b686a572321414e
Author: yi.wu <yi...@databricks.com>
AuthorDate: Thu Sep 23 16:29:54 2021 +0800
[SPARK-36782][CORE][FOLLOW-UP] Only handle shuffle block in separate thread pool
### What changes were proposed in this pull request?
This's a follow-up of https://github.com/apache/spark/pull/34043. This PR proposes to only handle shuffle blocks in the separate thread pool and leave other blocks the same behavior as it is.
### Why are the changes needed?
To avoid any potential overhead.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass existing tests.
Closes #34076 from Ngone51/spark-36782-follow-up.
Authored-by: yi.wu <yi...@databricks.com>
Signed-off-by: Gengliang Wang <ge...@apache.org>
(cherry picked from commit 9d8ac7c8e90c1c2a6060e8f1e8f2f21e19622567)
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../spark/storage/BlockManagerMasterEndpoint.scala | 69 +++++++++++++---------
1 file changed, 41 insertions(+), 28 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
index a7289ef..c22b16a 100644
--- a/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
+++ b/core/src/main/scala/org/apache/spark/storage/BlockManagerMasterEndpoint.scala
@@ -117,15 +117,20 @@ class BlockManagerMasterEndpoint(
case _updateBlockInfo @
UpdateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size) =>
- val response = updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size)
- response.foreach { isSuccess =>
+ @inline def handleResult(success: Boolean): Unit = {
// SPARK-30594: we should not post `SparkListenerBlockUpdated` when updateBlockInfo
// returns false since the block info would be updated again later.
- if (isSuccess) {
+ if (success) {
listenerBus.post(SparkListenerBlockUpdated(BlockUpdatedInfo(_updateBlockInfo)))
}
- context.reply(isSuccess)
+ context.reply(success)
+ }
+
+ if (blockId.isShuffle) {
+ updateShuffleBlockInfo(blockId, blockManagerId).foreach(handleResult)
+ } else {
+ handleResult(updateBlockInfo(blockManagerId, blockId, storageLevel, deserializedSize, size))
}
case GetLocations(blockId) =>
@@ -566,46 +571,54 @@ class BlockManagerMasterEndpoint(
id
}
+ private def updateShuffleBlockInfo(blockId: BlockId, blockManagerId: BlockManagerId)
+ : Future[Boolean] = {
+ blockId match {
+ case ShuffleIndexBlockId(shuffleId, mapId, _) =>
+ // SPARK-36782: Invoke `MapOutputTracker.updateMapOutput` within the thread
+ // `dispatcher-BlockManagerMaster` could lead to the deadlock when
+ // `MapOutputTracker.serializeOutputStatuses` broadcasts the serialized mapstatues under
+ // the acquired write lock. The broadcast block would report its status to
+ // `BlockManagerMasterEndpoint`, while the `BlockManagerMasterEndpoint` is occupied by
+ // `updateMapOutput` since it's waiting for the write lock. Thus, we use `Future` to call
+ // `updateMapOutput` in a separate thread to avoid the deadlock.
+ Future {
+ // We need to update this at index file because there exists the index-only block
+ logDebug(s"Received shuffle index block update for ${shuffleId} ${mapId}, updating.")
+ mapOutputTracker.updateMapOutput(shuffleId, mapId, blockManagerId)
+ true
+ }
+ case ShuffleDataBlockId(shuffleId: Int, mapId: Long, reduceId: Int) =>
+ logDebug(s"Received shuffle data block update for ${shuffleId} ${mapId}, ignore.")
+ Future.successful(true)
+ case _ =>
+ logDebug(s"Unexpected shuffle block type ${blockId}" +
+ s"as ${blockId.getClass().getSimpleName()}")
+ Future.successful(false)
+ }
+ }
+
private def updateBlockInfo(
blockManagerId: BlockManagerId,
blockId: BlockId,
storageLevel: StorageLevel,
memSize: Long,
- diskSize: Long): Future[Boolean] = {
+ diskSize: Long): Boolean = {
logDebug(s"Updating block info on master ${blockId} for ${blockManagerId}")
- if (blockId.isShuffle) {
- blockId match {
- case ShuffleIndexBlockId(shuffleId, mapId, _) =>
- // We need to update this at index file because there exists the index-only block
- return Future {
- logDebug(s"Received shuffle index block update for ${shuffleId} ${mapId}, updating.")
- mapOutputTracker.updateMapOutput(shuffleId, mapId, blockManagerId)
- true
- }
- case ShuffleDataBlockId(shuffleId: Int, mapId: Long, reduceId: Int) =>
- logDebug(s"Received shuffle data block update for ${shuffleId} ${mapId}, ignore.")
- return Future.successful(true)
- case _ =>
- logDebug(s"Unexpected shuffle block type ${blockId}" +
- s"as ${blockId.getClass().getSimpleName()}")
- return Future.successful(false)
- }
- }
-
if (!blockManagerInfo.contains(blockManagerId)) {
if (blockManagerId.isDriver && !isLocal) {
// We intentionally do not register the master (except in local mode),
// so we should not indicate failure.
- return Future.successful(true)
+ return true
} else {
- return Future.successful(false)
+ return false
}
}
if (blockId == null) {
blockManagerInfo(blockManagerId).updateLastSeenMs()
- return Future.successful(true)
+ return true
}
blockManagerInfo(blockManagerId).updateBlockInfo(blockId, storageLevel, memSize, diskSize)
@@ -637,7 +650,7 @@ class BlockManagerMasterEndpoint(
if (locations.size == 0) {
blockLocations.remove(blockId)
}
- Future.successful(true)
+ true
}
private def getLocations(blockId: BlockId): Seq[BlockManagerId] = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org