You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/02/07 10:18:18 UTC
[incubator-celeborn] branch main updated: [CELEBORN-275] WrappedCallback should only handle response from replica (#1209)
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new cfe81969c [CELEBORN-275] WrappedCallback should only handle response from replica (#1209)
cfe81969c is described below
commit cfe81969c943a072ad440b3d79d078866fd40552
Author: Rex(Hui) An <an...@126.com>
AuthorDate: Tue Feb 7 18:18:13 2023 +0800
[CELEBORN-275] WrappedCallback should only handle response from replica (#1209)
---
.../service/deploy/worker/PushDataHandler.scala | 273 +++++++++++----------
1 file changed, 149 insertions(+), 124 deletions(-)
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
index 5f4413d1d..199008169 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
@@ -28,7 +28,7 @@ import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.exception.AlreadyClosedException
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{WorkerInfo, WorkerPartitionLocationInfo}
-import org.apache.celeborn.common.metrics.source.RPCSource
+import org.apache.celeborn.common.metrics.source.{RPCSource, Source}
import org.apache.celeborn.common.network.buffer.{NettyManagedBuffer, NioManagedBuffer}
import org.apache.celeborn.common.network.client.{RpcResponseCallback, TransportClient, TransportClientFactory}
import org.apache.celeborn.common.network.protocol.{Message, PushData, PushDataHandShake, PushMergedData, RegionFinish, RegionStart, RequestMessage, RpcFailure, RpcRequest, RpcResponse}
@@ -146,11 +146,20 @@ class PushDataHandler extends BaseMessageHandler with Logging {
}
val key = s"${pushData.requestId}"
- if (isMaster) {
- workerSource.startTimer(WorkerSource.MasterPushDataTime, key)
- } else {
- workerSource.startTimer(WorkerSource.SlavePushDataTime, key)
- }
+ val callbackWithTimer =
+ if (isMaster) {
+ new RpcResponseCallbackWithTimer(
+ workerSource,
+ WorkerSource.MasterPushDataTime,
+ key,
+ callback)
+ } else {
+ new RpcResponseCallbackWithTimer(
+ workerSource,
+ WorkerSource.SlavePushDataTime,
+ key,
+ callback)
+ }
// find FileWriter responsible for the data
val location =
@@ -164,51 +173,6 @@ class PushDataHandler extends BaseMessageHandler with Logging {
val doReplicate = location != null && location.getPeer != null && isMaster
val batchId = if (doReplicate) pushState.nextBatchId() else -1
val softSplit = new AtomicBoolean(false)
- val wrappedCallback = new RpcResponseCallback() {
- override def onSuccess(response: ByteBuffer): Unit = {
- if (isMaster) {
- if (doReplicate) {
- // Only master data will push data to slave
- pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
- }
- workerSource.stopTimer(WorkerSource.MasterPushDataTime, key)
- if (response.remaining() > 0) {
- val resp = ByteBuffer.allocate(response.remaining())
- resp.put(response)
- resp.flip()
- callback.onSuccess(resp)
- } else if (softSplit.get()) {
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.SOFT_SPLIT.getValue)))
- } else {
- callback.onSuccess(response)
- }
- } else {
- workerSource.stopTimer(WorkerSource.SlavePushDataTime, key)
- callback.onSuccess(response)
- }
- }
-
- override def onFailure(e: Throwable): Unit = {
- logError(s"[handlePushData.onFailure] partitionLocation: $location", e)
- workerSource.incCounter(WorkerSource.PushDataFailCount)
- if (doReplicate) {
- pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
- }
- // Throw by slave peer worker
- if (e.getMessage.startsWith(StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage)) {
- callback.onFailure(e)
- } else if (e.getMessage.startsWith(StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage)) {
- callback.onFailure(new Exception(
- s"${StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage}! Push data to peer of $location timeout: ${e.getMessage}",
- e))
- } else {
- // Throw by connection
- callback.onFailure(new Exception(
- s"${StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_SLAVE.getMessage}! Push data to peer of $location failed: ${e.getMessage}",
- e))
- }
- }
- }
if (location == null) {
val (mapId, attemptId) = getMapAttempt(body)
@@ -221,12 +185,12 @@ class PushDataHandler extends BaseMessageHandler with Logging {
// partition data has already been committed
logInfo(s"Receive push data from speculative task(shuffle $shuffleKey, map $mapId, " +
s" attempt $attemptId), but this mapper has already been ended.")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.STAGE_ENDED.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.STAGE_ENDED.getValue)))
} else {
logInfo(
s"Receive push data for committed hard split partition of (shuffle $shuffleKey, " +
s"map $mapId attempt $attemptId)")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
}
} else {
if (storageManager.shuffleKeySet().contains(shuffleKey)) {
@@ -236,12 +200,12 @@ class PushDataHandler extends BaseMessageHandler with Logging {
logInfo(
s"Receive push data for committed hard split partition of (shuffle $shuffleKey, " +
s"map $mapId attempt $attemptId)")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
} else {
val msg = s"Partition location wasn't found for task(shuffle $shuffleKey, map $mapId, " +
s"attempt $attemptId, uniqueId ${pushData.partitionUniqueId})."
logWarning(s"[handlePushData] $msg")
- callback.onFailure(
+ callbackWithTimer.onFailure(
new Exception(StatusCode.PUSH_DATA_FAIL_PARTITION_NOT_FOUND.getMessage()))
}
}
@@ -252,7 +216,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
// This should before return exception to make current push data can revive and retry.
if (shutdown.get()) {
logInfo(s"Push data return HARD_SPLIT for shuffle $shuffleKey since worker shutdown.")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
return
}
@@ -266,7 +230,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
} else {
StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage()
}
- callback.onFailure(new Exception(s"$message! $location", exception))
+ callbackWithTimer.onFailure(new Exception(s"$message! $location", exception))
return
}
val diskFull =
@@ -282,7 +246,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
if (fileWriter.getSplitMode == PartitionSplitMode.SOFT) {
softSplit.set(true)
} else {
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
return
}
}
@@ -303,12 +267,50 @@ class PushDataHandler extends BaseMessageHandler with Logging {
if (unavailablePeers.containsKey(peerWorker)) {
pushData.body().release()
workerSource.incCounter(WorkerSource.PushDataFailCount)
- callback.onFailure(
+ callbackWithTimer.onFailure(
new Exception(
s"${StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_SLAVE.getMessage}! Peer $peerWorker unavailable for $location!"))
return
}
pushState.addBatch(batchId, location.getPeer.hostAndPushPort())
+
+ // Handle the response from replica
+ val wrappedCallback = new RpcResponseCallback() {
+ override def onSuccess(response: ByteBuffer): Unit = {
+ // Only master data will push data to slave
+ pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
+ if (response.remaining() > 0) {
+ val resp = ByteBuffer.allocate(response.remaining())
+ resp.put(response)
+ resp.flip()
+ callbackWithTimer.onSuccess(resp)
+ } else if (softSplit.get()) {
+ callbackWithTimer.onSuccess(
+ ByteBuffer.wrap(Array[Byte](StatusCode.SOFT_SPLIT.getValue)))
+ } else {
+ callbackWithTimer.onSuccess(response)
+ }
+ }
+
+ override def onFailure(e: Throwable): Unit = {
+ logError(s"[handlePushData.onFailure] partitionLocation: $location", e)
+ workerSource.incCounter(WorkerSource.PushDataFailCount)
+ pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
+ // Throw by slave peer worker
+ if (e.getMessage.startsWith(StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage)) {
+ callbackWithTimer.onFailure(e)
+ } else if (e.getMessage.startsWith(StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage)) {
+ callbackWithTimer.onFailure(new Exception(
+ s"${StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage}! Push data to peer of $location timeout: ${e.getMessage}",
+ e))
+ } else {
+ // Throw by connection
+ callbackWithTimer.onFailure(new Exception(
+ s"${StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_SLAVE.getMessage}! Push data to peer of $location failed: ${e.getMessage}",
+ e))
+ }
+ }
+ }
try {
val client =
pushClientFactory.createClient(peer.getHost, peer.getReplicatePort, location.getId)
@@ -330,7 +332,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
unavailablePeers.put(peerWorker, System.currentTimeMillis())
workerSource.incCounter(WorkerSource.PushDataFailCount)
- callback.onFailure(
+ callbackWithTimer.onFailure(
new Exception(
s"${StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_SLAVE.getMessage}! Create connection to peer $peerWorker failed for $location",
e))
@@ -342,18 +344,18 @@ class PushDataHandler extends BaseMessageHandler with Logging {
case Some(rateLimitController) =>
if (rateLimitController.isUserCongested(fileWriter.getFileInfo.getUserIdentifier)) {
if (isMaster) {
- callback.onSuccess(
+ callbackWithTimer.onSuccess(
ByteBuffer.wrap(
Array[Byte](StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue)))
} else {
- callback.onSuccess(
+ callbackWithTimer.onSuccess(
ByteBuffer.wrap(Array[Byte](StatusCode.PUSH_DATA_SUCCESS_SLAVE_CONGESTED.getValue)))
}
} else {
- callback.onSuccess(ByteBuffer.wrap(Array[Byte]()))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte]()))
}
case None =>
- callback.onSuccess(ByteBuffer.wrap(Array[Byte]()))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte]()))
}
}
@@ -385,11 +387,20 @@ class PushDataHandler extends BaseMessageHandler with Logging {
val isMaster = mode == PartitionLocation.Mode.MASTER
val key = s"${pushMergedData.requestId}"
- if (isMaster) {
- workerSource.startTimer(WorkerSource.MasterPushDataTime, key)
- } else {
- workerSource.startTimer(WorkerSource.SlavePushDataTime, key)
- }
+ val callbackWithTimer =
+ if (isMaster) {
+ new RpcResponseCallbackWithTimer(
+ workerSource,
+ WorkerSource.MasterPushDataTime,
+ key,
+ callback)
+ } else {
+ new RpcResponseCallbackWithTimer(
+ workerSource,
+ WorkerSource.SlavePushDataTime,
+ key,
+ callback)
+ }
// For test
if (isMaster && conf.testPushMasterDataTimeout && !PushDataHandler.pushMasterDataTimeoutTested) {
@@ -414,49 +425,6 @@ class PushDataHandler extends BaseMessageHandler with Logging {
val doReplicate = locations.head != null && locations.head.getPeer != null && isMaster
val batchId = if (doReplicate) pushState.nextBatchId() else -1
- val wrappedCallback = new RpcResponseCallback() {
- override def onSuccess(response: ByteBuffer): Unit = {
- if (isMaster) {
- // Only master data enable replication will push data to slave
- if (doReplicate) {
- pushState.removeBatch(batchId, locations.head.getPeer.hostAndPushPort())
- }
- workerSource.stopTimer(WorkerSource.MasterPushDataTime, key)
- if (response.remaining() > 0) {
- val resp = ByteBuffer.allocate(response.remaining())
- resp.put(response)
- resp.flip()
- callback.onSuccess(resp)
- } else {
- callback.onSuccess(response)
- }
- } else {
- workerSource.stopTimer(WorkerSource.SlavePushDataTime, key)
- callback.onSuccess(response)
- }
- }
-
- override def onFailure(e: Throwable): Unit = {
- workerSource.incCounter(WorkerSource.PushDataFailCount)
- if (doReplicate) {
- pushState.removeBatch(batchId, locations.head.getPeer.hostAndPushPort())
- }
- // Throw by slave peer worker
- if (e.getMessage.startsWith(StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage)) {
- callback.onFailure(e)
- } else if (e.getMessage.startsWith(StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage)) {
- callback.onFailure(new Exception(
- s"${StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage}! Push data to peer of ${locations.head} timeout: ${e.getMessage}",
- e))
- } else {
- // Throw by connection
- callback.onFailure(new Exception(
- s"${StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_SLAVE.getMessage}! Push data to peer of ${locations.head} failed: ${e.getMessage}",
- e))
- }
- }
- }
-
// find FileWriters responsible for the data
locations.foreach { loc =>
if (loc == null) {
@@ -471,12 +439,14 @@ class PushDataHandler extends BaseMessageHandler with Logging {
s"Receive push merged data from speculative task(shuffle $shuffleKey, map $mapId," +
s" attempt $attemptId), but this mapper has already been ended."
logInfo(msg)
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.STAGE_ENDED.getValue)))
+ callbackWithTimer.onSuccess(
+ ByteBuffer.wrap(Array[Byte](StatusCode.STAGE_ENDED.getValue)))
} else {
logInfo(
s"Receive push merged data for committed hard split partition of (shuffle $shuffleKey, " +
s"map $mapId attempt $attemptId)")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(
+ ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
}
} else {
if (storageManager.shuffleKeySet().contains(shuffleKey)) {
@@ -486,12 +456,13 @@ class PushDataHandler extends BaseMessageHandler with Logging {
logInfo(
s"Receive push merged data for committed hard split partition of (shuffle $shuffleKey, " +
s"map $mapId attempt $attemptId)")
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(
+ ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
} else {
val msg = s"Partition location wasn't found for task(shuffle $shuffleKey, map $mapId," +
s" attempt $attemptId, uniqueId ${loc.getUniqueId})."
logWarning(s"[handlePushMergedData] $msg")
- callback.onFailure(
+ callbackWithTimer.onFailure(
new Exception(StatusCode.PUSH_DATA_FAIL_PARTITION_NOT_FOUND.getMessage()))
}
}
@@ -502,7 +473,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
// During worker shutdown, worker will return HARD_SPLIT for all existed partition.
// This should before return exception to make current push data can revive and retry.
if (shutdown.get()) {
- callback.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte](StatusCode.HARD_SPLIT.getValue)))
return
}
@@ -518,7 +489,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
} else {
StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage()
}
- callback.onFailure(new Exception(s"$message! ${locations.head}", exception))
+ callbackWithTimer.onFailure(new Exception(s"$message! ${locations.head}", exception))
return
}
fileWriters.foreach(_.incrementPendingWrites())
@@ -539,11 +510,46 @@ class PushDataHandler extends BaseMessageHandler with Logging {
if (unavailablePeers.containsKey(peerWorker)) {
pushMergedData.body().release()
workerSource.incCounter(WorkerSource.PushDataFailCount)
- callback.onFailure(new Exception(
+ callbackWithTimer.onFailure(new Exception(
s"${StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_SLAVE.getMessage}! Peer $peerWorker unavailable for $location!"))
return
}
pushState.addBatch(batchId, location.getPeer.hostAndPushPort())
+
+ // Handle the response from replica
+ val wrappedCallback = new RpcResponseCallback() {
+ override def onSuccess(response: ByteBuffer): Unit = {
+ // Only master data enable replication will push data to slave
+ pushState.removeBatch(batchId, locations.head.getPeer.hostAndPushPort())
+ if (response.remaining() > 0) {
+ val resp = ByteBuffer.allocate(response.remaining())
+ resp.put(response)
+ resp.flip()
+ callbackWithTimer.onSuccess(resp)
+ } else {
+ callbackWithTimer.onSuccess(response)
+ }
+ }
+
+ override def onFailure(e: Throwable): Unit = {
+ workerSource.incCounter(WorkerSource.PushDataFailCount)
+ pushState.removeBatch(batchId, locations.head.getPeer.hostAndPushPort())
+ // Throw by slave peer worker
+ if (e.getMessage.startsWith(StatusCode.PUSH_DATA_FAIL_SLAVE.getMessage)) {
+ callbackWithTimer.onFailure(e)
+ } else if (e.getMessage.startsWith(StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage)) {
+ callbackWithTimer.onFailure(new Exception(
+ s"${StatusCode.PUSH_DATA_TIMEOUT_SLAVE.getMessage}! Push data to peer of ${locations.head} timeout: ${e.getMessage}",
+ e))
+ } else {
+ // Throw by connection
+ callbackWithTimer.onFailure(new Exception(
+ s"${StatusCode.PUSH_DATA_CONNECTION_EXCEPTION_SLAVE.getMessage}! Push data to peer of ${locations.head} failed: ${e.getMessage}",
+ e))
+ }
+ }
+ }
+
try {
val client = pushClientFactory.createClient(
peer.getHost,
@@ -568,7 +574,7 @@ class PushDataHandler extends BaseMessageHandler with Logging {
unavailablePeers.put(peerWorker, System.currentTimeMillis())
workerSource.incCounter(WorkerSource.PushDataFailCount)
pushState.removeBatch(batchId, location.getPeer.hostAndPushPort())
- callback.onFailure(
+ callbackWithTimer.onFailure(
new Exception(
s"${StatusCode.PUSH_DATA_CREATE_CONNECTION_FAIL_SLAVE.getMessage}! Create connection to peer $peerWorker failed for $location",
e))
@@ -580,18 +586,18 @@ class PushDataHandler extends BaseMessageHandler with Logging {
case Some(rateLimitController) if fileWriters.nonEmpty =>
if (rateLimitController.isUserCongested(fileWriters.head.getFileInfo.getUserIdentifier)) {
if (isMaster) {
- callback.onSuccess(
+ callbackWithTimer.onSuccess(
ByteBuffer.wrap(
Array[Byte](StatusCode.PUSH_DATA_SUCCESS_MASTER_CONGESTED.getValue)))
} else {
- callback.onSuccess(
+ callbackWithTimer.onSuccess(
ByteBuffer.wrap(Array[Byte](StatusCode.PUSH_DATA_SUCCESS_SLAVE_CONGESTED.getValue)))
}
} else {
- callback.onSuccess(ByteBuffer.wrap(Array[Byte]()))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte]()))
}
case None =>
- callback.onSuccess(ByteBuffer.wrap(Array[Byte]()))
+ callbackWithTimer.onSuccess(ByteBuffer.wrap(Array[Byte]()))
}
}
@@ -645,6 +651,25 @@ class PushDataHandler extends BaseMessageHandler with Logging {
override def checkRegistered(): Boolean = registered.get()
+ class RpcResponseCallbackWithTimer(
+ source: Source,
+ metricName: String,
+ key: String,
+ callback: RpcResponseCallback)
+ extends RpcResponseCallback {
+ source.startTimer(metricName, key)
+
+ override def onSuccess(response: ByteBuffer): Unit = {
+ callback.onSuccess(response)
+ source.stopTimer(metricName, key)
+ }
+
+ override def onFailure(e: Throwable): Unit = {
+ callback.onFailure(e)
+ source.stopTimer(metricName, key)
+ }
+ }
+
class SimpleRpcResponseCallback(
messageType: Message.Type,
client: TransportClient,