You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by an...@apache.org on 2023/04/12 10:47:10 UTC
[incubator-celeborn] branch main updated: [CELEBORN-516][PERF] Remove RPCSource since it cost too much CPU (#1420)
This is an automated email from the ASF dual-hosted git repository.
angerszhuuuu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new da98ed9be [CELEBORN-516][PERF] Remove RPCSource since it cost too much CPU (#1420)
da98ed9be is described below
commit da98ed9beac59f8fb35ee6f1c332257570457cb6
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Wed Apr 12 18:47:06 2023 +0800
[CELEBORN-516][PERF] Remove RPCSource since it cost too much CPU (#1420)
---
METRICS.md | 11 --
.../celeborn/common/metrics/source/RPCSource.scala | 126 ---------------------
.../org/apache/celeborn/common/rpc/RpcEnv.scala | 6 +-
.../celeborn/common/rpc/netty/NettyRpcEnv.scala | 11 +-
.../celeborn/service/deploy/master/Master.scala | 6 +-
.../service/deploy/worker/FetchHandler.scala | 7 --
.../service/deploy/worker/PushDataHandler.scala | 5 +-
.../celeborn/service/deploy/worker/Worker.scala | 6 +-
.../deploy/worker/storage/FileWriterSuiteJ.java | 7 --
9 files changed, 7 insertions(+), 178 deletions(-)
diff --git a/METRICS.md b/METRICS.md
index 77f49b4d0..4f191a92e 100644
--- a/METRICS.md
+++ b/METRICS.md
@@ -87,17 +87,6 @@ Here is an example of grafana dashboard importing.
| DiskBuffer | worker | Disk buffers are part of netty used memory, means data need to write to disk but haven't been written to disk. |
| PausePushData | worker | PausePushData means the count of worker stopped receiving data from client. |
| PausePushDataAndReplicate | worker | PausePushDataAndReplicate means the count of worker stopped receiving data from client and other workers. |
-| RPCReserveSlotsNum | worker | The count of the RPC `ReserveSlots` received by the worker. |
-| RPCReserveSlotsSize | worker | The size of the RPC `ReserveSlots` 's body received by the worker. |
-| RPCPushDataNum | worker | The count of the RPC `PushData` received by the worker. |
-| RPCPushDataSize | worker | The size of the RPC `PushData` 's body received by the worker. |
-| RPCPushMergedDataNum | worker | The count of the RPC `PushMergedData` RPC received by the worker. |
-| RPCPushMergedDataSize | worker | The size of the RPC `PushMergedData` 's body received by the worker. |
-| RPCCommitFilesNum | worker | The count of the RPC `CommitFiles` received by the worker. |
-| RPCCommitFilesSize | worker | The size of the RPC `CommitFiles` 's body received by the worker. |
-| RPCDestroyNum | worker | The count of the RPC `Destroy` received by the worker. |
-| RPCDestroySize | worker | The size of the RPC `Destroy` 's body received by the worker. |
-| RPCChunkFetchRequestNum | worker | The count of the RPC `ChunkFetchRequest` RPC received by the worker. |
## Implementation
diff --git a/common/src/main/scala/org/apache/celeborn/common/metrics/source/RPCSource.scala b/common/src/main/scala/org/apache/celeborn/common/metrics/source/RPCSource.scala
deleted file mode 100644
index 52ac8e2e3..000000000
--- a/common/src/main/scala/org/apache/celeborn/common/metrics/source/RPCSource.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.celeborn.common.metrics.source
-
-import org.apache.celeborn.common.CelebornConf
-import org.apache.celeborn.common.network.protocol.{ChunkFetchRequest, OpenStream, PushData, PushMergedData}
-import org.apache.celeborn.common.protocol.{PbRegisterWorker, PbUnregisterShuffle}
-import org.apache.celeborn.common.protocol.message.ControlMessages._
-
-class RPCSource(conf: CelebornConf, role: String) extends AbstractSource(conf, role) {
- override val sourceName = "rpc"
-
- import RPCSource._
-
- // Worker RPC
- addCounter(RPCReserveSlotsNum)
- addCounter(RPCReserveSlotsSize)
- addCounter(RPCCommitFilesNum)
- addCounter(RPCCommitFilesSize)
- addCounter(RPCDestroyNum)
- addCounter(RPCDestroySize)
- addCounter(RPCPushDataNum)
- addCounter(RPCPushDataSize)
- addCounter(RPCPushMergedDataNum)
- addCounter(RPCPushMergedDataSize)
- addCounter(RPCOpenStreamNum)
- addCounter(RPCChunkFetchRequestNum)
-
- // Master RPC
- addCounter(RPCHeartbeatFromApplicationNum)
- addCounter(RPCHeartbeatFromWorkerNum)
- addCounter(RPCRegisterWorkerNum)
- addCounter(RPCRequestSlotsNum)
- addCounter(RPCReleaseSlotsNum)
- addCounter(RPCReleaseSlotsSize)
- addCounter(RPCUnregisterShuffleNum)
- addCounter(RPCReportWorkerUnavailableNum)
- addCounter(RPCReportWorkerUnavailableSize)
- addCounter(RPCCheckQuotaNum)
-
- def updateMessageMetrics(message: Any, messageLen: Long): Unit = {
- message match {
- case _: ReserveSlots =>
- incCounter(RPCReserveSlotsNum)
- incCounter(RPCReserveSlotsSize, messageLen)
- case _: CommitFiles =>
- incCounter(RPCCommitFilesNum)
- incCounter(RPCCommitFilesSize, messageLen)
- case _: Destroy =>
- incCounter(RPCDestroyNum)
- incCounter(RPCDestroySize, messageLen)
- case _: PushData =>
- incCounter(RPCPushDataNum)
- incCounter(RPCPushDataSize, messageLen)
- case _: PushMergedData =>
- incCounter(RPCPushMergedDataNum)
- incCounter(RPCPushMergedDataSize, messageLen)
- case _: ChunkFetchRequest =>
- incCounter(RPCChunkFetchRequestNum)
- case _: OpenStream =>
- incCounter(RPCOpenStreamNum)
- case _: HeartbeatFromApplication =>
- incCounter(RPCHeartbeatFromApplicationNum)
- case _: HeartbeatFromWorker =>
- incCounter(RPCHeartbeatFromWorkerNum)
- case _: PbRegisterWorker =>
- incCounter(RPCRegisterWorkerNum)
- case _: RequestSlots =>
- incCounter(RPCRequestSlotsNum)
- case _: ReleaseSlots =>
- incCounter(RPCReleaseSlotsNum)
- incCounter(RPCReleaseSlotsSize, messageLen)
- case _: PbUnregisterShuffle =>
- incCounter(RPCUnregisterShuffleNum)
- case _: ReportWorkerUnavailable =>
- incCounter(RPCReportWorkerUnavailableNum)
- incCounter(RPCReportWorkerUnavailableSize, messageLen)
- case CheckQuota =>
- incCounter(RPCCheckQuotaNum)
- case _ => // Do nothing
- }
- }
-}
-
-object RPCSource {
- // Worker RPC
- val RPCReserveSlotsNum = "RPCReserveSlotsNum"
- val RPCReserveSlotsSize = "RPCReserveSlotsSize"
- val RPCCommitFilesNum = "RPCCommitFilesNum"
- val RPCCommitFilesSize = "RPCCommitFilesSize"
- val RPCDestroyNum = "RPCDestroyNum"
- val RPCDestroySize = "RPCDestroySize"
- val RPCPushDataNum = "RPCPushDataNum"
- val RPCPushDataSize = "RPCPushDataSize"
- val RPCPushMergedDataNum = "RPCPushMergedDataNum"
- val RPCPushMergedDataSize = "RPCPushMergedDataSize"
- val RPCOpenStreamNum = "RPCOpenStreamNum"
- val RPCChunkFetchRequestNum = "RPCChunkFetchRequestNum"
-
- // Master RPC
- val RPCHeartbeatFromApplicationNum = "RPCHeartbeatFromApplicationNum"
- val RPCHeartbeatFromWorkerNum = "RPCHeartbeatFromWorkerNum"
- val RPCRegisterWorkerNum = "RPCRegisterWorkerNum"
- val RPCRequestSlotsNum = "RPCRequestSlotsNum"
- val RPCReleaseSlotsNum = "RPCReleaseSlotsNum"
- val RPCReleaseSlotsSize = "RPCReleaseSlotsSize"
- val RPCUnregisterShuffleNum = "RPCUnregisterShuffleNum"
- val RPCReportWorkerUnavailableNum = "RPCReportWorkerUnavailableNum"
- val RPCReportWorkerUnavailableSize = "RPCReportWorkerUnavailableSize"
- val RPCCheckQuotaNum = "RPCCheckQuotaNum"
-}
diff --git a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEnv.scala b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEnv.scala
index 0f5177587..673724304 100644
--- a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEnv.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEnv.scala
@@ -22,7 +22,6 @@ import java.io.File
import scala.concurrent.Future
import org.apache.celeborn.common.CelebornConf
-import org.apache.celeborn.common.metrics.source.RPCSource
import org.apache.celeborn.common.rpc.netty.NettyRpcEnvFactory
/**
@@ -79,10 +78,7 @@ abstract class RpcEnv(conf: CelebornConf) {
* Register a [[RpcEndpoint]] with a name and return its [[RpcEndpointRef]]. [[RpcEnv]] does not
* guarantee thread-safety.
*/
- def setupEndpoint(
- name: String,
- endpoint: RpcEndpoint,
- source: Option[RPCSource] = None): RpcEndpointRef
+ def setupEndpoint(name: String, endpoint: RpcEndpoint): RpcEndpointRef
/**
* Retrieve the [[RpcEndpointRef]] represented by `uri` asynchronously.
diff --git a/common/src/main/scala/org/apache/celeborn/common/rpc/netty/NettyRpcEnv.scala b/common/src/main/scala/org/apache/celeborn/common/rpc/netty/NettyRpcEnv.scala
index 1cde80304..57f2c1c33 100644
--- a/common/src/main/scala/org/apache/celeborn/common/rpc/netty/NettyRpcEnv.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/rpc/netty/NettyRpcEnv.scala
@@ -33,7 +33,6 @@ import com.google.common.base.Throwables
import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.internal.Logging
-import org.apache.celeborn.common.metrics.source.RPCSource
import org.apache.celeborn.common.network.TransportContext
import org.apache.celeborn.common.network.buffer.NioManagedBuffer
import org.apache.celeborn.common.network.client._
@@ -59,8 +58,6 @@ class NettyRpcEnv(
private var worker: RpcEndpoint = null
- var source: Option[RPCSource] = None
-
private val transportContext =
new TransportContext(transportConf, new NettyRpcHandler(dispatcher, this))
@@ -107,14 +104,10 @@ class NettyRpcEnv(
if (server != null) RpcAddress(host, server.getPort()) else null
}
- override def setupEndpoint(
- name: String,
- endpoint: RpcEndpoint,
- abstractSource: Option[RPCSource] = None): RpcEndpointRef = {
+ override def setupEndpoint(name: String, endpoint: RpcEndpoint): RpcEndpointRef = {
if (name == RpcNameConstants.WORKER_EP) {
worker = endpoint
}
- source = abstractSource
dispatcher.registerRpcEndpoint(name, endpoint)
}
@@ -595,10 +588,8 @@ private[celeborn] class NettyRpcHandler(
private def internalReceive(client: TransportClient, message: ByteBuffer): RequestMessage = {
val addr = client.getChannel().remoteAddress().asInstanceOf[InetSocketAddress]
assert(addr != null)
- val messageLen = message.remaining()
val clientAddr = RpcAddress(addr.getHostString, addr.getPort)
val requestMessage = RequestMessage(nettyEnv, client, message)
- nettyEnv.source.foreach(_.updateMessageMetrics(requestMessage.content, messageLen))
if (requestMessage.senderAddress == null) {
// Create a new message with the socket address of the client as the sender.
new RequestMessage(clientAddr, requestMessage.receiver, requestMessage.content)
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 3cf4df280..2aa7c2c25 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -31,7 +31,7 @@ import org.apache.celeborn.common.identity.UserIdentifier
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{DiskInfo, WorkerInfo}
import org.apache.celeborn.common.metrics.MetricsSystem
-import org.apache.celeborn.common.metrics.source.{JVMCPUSource, JVMSource, ResourceConsumptionSource, RPCSource}
+import org.apache.celeborn.common.metrics.source.{JVMCPUSource, JVMSource, ResourceConsumptionSource}
import org.apache.celeborn.common.protocol._
import org.apache.celeborn.common.protocol.message.{ControlMessages, StatusCode}
import org.apache.celeborn.common.protocol.message.ControlMessages._
@@ -138,7 +138,6 @@ private[celeborn] class Master(
private val slotsAssignPolicy = conf.slotsAssignPolicy
// init and register master metrics
- val rpcSource = new RPCSource(conf, MetricsSystem.ROLE_MASTER)
val resourceConsumptionSource = new ResourceConsumptionSource(conf)
private val masterSource = new MasterSource(conf)
masterSource.addGauge(
@@ -153,13 +152,12 @@ private[celeborn] class Master(
// is master active under HA mode
masterSource.addGauge(MasterSource.IsActiveMaster, _ => isMasterActive)
- metricsSystem.registerSource(rpcSource)
metricsSystem.registerSource(resourceConsumptionSource)
metricsSystem.registerSource(masterSource)
metricsSystem.registerSource(new JVMSource(conf, MetricsSystem.ROLE_MASTER))
metricsSystem.registerSource(new JVMCPUSource(conf, MetricsSystem.ROLE_MASTER))
- rpcEnv.setupEndpoint(RpcNameConstants.MASTER_EP, this, Some(rpcSource))
+ rpcEnv.setupEndpoint(RpcNameConstants.MASTER_EP, this)
// start threads to check timeout for workers and applications
override def onStart(): Unit = {
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
index 864be6f44..e38f329ba 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/FetchHandler.scala
@@ -29,7 +29,6 @@ import io.netty.util.concurrent.{Future, GenericFutureListener}
import org.apache.celeborn.common.exception.CelebornException
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{FileInfo, FileManagedBuffers}
-import org.apache.celeborn.common.metrics.source.RPCSource
import org.apache.celeborn.common.network.buffer.NioManagedBuffer
import org.apache.celeborn.common.network.client.TransportClient
import org.apache.celeborn.common.network.protocol._
@@ -46,14 +45,12 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
conf.getCelebornConf.partitionReadBuffersMax,
conf.getCelebornConf.bufferStreamThreadsPerMountpoint)
var workerSource: WorkerSource = _
- var rpcSource: RPCSource = _
var storageManager: StorageManager = _
var partitionsSorter: PartitionFilesSorter = _
var registered: AtomicBoolean = new AtomicBoolean(false)
def init(worker: Worker): Unit = {
this.workerSource = worker.workerSource
- this.rpcSource = worker.rpcSource
this.storageManager = worker.storageManager
this.partitionsSorter = worker.partitionsSorter
this.registered = worker.registered
@@ -75,17 +72,13 @@ class FetchHandler(val conf: TransportConf) extends BaseMessageHandler with Logg
override def receive(client: TransportClient, msg: RequestMessage): Unit = {
msg match {
case r: BufferStreamEnd =>
- rpcSource.updateMessageMetrics(r, 0)
handleEndStreamFromClient(client, r)
case r: ReadAddCredit =>
- rpcSource.updateMessageMetrics(r, 0)
handleReadAddCredit(client, r)
case r: ChunkFetchRequest =>
- rpcSource.updateMessageMetrics(r, 0)
handleChunkFetchRequest(client, r)
case r: RpcRequest =>
val msg = Message.decode(r.body().nioByteBuffer())
- rpcSource.updateMessageMetrics(msg, 0)
handleOpenStream(client, r, msg)
case unknown: RequestMessage =>
throw new IllegalArgumentException(s"Unknown message type id: ${unknown.`type`.id}")
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
index d4f6c72db..bdd390337 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/PushDataHandler.scala
@@ -28,7 +28,7 @@ import org.apache.celeborn.common.CelebornConf
import org.apache.celeborn.common.exception.{AlreadyClosedException, CelebornIOException}
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{WorkerInfo, WorkerPartitionLocationInfo}
-import org.apache.celeborn.common.metrics.source.{RPCSource, Source}
+import org.apache.celeborn.common.metrics.source.Source
import org.apache.celeborn.common.network.buffer.{NettyManagedBuffer, NioManagedBuffer}
import org.apache.celeborn.common.network.client.{RpcResponseCallback, TransportClient, TransportClientFactory}
import org.apache.celeborn.common.network.protocol.{Message, PushData, PushDataHandShake, PushMergedData, RegionFinish, RegionStart, RequestMessage, RpcFailure, RpcRequest, RpcResponse}
@@ -44,7 +44,6 @@ import org.apache.celeborn.service.deploy.worker.storage.{FileWriter, HdfsFlushe
class PushDataHandler extends BaseMessageHandler with Logging {
var workerSource: WorkerSource = _
- var rpcSource: RPCSource = _
var partitionLocationInfo: WorkerPartitionLocationInfo = _
var shuffleMapperAttempts: ConcurrentHashMap[String, AtomicIntegerArray] = _
var shufflePartitionType: ConcurrentHashMap[String, PartitionType] = _
@@ -64,7 +63,6 @@ class PushDataHandler extends BaseMessageHandler with Logging {
def init(worker: Worker): Unit = {
workerSource = worker.workerSource
- rpcSource = worker.rpcSource
partitionLocationInfo = worker.partitionLocationInfo
shufflePartitionType = worker.shufflePartitionType
shufflePushDataTimeout = worker.shufflePushDataTimeout
@@ -692,7 +690,6 @@ class PushDataHandler extends BaseMessageHandler with Logging {
requestId: Long,
handler: () => Unit): Unit = {
try {
- rpcSource.updateMessageMetrics(message, message.body().size())
handler()
} catch {
case e: Exception =>
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 741cfb596..7199979c8 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -35,7 +35,7 @@ import org.apache.celeborn.common.identity.UserIdentifier
import org.apache.celeborn.common.internal.Logging
import org.apache.celeborn.common.meta.{DiskInfo, WorkerInfo, WorkerPartitionLocationInfo}
import org.apache.celeborn.common.metrics.MetricsSystem
-import org.apache.celeborn.common.metrics.source.{JVMCPUSource, JVMSource, RPCSource}
+import org.apache.celeborn.common.metrics.source.{JVMCPUSource, JVMSource}
import org.apache.celeborn.common.network.TransportContext
import org.apache.celeborn.common.network.server.ChannelsLimiter
import org.apache.celeborn.common.network.server.memory.MemoryManager
@@ -81,10 +81,8 @@ private[celeborn] class Worker(
conf.workerPushPort != 0 && conf.workerReplicatePort != 0),
"If enable graceful shutdown, the worker should use stable server port.")
- val rpcSource = new RPCSource(conf, MetricsSystem.ROLE_WORKER)
val workerSource = new WorkerSource(conf)
metricsSystem.registerSource(workerSource)
- metricsSystem.registerSource(rpcSource)
metricsSystem.registerSource(new JVMSource(conf, MetricsSystem.ROLE_WORKER))
metricsSystem.registerSource(new JVMCPUSource(conf, MetricsSystem.ROLE_WORKER))
@@ -118,7 +116,7 @@ private[celeborn] class Worker(
}
var controller = new Controller(rpcEnv, conf, metricsSystem)
- rpcEnv.setupEndpoint(RpcNameConstants.WORKER_EP, controller, Some(rpcSource))
+ rpcEnv.setupEndpoint(RpcNameConstants.WORKER_EP, controller)
val pushDataHandler = new PushDataHandler()
val (pushServer, pushClientFactory) = {
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
index ab8b9d026..84c1e1b28 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/FileWriterSuiteJ.java
@@ -57,8 +57,6 @@ import org.slf4j.LoggerFactory;
import org.apache.celeborn.common.CelebornConf;
import org.apache.celeborn.common.identity.UserIdentifier;
import org.apache.celeborn.common.meta.FileInfo;
-import org.apache.celeborn.common.metrics.MetricsSystem;
-import org.apache.celeborn.common.metrics.source.RPCSource;
import org.apache.celeborn.common.network.TransportContext;
import org.apache.celeborn.common.network.buffer.ManagedBuffer;
import org.apache.celeborn.common.network.client.ChunkReceivedCallback;
@@ -140,11 +138,6 @@ public class FileWriterSuiteJ {
return source;
}
- @Override
- public RPCSource rpcSource() {
- return new RPCSource(CONF, MetricsSystem.ROLE_WORKER());
- }
-
@Override
public boolean checkRegistered() {
return true;