You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by re...@apache.org on 2023/03/29 02:23:31 UTC
[incubator-celeborn] 16/42: [CELEBORN-409] Add PartitionLocationInfo to worker's http request (#1335)
This is an automated email from the ASF dual-hosted git repository.
rexxiong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
commit 59aab4eedc7a1b92a401371c8bbb1183724b47be
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Mon Mar 13 17:02:28 2023 +0800
[CELEBORN-409] Add PartitionLocationInfo to worker's http request (#1335)
---
.../main/scala/org/apache/celeborn/service/deploy/master/Master.scala | 2 ++
.../main/scala/org/apache/celeborn/server/common/HttpService.scala | 2 ++
.../org/apache/celeborn/server/common/http/HttpRequestHandler.scala | 2 ++
.../main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala | 4 ++++
4 files changed, 10 insertions(+)
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
index 5367d9482..5c0913cc3 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/Master.scala
@@ -773,6 +773,8 @@ private[celeborn] class Master(
statusSystem.appDiskUsageMetric.summary
}
+ override def listPartitionLocationInfo: String = throw new UnsupportedOperationException()
+
private def requestGetWorkerInfos(endpoint: RpcEndpointRef): GetWorkerInfosResponse = {
try {
if (endpoint != null) {
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
index b612460df..821d8e9fa 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/HttpService.scala
@@ -41,6 +41,8 @@ abstract class HttpService extends Service with Logging {
def listTopDiskUseApps: String
+ def listPartitionLocationInfo: String
+
def startHttpServer(): Unit = {
val handlers =
if (metricsSystem.running) {
diff --git a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
index 6e2fe0b0c..8c0373f19 100644
--- a/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
+++ b/service/src/main/scala/org/apache/celeborn/server/common/http/HttpRequestHandler.scala
@@ -78,6 +78,8 @@ class HttpRequestHandler(
service.getShuffleList
case "/listTopDiskUsedApps" =>
service.listTopDiskUseApps
+ case "/listPartitionLocationInfo" if service.serviceName == Service.WORKER =>
+ service.listPartitionLocationInfo
case _ => INVALID
}
}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index b13dc5faf..7f0e8de49 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -471,6 +471,10 @@ private[celeborn] class Worker(
stringBuilder.toString()
}
+ override def listPartitionLocationInfo: String = {
+ partitionLocationInfo.toString
+ }
+
@VisibleForTesting
def isRegistered(): Boolean = {
registered.get()