You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/06/26 14:30:50 UTC
[incubator-celeborn] branch branch-0.3 updated: [CELEBORN-715] Change master URL schema from rss to celeborn
This is an automated email from the ASF dual-hosted git repository.
zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/branch-0.3 by this push:
new 4cf7f603a [CELEBORN-715] Change master URL schema from rss to celeborn
4cf7f603a is described below
commit 4cf7f603aba38e469f1bd68fe10473f30de2fef0
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Mon Jun 26 22:30:20 2023 +0800
[CELEBORN-715] Change master URL schema from rss to celeborn
### What changes were proposed in this pull request?
Change Celeborn Master URL from `rss://<host>:<port>` to `celeborn://<host>:<port>`
### Why are the changes needed?
Respect the project name.
### Does this PR introduce _any_ user-facing change?
Yes, migration guide is updated accordingly.
### How was this patch tested?
Pass GA.
Closes #1624 from pan3793/CELEBORN-715.
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
(cherry picked from commit 2b82194ce0e37f38d5896577757181fac9921277)
Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
.../apache/celeborn/common/rpc/RpcAddress.scala | 12 ++--
.../celeborn/common/rpc/RpcEndpointAddress.scala | 30 ++------
.../org/apache/celeborn/common/util/Utils.scala | 82 +++++++++-------------
.../apache/celeborn/common/util/UtilsSuite.scala | 2 +-
docs/migration.md | 3 +
.../service/deploy/master/MasterArguments.scala | 2 +-
.../tests/spark/RetryCommitFilesTest.scala | 2 +-
.../celeborn/tests/spark/SparkTestBase.scala | 4 +-
toolkit/reg.sh | 2 +-
.../celeborn/service/deploy/worker/Worker.scala | 4 +-
.../service/deploy/worker/WorkerArguments.scala | 4 +-
.../worker/storage/PartitionFilesSorterSuiteJ.java | 2 +-
.../service/deploy/MiniClusterFeature.scala | 2 +-
.../service/deploy/cluster/ReadWriteTestBase.scala | 4 +-
14 files changed, 63 insertions(+), 92 deletions(-)
diff --git a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcAddress.scala b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcAddress.scala
index baf13a79a..e4ac1274e 100644
--- a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcAddress.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcAddress.scala
@@ -24,10 +24,10 @@ import org.apache.celeborn.common.util.Utils
*/
case class RpcAddress(host: String, port: Int) {
- def hostPort: String = host + ":" + port
+ def hostPort: String = s"$host:$port"
- /** Returns a string in the form of "rss://host:port". */
- def toRssURL: String = "rss://" + hostPort
+ /** Returns a string in the form of "celeborn://host:port". */
+ def toCelebornURL: String = s"celeborn://$hostPort"
override def toString: String = hostPort
}
@@ -45,9 +45,9 @@ private[celeborn] object RpcAddress {
RpcAddress(uriObj.getHost, uriObj.getPort)
}
- /** Returns the [[RpcAddress]] encoded in the form of "rss://host:port" */
- def fromRssURL(essUrl: String): RpcAddress = {
- val (host, port) = Utils.extractHostPortFromRssUrl(essUrl)
+ /** Returns the [[RpcAddress]] encoded in the form of "celeborn://host:port" */
+ def fromCelebornURL(celebornUrl: String): RpcAddress = {
+ val (host, port) = Utils.extractHostPortFromCelebornUrl(celebornUrl)
RpcAddress(host, port)
}
}
diff --git a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEndpointAddress.scala b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEndpointAddress.scala
index f287b06e8..0bd584882 100644
--- a/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEndpointAddress.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/rpc/RpcEndpointAddress.scala
@@ -18,6 +18,7 @@
package org.apache.celeborn.common.rpc
import org.apache.celeborn.common.exception.CelebornException
+import org.apache.celeborn.common.util.Utils
/**
* An address identifier for an RPC endpoint.
@@ -37,11 +38,11 @@ private[celeborn] case class RpcEndpointAddress(rpcAddress: RpcAddress, name: St
this(RpcAddress(host, port), name)
}
- override val toString =
+ override val toString: String =
if (rpcAddress != null) {
- s"rss://$name@${rpcAddress.host}:${rpcAddress.port}"
+ s"celeborn://$name@${rpcAddress.host}:${rpcAddress.port}"
} else {
- s"rss-client://$name"
+ s"celeborn-client://$name"
}
}
@@ -51,25 +52,8 @@ private[celeborn] object RpcEndpointAddress {
new RpcEndpointAddress(host, port, name)
}
- def apply(essUrl: String): RpcEndpointAddress = {
- try {
- val uri = new java.net.URI(essUrl)
- val host = uri.getHost
- val port = uri.getPort
- val name = uri.getUserInfo
- if (uri.getScheme != "rss" ||
- host == null ||
- port < 0 ||
- name == null ||
- (uri.getPath != null && !uri.getPath.isEmpty) || // uri.getPath returns "" instead of null
- uri.getFragment != null ||
- uri.getQuery != null) {
- throw new CelebornException("Invalid RSS URL: " + essUrl)
- }
- new RpcEndpointAddress(host, port, name)
- } catch {
- case e: java.net.URISyntaxException =>
- throw new CelebornException("Invalid RSS URL: " + essUrl, e)
- }
+ def apply(celebornUrl: String): RpcEndpointAddress = {
+ val (host, port, name) = Utils.extractHostPortNameFromCelebornUrl(celebornUrl)
+ new RpcEndpointAddress(host, port, name)
}
}
diff --git a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
index dd5dae67e..d7fb75073 100644
--- a/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
+++ b/common/src/main/scala/org/apache/celeborn/common/util/Utils.scala
@@ -29,6 +29,7 @@ import java.util
import java.util.{Locale, Properties, Random, UUID}
import java.util.concurrent.{Callable, ThreadPoolExecutor, TimeoutException, TimeUnit}
+import scala.annotation.tailrec
import scala.collection.JavaConverters._
import scala.reflect.ClassTag
import scala.util.Try
@@ -52,8 +53,6 @@ import org.apache.celeborn.common.protocol.message.ControlMessages.WorkerResourc
object Utils extends Logging {
- def createDateFormat: SimpleDateFormat = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US)
-
def stringToSeq(str: String): Seq[String] = {
str.split(",").map(_.trim()).filter(_.nonEmpty)
}
@@ -147,24 +146,47 @@ object Utils extends Logging {
}
@throws(classOf[CelebornException])
- def extractHostPortFromRssUrl(essUrl: String): (String, Int) = {
+ def extractHostPortFromCelebornUrl(celebornUrl: String): (String, Int) = {
try {
- val uri = new java.net.URI(essUrl)
+ val uri = new java.net.URI(celebornUrl)
val host = uri.getHost
val port = uri.getPort
- if (uri.getScheme != "rss" ||
+ if (uri.getScheme != "celeborn" ||
host == null ||
port < 0 ||
- (uri.getPath != null && !uri.getPath.isEmpty) || // uri.getPath returns "" instead of null
+ (uri.getPath != null && uri.getPath.nonEmpty) || // uri.getPath returns "" instead of null
uri.getFragment != null ||
uri.getQuery != null ||
uri.getUserInfo != null) {
- throw new CelebornException("Invalid master URL: " + essUrl)
+ throw new CelebornException(s"Invalid master URL: $celebornUrl")
}
(host, port)
} catch {
case e: java.net.URISyntaxException =>
- throw new CelebornException("Invalid master URL: " + essUrl, e)
+ throw new CelebornException(s"Invalid master URL: $celebornUrl", e)
+ }
+ }
+
+ @throws(classOf[CelebornException])
+ def extractHostPortNameFromCelebornUrl(celebornUrl: String): (String, Int, String) = {
+ try {
+ val uri = new java.net.URI(celebornUrl)
+ val host = uri.getHost
+ val port = uri.getPort
+ val name = uri.getUserInfo
+ if (uri.getScheme != "celeborn" ||
+ host == null ||
+ port < 0 ||
+ name == null ||
+ (uri.getPath != null && uri.getPath.nonEmpty) || // uri.getPath returns "" instead of null
+ uri.getFragment != null ||
+ uri.getQuery != null) {
+ throw new CelebornException(s"Invalid Celeborn URL: $celebornUrl")
+ }
+ (host, port, name)
+ } catch {
+ case e: java.net.URISyntaxException =>
+ throw new CelebornException(s"Invalid Celeborn URL: $celebornUrl", e)
}
}
@@ -181,7 +203,7 @@ object Utils extends Logging {
}
}
- def tryLogNonFatalError(block: => Unit) {
+ def tryLogNonFatalError(block: => Unit): Unit = {
try {
block
} catch {
@@ -190,7 +212,7 @@ object Utils extends Logging {
}
}
- def tryOrExit(block: => Unit) {
+ def tryOrExit(block: => Unit): Unit = {
try {
block
} catch {
@@ -199,41 +221,6 @@ object Utils extends Logging {
}
}
- def tryWithSafeFinallyAndFailureCallbacks[T](block: => T)(
- catchBlock: => Unit = (),
- finallyBlock: => Unit = ()): T = {
- var originalThrowable: Throwable = null
- try {
- block
- } catch {
- case cause: Throwable =>
- // Purposefully not using NonFatal, because even fatal exceptions
- // we don't want to have our finallyBlock suppress
- originalThrowable = cause
- try {
- logError("Aborting task", originalThrowable)
- // TaskContext.get().markTaskFailed(originalThrowable)
- catchBlock
- } catch {
- case t: Throwable =>
- if (originalThrowable != t) {
- originalThrowable.addSuppressed(t)
- logWarning(s"Suppressing exception in catch: ${t.getMessage}", t)
- }
- }
- throw originalThrowable
- } finally {
- try {
- finallyBlock
- } catch {
- case t: Throwable if (originalThrowable != null && originalThrowable != t) =>
- originalThrowable.addSuppressed(t)
- logWarning(s"Suppressing exception in finally: ${t.getMessage}", t)
- throw originalThrowable
- }
- }
- }
-
def startServiceOnPort[T](
startPort: Int,
startService: Int => (T, Int),
@@ -298,6 +285,7 @@ object Utils extends Logging {
(base + offset - 1024) % (65536 - 1024) + 1024
}
+ @tailrec
def isBindCollision(exception: Throwable): Boolean = {
exception match {
case e: BindException =>
@@ -305,8 +293,6 @@ object Utils extends Logging {
return true
}
isBindCollision(e.getCause)
- // case e: MultiException =>
- // e.getThrowables.asScala.exists(isBindCollision)
case e: NativeIoException =>
(e.getMessage != null && e.getMessage.startsWith("bind() failed: ")) ||
isBindCollision(e.getCause)
@@ -529,7 +515,7 @@ object Utils extends Logging {
new File(clazz.getProtectionDomain.getCodeSource.getLocation.toURI).getPath
}
- def loadDefaultRssProperties(conf: CelebornConf, filePath: String = null): String = {
+ def loadDefaultCelebornProperties(conf: CelebornConf, filePath: String = null): String = {
val path = Option(filePath).getOrElse(getDefaultPropertiesFile())
Option(path).foreach { confFile =>
getPropertiesFromFile(confFile).filter { case (k, v) =>
diff --git a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
index 23f9ce368..6ae011169 100644
--- a/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
+++ b/common/src/test/scala/org/apache/celeborn/common/util/UtilsSuite.scala
@@ -62,7 +62,7 @@ class UtilsSuite extends CelebornFunSuite {
test("extractHostPortFromRssUrl") {
val target = ("abc", 123)
- val result = Utils.extractHostPortFromRssUrl("rss://abc:123")
+ val result = Utils.extractHostPortFromCelebornUrl("celeborn://abc:123")
assert(target.equals(result))
}
diff --git a/docs/migration.md b/docs/migration.md
index f5e2b0e50..cd97080a8 100644
--- a/docs/migration.md
+++ b/docs/migration.md
@@ -34,6 +34,9 @@ license: |
- Since 0.3.0, environment variables `CELEBORN_MASTER_HOST` and `CELEBORN_MASTER_PORT` are removed.
Instead `CELEBORN_LOCAL_HOSTNAME` works on both master and worker, which takes high priority than configurations defined in properties file.
+ - Since 0.3.0, the Celeborn Master URL schema is changed from `rss://` to `celeborn://`, for users who start Worker by
+ `sbin/start-worker.sh rss://<master-host>:<master-port>`, should migrate to `sbin/start-worker.sh celeborn://<master-host>:<master-port>`.
+
- When using 0.2.1 as client side and 0.3.0 as server side, you may see the following Exception in LifecycleManger's
log. You can safely ignore the log, it's caused by the behavior change when Master receives heartbeat from Application.
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
index 1e7421f94..195647463 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
@@ -37,7 +37,7 @@ class MasterArguments(args: Array[String], conf: CelebornConf) {
_host = _host.orElse(sys.env.get("CELEBORN_LOCAL_HOSTNAME"))
// 3rd read from configuration file
- _propertiesFile = Some(Utils.loadDefaultRssProperties(conf, _propertiesFile.orNull))
+ _propertiesFile = Some(Utils.loadDefaultCelebornProperties(conf, _propertiesFile.orNull))
if (conf.haEnabled) {
val clusterInfo = MasterClusterInfo.loadHAConfig(conf)
val localNode = clusterInfo.localNode
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
index f1fba73b9..01255812c 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/RetryCommitFilesTest.scala
@@ -31,7 +31,7 @@ class RetryCommitFilesTest extends AnyFunSuite
with BeforeAndAfterEach {
override def beforeAll(): Unit = {
- logInfo("test initialized , setup rss mini cluster")
+ logInfo("test initialized , setup Celeborn mini cluster")
val workerConf = Map(
"celeborn.test.retryCommitFiles" -> s"true")
setUpMiniCluster(masterConfs = null, workerConfs = workerConf)
diff --git a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
index 69da6a731..cd8398abe 100644
--- a/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
+++ b/tests/spark-it/src/test/scala/org/apache/celeborn/tests/spark/SparkTestBase.scala
@@ -37,12 +37,12 @@ trait SparkTestBase extends AnyFunSuite
.map(v => (v.toUpper, Random.nextInt(12) + 1))
override def beforeAll(): Unit = {
- logInfo("test initialized , setup rss mini cluster")
+ logInfo("test initialized , setup Celeborn mini cluster")
setUpMiniCluster(workerNum = 5)
}
override def afterAll(): Unit = {
- logInfo("all test complete , stop rss mini cluster")
+ logInfo("all test complete , stop Celeborn mini cluster")
shutdownMiniCluster()
}
diff --git a/toolkit/reg.sh b/toolkit/reg.sh
index 250c40451..54d3300ab 100644
--- a/toolkit/reg.sh
+++ b/toolkit/reg.sh
@@ -335,7 +335,7 @@ function updateCeleborn() {
for host in "${REG_HOSTS[@]}"; do
echo -e "start worker on ${host} \n"
- ssh ${host} "export CELEBORN_CONF_DIR=/home/hadoop/conf ; /home/hadoop/${CELEBORN_DIST}/sbin/start-worker.sh rss://master-1-1:9097"
+ ssh ${host} "export CELEBORN_CONF_DIR=/home/hadoop/conf ; /home/hadoop/${CELEBORN_DIST}/sbin/start-worker.sh celeborn://master-1-1:9097"
done
}
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
index 4c360bc6d..303c30342 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/Worker.scala
@@ -589,9 +589,7 @@ private[deploy] object Worker extends Logging {
// starting the Worker, we should set it in the parameters and automatically calculate what the
// address of the Master should be used in the end.
workerArgs.master.foreach { master =>
- conf.set(
- MASTER_ENDPOINTS.key,
- RpcAddress.fromRssURL(master).toString.replace("rss://", ""))
+ conf.set(MASTER_ENDPOINTS.key, RpcAddress.fromCelebornURL(master).hostPort)
}
val worker = new Worker(conf, workerArgs)
diff --git a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerArguments.scala b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerArguments.scala
index 674ae44da..32231f8f5 100644
--- a/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerArguments.scala
+++ b/worker/src/main/scala/org/apache/celeborn/service/deploy/worker/WorkerArguments.scala
@@ -33,7 +33,7 @@ class WorkerArguments(args: Array[String], conf: CelebornConf) {
// 1st parse from cli args
parse(args.toList)
// 2nd read from configuration file
- _propertiesFile = Some(Utils.loadDefaultRssProperties(conf, _propertiesFile.orNull))
+ _propertiesFile = Some(Utils.loadDefaultCelebornProperties(conf, _propertiesFile.orNull))
_host = _host.orElse(Some(Utils.localHostName))
_port = _port.orElse(Some(conf.workerRpcPort))
@@ -79,7 +79,7 @@ class WorkerArguments(args: Array[String], conf: CelebornConf) {
System.err.println(
"""Usage: Worker [options] <master>
|
- |<master> must be a URL of the form rss://hostname:port
+ |<master> must be a URL of the form celeborn://hostname:port
|Options:
| -h HOST, --host HOST Hostname to listen on
| -p PORT, --port PORT Port to listen on (default: random)
diff --git a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
index 6b7b6a084..be73f6c2e 100644
--- a/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
+++ b/worker/src/test/java/org/apache/celeborn/service/deploy/worker/storage/PartitionFilesSorterSuiteJ.java
@@ -59,7 +59,7 @@ public class PartitionFilesSorterSuiteJ {
public void prepare(boolean largefile) throws IOException {
byte[] batchHeader = new byte[16];
Random random = new Random();
- shuffleFile = File.createTempFile("RSS", "sort-suite");
+ shuffleFile = File.createTempFile("Celeborn", "sort-suite");
originFileName = shuffleFile.getAbsolutePath();
fileInfo = new FileInfo(shuffleFile, userIdentifier);
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
index 1ddb3a9cc..683333ba0 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
@@ -78,7 +78,7 @@ trait MiniClusterFeature extends Logging {
if (map != null) {
map.foreach(m => conf.set(m._1, m._2))
}
- logInfo("rss conf created")
+ logInfo("celeborn conf created")
val workerArguments = new WorkerArguments(Array(), conf)
logInfo("worker argument created")
diff --git a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
index 0b53aef41..69fe71906 100644
--- a/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
+++ b/worker/src/test/scala/org/apache/celeborn/service/deploy/cluster/ReadWriteTestBase.scala
@@ -42,12 +42,12 @@ trait ReadWriteTestBase extends AnyFunSuite
"celeborn.master.port" -> masterPort.toString)
val workerConf = Map(
"celeborn.master.endpoints" -> s"localhost:$masterPort")
- logInfo("test initialized , setup rss mini cluster")
+ logInfo("test initialized , setup Celeborn mini cluster")
setUpMiniCluster(masterConf, workerConf)
}
override def afterAll(): Unit = {
- logInfo("all test complete , stop rss mini cluster")
+ logInfo("all test complete , stop Celeborn mini cluster")
shutdownMiniCluster()
}