You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by zh...@apache.org on 2023/06/25 08:01:20 UTC

[incubator-celeborn] branch branch-0.3 updated: [CELEBORN-707][MASTER] Remove env CELEBORN_MASTER_HOST and CELEBORN_MASTER_PORT

This is an automated email from the ASF dual-hosted git repository.

zhouky pushed a commit to branch branch-0.3
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git


The following commit(s) were added to refs/heads/branch-0.3 by this push:
     new 0723e30f0 [CELEBORN-707][MASTER] Remove env CELEBORN_MASTER_HOST and CELEBORN_MASTER_PORT
0723e30f0 is described below

commit 0723e30f011d0fa4c0df64339f7576456e026205
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Sun Jun 25 16:00:59 2023 +0800

    [CELEBORN-707][MASTER] Remove env CELEBORN_MASTER_HOST and CELEBORN_MASTER_PORT
    
    ### What changes were proposed in this pull request?
    
    Remove environment variables `CELEBORN_MASTER_HOST` and `CELEBORN_MASTER_PORT`, and makes `CELEBORN_LOCAL_HOSTNAME` takes effect on both master and worker.
    
    ### Why are the changes needed?
    
    There are many different ways to configure the master/worker host and port, which makes the thing complex and inconsistent.
    
    After this change,
    
    #### master
    
    1. cli args `--host` `--port` takes the highest priority
    2. then lookup env `CELEBORN_LOCAL_HOSTNAME`
    3. things are different when HA enabled and disabled
      3.1. when HA is disabled, lookup configurations `celeborn.master.host` and `celeborn.master.port`
      3.2. when HA is enabled, each node needs to know the whole cluster info,
         ```
         celeborn.master.ha.node.1.host clb-1
         celeborn.master.ha.node.1.port 9097
         celeborn.master.ha.node.2.host clb-2
         celeborn.master.ha.node.2.port 9097
         celeborn.master.ha.node.3.host clb-3
         celeborn.master.ha.node.3.port 9097
         ```
         in addition, `celeborn.master.ha.node.id=1` can be used to indicate the node id, otherwise, the master will try to bind each host to match the node id.
    
    #### worker
    
    1. cli args `--host` `--port` takes the highest priority
    2. then lookup env `CELEBORN_LOCAL_HOSTNAME`
    
    things are simple than the master case because each worker is not required to know others.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes.
    
    ### How was this patch tested?
    
    UT.
    
    Closes #1616 from pan3793/CELEBORN-707.
    
    Authored-by: Cheng Pan <ch...@apache.org>
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
    (cherry picked from commit ac84d64d51293f0935085a6f0b7a2d913eb49465)
    Signed-off-by: zky.zhoukeyong <zk...@alibaba-inc.com>
---
 docs/migration.md                                            |  3 +++
 .../celeborn/service/deploy/master/MasterArguments.scala     |  3 +--
 .../service/deploy/master/MasterArgumentsSuite.scala         | 12 ++++++------
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/migration.md b/docs/migration.md
index baa1a28fd..f5e2b0e50 100644
--- a/docs/migration.md
+++ b/docs/migration.md
@@ -31,6 +31,9 @@ license: |
  - Since 0.3.0, configuration namespace `celeborn.ha.master` is deprecated, and will be removed in the future versions.
    All configurations `celeborn.ha.master.*` should migrate to `celeborn.master.ha.*`.
 
+ - Since 0.3.0, environment variables `CELEBORN_MASTER_HOST` and `CELEBORN_MASTER_PORT` are removed.
+   Instead `CELEBORN_LOCAL_HOSTNAME` works on both master and worker, which takes high priority than configurations defined in properties file.
+
  - When using 0.2.1 as client side and 0.3.0 as server side, you may see the following Exception in LifecycleManger's
    log. You can safely ignore the log, it's caused by the behavior change when Master receives heartbeat from Application.
 
diff --git a/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala b/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
index 331e9a8b3..1e7421f94 100644
--- a/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
+++ b/master/src/main/scala/org/apache/celeborn/service/deploy/master/MasterArguments.scala
@@ -34,8 +34,7 @@ class MasterArguments(args: Array[String], conf: CelebornConf) {
   parse(args.toList)
 
   // 2nd parse from environment variables
-  _host = _host.orElse(sys.env.get("CELEBORN_MASTER_HOST"))
-  _port = _port.orElse(sys.env.get("CELEBORN_MASTER_PORT").map(_.toInt))
+  _host = _host.orElse(sys.env.get("CELEBORN_LOCAL_HOSTNAME"))
 
   // 3rd read from configuration file
   _propertiesFile = Some(Utils.loadDefaultRssProperties(conf, _propertiesFile.orNull))
diff --git a/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterArgumentsSuite.scala b/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterArgumentsSuite.scala
index c32d6fe32..3e17e1ab7 100644
--- a/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterArgumentsSuite.scala
+++ b/master/src/test/scala/org/apache/celeborn/service/deploy/master/MasterArgumentsSuite.scala
@@ -31,8 +31,8 @@ class MasterArgumentsSuite extends AnyFunSuite with Logging {
     val conf1 = new CelebornConf()
 
     val arguments1 = new MasterArguments(args1, conf1)
-    assert(arguments1.host.equals(Utils.localHostName))
-    assert(arguments1.port == 9097)
+    assert(arguments1.host === sys.env.getOrElse("CELEBORN_LOCAL_HOSTNAME", Utils.localHostName))
+    assert(arguments1.port === 9097)
 
     // should use celeborn conf
     val conf2 = new CelebornConf()
@@ -40,14 +40,14 @@ class MasterArgumentsSuite extends AnyFunSuite with Logging {
     conf2.set(MASTER_PORT, 19097)
 
     val arguments2 = new MasterArguments(args1, conf2)
-    assert(arguments2.host.equals("test-host-1"))
-    assert(arguments2.port == 19097)
+    assert(arguments2.host === sys.env.getOrElse("CELEBORN_LOCAL_HOSTNAME", "test-host-1"))
+    assert(arguments2.port === 19097)
 
     // should use cli args
     val args2 = Array("-h", "test-host-2", "-p", "29097")
 
     val arguments3 = new MasterArguments(args2, conf2)
-    assert(arguments3.host.equals("test-host-2"))
-    assert(arguments3.port == 29097)
+    assert(arguments3.host === "test-host-2")
+    assert(arguments3.port === 29097)
   }
 }