You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sh...@apache.org on 2014/02/23 05:21:35 UTC
git commit: [SPARK-1041] remove dead code in start script,
remind user to set that in spark-env.sh
Repository: incubator-spark
Updated Branches:
refs/heads/master 29ac7ea52 -> 437b62fcb
[SPARK-1041] remove dead code in start script, remind user to set that in spark-env.sh
the lines in start-master.sh and start-slave.sh no longer work
in ec2, the host name has changed, e.g.
ubuntu@ip-172-31-36-93:~$ hostname
ip-172-31-36-93
also, the URL to fetch public DNS name also changed, e.g.
ubuntu@ip-172-31-36-93:~$ wget -q -O - http://instance-data.ec2.internal/latest/meta-data/public-hostname
ubuntu@ip-172-31-36-93:~$ (returns nothing)
since we have spark-ec2 project, we don't need to have such ec2-specific lines here, instead, user only need to set in spark-env.sh
Author: CodingCat <zh...@gmail.com>
Closes #588 from CodingCat/deadcode_in_sbin and squashes the following commits:
e4236e0 [CodingCat] remove dead code in start script, remind user set that in spark-env.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/437b62fc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/437b62fc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/437b62fc
Branch: refs/heads/master
Commit: 437b62fcb03cc84c6581eb667ddbddd817f7f9f0
Parents: 29ac7ea
Author: CodingCat <zh...@gmail.com>
Authored: Sat Feb 22 20:21:15 2014 -0800
Committer: Shivaram Venkataraman <sh...@eecs.berkeley.edu>
Committed: Sat Feb 22 20:21:15 2014 -0800
----------------------------------------------------------------------
conf/spark-env.sh.template | 1 +
sbin/start-master.sh | 9 ---------
sbin/start-slave.sh | 9 ---------
3 files changed, 1 insertion(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/437b62fc/conf/spark-env.sh.template
----------------------------------------------------------------------
diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template
index e2071e2..6432a56 100755
--- a/conf/spark-env.sh.template
+++ b/conf/spark-env.sh.template
@@ -19,3 +19,4 @@
# - SPARK_WORKER_PORT / SPARK_WORKER_WEBUI_PORT
# - SPARK_WORKER_INSTANCES, to set the number of worker processes per node
# - SPARK_WORKER_DIR, to set the working directory of worker processes
+# - SPARK_PUBLIC_DNS, to set the public dns name of the master
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/437b62fc/sbin/start-master.sh
----------------------------------------------------------------------
diff --git a/sbin/start-master.sh b/sbin/start-master.sh
index 3dcf7cc..ec3dfdb 100755
--- a/sbin/start-master.sh
+++ b/sbin/start-master.sh
@@ -40,13 +40,4 @@ if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then
SPARK_MASTER_WEBUI_PORT=8080
fi
-# Set SPARK_PUBLIC_DNS so the master report the correct webUI address to the slaves
-if [ "$SPARK_PUBLIC_DNS" = "" ]; then
- # If we appear to be running on EC2, use the public address by default:
- # NOTE: ec2-metadata is installed on Amazon Linux AMI. Check based on that and hostname
- if command -v ec2-metadata > /dev/null || [[ `hostname` == *ec2.internal ]]; then
- export SPARK_PUBLIC_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/public-hostname`
- fi
-fi
-
"$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/437b62fc/sbin/start-slave.sh
----------------------------------------------------------------------
diff --git a/sbin/start-slave.sh b/sbin/start-slave.sh
index 524be38..b563400 100755
--- a/sbin/start-slave.sh
+++ b/sbin/start-slave.sh
@@ -23,13 +23,4 @@
sbin=`dirname "$0"`
sbin=`cd "$sbin"; pwd`
-# Set SPARK_PUBLIC_DNS so slaves can be linked in master web UI
-if [ "$SPARK_PUBLIC_DNS" = "" ]; then
- # If we appear to be running on EC2, use the public address by default:
- # NOTE: ec2-metadata is installed on Amazon Linux AMI. Check based on that and hostname
- if command -v ec2-metadata > /dev/null || [[ `hostname` == *ec2.internal ]]; then
- export SPARK_PUBLIC_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/public-hostname`
- fi
-fi
-
"$sbin"/spark-daemon.sh start org.apache.spark.deploy.worker.Worker "$@"