You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/03/19 06:05:10 UTC
git commit: Bundle tachyon: SPARK-1269
Repository: spark
Updated Branches:
refs/heads/master cc2655a23 -> a18ea00f3
Bundle tachyon: SPARK-1269
This should all work as expected with the current version of the tachyon tarball (0.4.1)
Author: Nick Lanham <ni...@afternight.org>
Closes #137 from nicklan/bundle-tachyon and squashes the following commits:
2eee15b [Nick Lanham] Put back in exec, start tachyon first
738ba23 [Nick Lanham] Move tachyon out of sbin
f2f9bc6 [Nick Lanham] More checks for tachyon script
111e8e1 [Nick Lanham] Only try tachyon operations if tachyon script exists
0561574 [Nick Lanham] Copy over web resources so web interface can run
4dc9809 [Nick Lanham] Update to tachyon 0.4.1
0a1a20c [Nick Lanham] Add scripts using tachyon tarball
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a18ea00f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a18ea00f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a18ea00f
Branch: refs/heads/master
Commit: a18ea00f3af0fa4c6b2c59933e22b6c9f0f636c8
Parents: cc2655a
Author: Nick Lanham <ni...@afternight.org>
Authored: Tue Mar 18 22:04:57 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Mar 18 22:04:57 2014 -0700
----------------------------------------------------------------------
make-distribution.sh | 32 ++++++++++++++++++++++++++++++++
sbin/start-all.sh | 15 +++++++++++++--
sbin/start-master.sh | 21 +++++++++++++++++++++
sbin/start-slaves.sh | 23 +++++++++++++++++++++++
sbin/stop-master.sh | 4 ++++
sbin/stop-slaves.sh | 5 +++++
6 files changed, 98 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index e6b5956..6bc6819 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -58,6 +58,7 @@ echo "Version is ${VERSION}"
# Initialize defaults
SPARK_HADOOP_VERSION=1.0.4
SPARK_YARN=false
+SPARK_TACHYON=false
MAKE_TGZ=false
# Parse arguments
@@ -70,6 +71,9 @@ while (( "$#" )); do
--with-yarn)
SPARK_YARN=true
;;
+ --with-tachyon)
+ SPARK_TACHYON=true
+ ;;
--tgz)
MAKE_TGZ=true
;;
@@ -90,6 +94,12 @@ else
echo "YARN disabled"
fi
+if [ "$SPARK_TACHYON" == "true" ]; then
+ echo "Tachyon Enabled"
+else
+ echo "Tachyon Disabled"
+fi
+
# Build fat JAR
export SPARK_HADOOP_VERSION
export SPARK_YARN
@@ -113,6 +123,28 @@ cp -r "$FWDIR/python" "$DISTDIR"
cp -r "$FWDIR/sbin" "$DISTDIR"
+# Download and copy in tachyon, if requested
+if [ "$SPARK_TACHYON" == "true" ]; then
+ TACHYON_VERSION="0.4.1"
+ TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/tachyon-${TACHYON_VERSION}-bin.tar.gz"
+
+ TMPD=`mktemp -d`
+
+ pushd $TMPD > /dev/null
+ echo "Fetchting tachyon tgz"
+ wget "$TACHYON_URL"
+
+ tar xf "tachyon-${TACHYON_VERSION}-bin.tar.gz"
+ cp "tachyon-${TACHYON_VERSION}/target/tachyon-${TACHYON_VERSION}-jar-with-dependencies.jar" "$DISTDIR/jars"
+ mkdir -p "$DISTDIR/tachyon/src/main/java/tachyon/web"
+ cp -r "tachyon-${TACHYON_VERSION}"/{bin,conf,libexec} "$DISTDIR/tachyon"
+ cp -r "tachyon-${TACHYON_VERSION}"/src/main/java/tachyon/web/resources "$DISTDIR/tachyon/src/main/java/tachyon/web"
+ sed -i "s|export TACHYON_JAR=\$TACHYON_HOME/target/\(.*\)|# This is set for spark's make-distribution\n export TACHYON_JAR=\$TACHYON_HOME/../../jars/\1|" "$DISTDIR/tachyon/libexec/tachyon-config.sh"
+
+ popd > /dev/null
+ rm -rf $TMPD
+fi
+
if [ "$MAKE_TGZ" == "true" ]; then
TARDIR="$FWDIR/spark-$VERSION"
cp -r "$DISTDIR" "$TARDIR"
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/sbin/start-all.sh
----------------------------------------------------------------------
diff --git a/sbin/start-all.sh b/sbin/start-all.sh
index 2daf49d..5c89ab4 100755
--- a/sbin/start-all.sh
+++ b/sbin/start-all.sh
@@ -24,11 +24,22 @@
sbin=`dirname "$0"`
sbin=`cd "$sbin"; pwd`
+TACHYON_STR=""
+
+while (( "$#" )); do
+case $1 in
+ --with-tachyon)
+ TACHYON_STR="--with-tachyon"
+ ;;
+ esac
+shift
+done
+
# Load the Spark configuration
. "$sbin/spark-config.sh"
# Start Master
-"$sbin"/start-master.sh
+"$sbin"/start-master.sh $TACHYON_STR
# Start Workers
-"$sbin"/start-slaves.sh
+"$sbin"/start-slaves.sh $TACHYON_STR
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/sbin/start-master.sh
----------------------------------------------------------------------
diff --git a/sbin/start-master.sh b/sbin/start-master.sh
index ec3dfdb..03a3428 100755
--- a/sbin/start-master.sh
+++ b/sbin/start-master.sh
@@ -22,6 +22,21 @@
sbin=`dirname "$0"`
sbin=`cd "$sbin"; pwd`
+START_TACHYON=false
+
+while (( "$#" )); do
+case $1 in
+ --with-tachyon)
+ if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then
+ echo "Error: --with-tachyon specified, but tachyon not found."
+ exit -1
+ fi
+ START_TACHYON=true
+ ;;
+ esac
+shift
+done
+
. "$sbin/spark-config.sh"
if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then
@@ -41,3 +56,9 @@ if [ "$SPARK_MASTER_WEBUI_PORT" = "" ]; then
fi
"$sbin"/spark-daemon.sh start org.apache.spark.deploy.master.Master 1 --ip $SPARK_MASTER_IP --port $SPARK_MASTER_PORT --webui-port $SPARK_MASTER_WEBUI_PORT
+
+if [ "$START_TACHYON" == "true" ]; then
+ "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP
+ "$sbin"/../tachyon/bin/tachyon format -s
+ "$sbin"/../tachyon/bin/tachyon-start.sh master
+fi
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/sbin/start-slaves.sh
----------------------------------------------------------------------
diff --git a/sbin/start-slaves.sh b/sbin/start-slaves.sh
index fd5cdeb..da641cf 100755
--- a/sbin/start-slaves.sh
+++ b/sbin/start-slaves.sh
@@ -20,6 +20,22 @@
sbin=`dirname "$0"`
sbin=`cd "$sbin"; pwd`
+
+START_TACHYON=false
+
+while (( "$#" )); do
+case $1 in
+ --with-tachyon)
+ if [ ! -e "$sbin"/../tachyon/bin/tachyon ]; then
+ echo "Error: --with-tachyon specified, but tachyon not found."
+ exit -1
+ fi
+ START_TACHYON=true
+ ;;
+ esac
+shift
+done
+
. "$sbin/spark-config.sh"
if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then
@@ -35,6 +51,13 @@ if [ "$SPARK_MASTER_IP" = "" ]; then
SPARK_MASTER_IP=`hostname`
fi
+if [ "$START_TACHYON" == "true" ]; then
+ "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon bootstrap-conf $SPARK_MASTER_IP
+
+ # set -t so we can call sudo
+ SPARK_SSH_OPTS="-o StrictHostKeyChecking=no -t" "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/../tachyon/bin/tachyon-start.sh" worker SudoMount \; sleep 1
+fi
+
# Launch the slaves
if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
exec "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin/start-slave.sh" 1 spark://$SPARK_MASTER_IP:$SPARK_MASTER_PORT
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/sbin/stop-master.sh
----------------------------------------------------------------------
diff --git a/sbin/stop-master.sh b/sbin/stop-master.sh
index 2adabd4..b6bdaa4 100755
--- a/sbin/stop-master.sh
+++ b/sbin/stop-master.sh
@@ -25,3 +25,7 @@ sbin=`cd "$sbin"; pwd`
. "$sbin/spark-config.sh"
"$sbin"/spark-daemon.sh stop org.apache.spark.deploy.master.Master 1
+
+if [ -e "$sbin"/../tachyon/bin/tachyon ]; then
+ "$sbin"/../tachyon/bin/tachyon killAll tachyon.master.Master
+fi
http://git-wip-us.apache.org/repos/asf/spark/blob/a18ea00f/sbin/stop-slaves.sh
----------------------------------------------------------------------
diff --git a/sbin/stop-slaves.sh b/sbin/stop-slaves.sh
index eb803b4..6bf393c 100755
--- a/sbin/stop-slaves.sh
+++ b/sbin/stop-slaves.sh
@@ -26,6 +26,11 @@ if [ -f "${SPARK_CONF_DIR}/spark-env.sh" ]; then
. "${SPARK_CONF_DIR}/spark-env.sh"
fi
+# do before the below calls as they exec
+if [ -e "$sbin"/../tachyon/bin/tachyon ]; then
+ "$sbin/slaves.sh" cd "$SPARK_HOME" \; "$sbin"/../tachyon/bin/tachyon killAll tachyon.worker.Worker
+fi
+
if [ "$SPARK_WORKER_INSTANCES" = "" ]; then
"$sbin"/spark-daemons.sh stop org.apache.spark.deploy.worker.Worker 1
else