You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/07/17 10:02:43 UTC
git commit: SPARK-2526: Simplify options in make-distribution.sh
Repository: spark
Updated Branches:
refs/heads/master 7c23c0dc3 -> d0ea49687
SPARK-2526: Simplify options in make-distribution.sh
Right now we have a bunch of parallel logic in make-distribution.sh
that's just extra work to maintain. We should just pass through
Maven profiles in this case and keep the script simple. See
the JIRA for more details.
Author: Patrick Wendell <pw...@gmail.com>
Closes #1445 from pwendell/make-distribution.sh and squashes the following commits:
f1294ea [Patrick Wendell] Simplify options in make-distribution.sh.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0ea4968
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0ea4968
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0ea4968
Branch: refs/heads/master
Commit: d0ea496877b53f3253e6e5f6d053c7f79c7991a5
Parents: 7c23c0d
Author: Patrick Wendell <pw...@gmail.com>
Authored: Thu Jul 17 01:02:35 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Thu Jul 17 01:02:35 2014 -0700
----------------------------------------------------------------------
dev/create-release/create-release.sh | 9 ++--
make-distribution.sh | 79 ++++++++++---------------------
2 files changed, 30 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d0ea4968/dev/create-release/create-release.sh
----------------------------------------------------------------------
diff --git a/dev/create-release/create-release.sh b/dev/create-release/create-release.sh
index 49bf78f..3883010 100755
--- a/dev/create-release/create-release.sh
+++ b/dev/create-release/create-release.sh
@@ -95,7 +95,7 @@ make_binary_release() {
cp -r spark spark-$RELEASE_VERSION-bin-$NAME
cd spark-$RELEASE_VERSION-bin-$NAME
- ./make-distribution.sh $FLAGS --name $NAME --tgz
+ ./make-distribution.sh --name $NAME --tgz $FLAGS
cd ..
cp spark-$RELEASE_VERSION-bin-$NAME/spark-$RELEASE_VERSION-bin-$NAME.tgz .
rm -rf spark-$RELEASE_VERSION-bin-$NAME
@@ -111,9 +111,10 @@ make_binary_release() {
spark-$RELEASE_VERSION-bin-$NAME.tgz.sha
}
-make_binary_release "hadoop1" "--with-hive --hadoop 1.0.4"
-make_binary_release "cdh4" "--with-hive --hadoop 2.0.0-mr1-cdh4.2.0"
-make_binary_release "hadoop2" "--with-hive --with-yarn --hadoop 2.2.0"
+make_binary_release "hadoop1" "-Phive -Dhadoop.version=1.0.4"
+make_binary_release "cdh4" "-Phive -Dhadoop.version=2.0.0-mr1-cdh4.2.0"
+make_binary_release "hadoop2" \
+ "-Phive -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 -Pyarn.version=2.2.0"
# Copy data
echo "Copying release tarballs"
http://git-wip-us.apache.org/repos/asf/spark/blob/d0ea4968/make-distribution.sh
----------------------------------------------------------------------
diff --git a/make-distribution.sh b/make-distribution.sh
index 94b473b..b5a90f0 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -23,21 +23,6 @@
# The distribution contains fat (assembly) jars that include the Scala library,
# so it is completely self contained.
# It does not contain source or *.class files.
-#
-# Optional Arguments
-# --tgz: Additionally creates spark-$VERSION-bin.tar.gz
-# --hadoop VERSION: Builds against specified version of Hadoop.
-# --with-yarn: Enables support for Hadoop YARN.
-# --with-hive: Enable support for reading Hive tables.
-# --name: A moniker for the release target. Defaults to the Hadoop verison.
-#
-# Recommended deploy/testing procedure (standalone mode):
-# 1) Rsync / deploy the dist/ dir to one host
-# 2) cd to deploy dir; ./sbin/start-master.sh
-# 3) Verify master is up by visiting web page, ie http://master-ip:8080. Note the spark:// URL.
-# 4) ./sbin/start-slave.sh 1 <<spark:// URL>>
-# 5) ./bin/spark-shell --master spark://my-master-ip:7077
-#
set -o pipefail
set -e
@@ -46,26 +31,35 @@ set -e
FWDIR="$(cd `dirname $0`; pwd)"
DISTDIR="$FWDIR/dist"
-# Initialize defaults
-SPARK_HADOOP_VERSION=1.0.4
-SPARK_YARN=false
-SPARK_HIVE=false
SPARK_TACHYON=false
MAKE_TGZ=false
NAME=none
+function exit_with_usage {
+ echo "make-distribution.sh - tool for making binary distributions of Spark"
+ echo ""
+ echo "usage:"
+ echo "./make-distribution.sh [--name] [--tgz] [--with-tachyon] <maven build options>"
+ echo "See Spark's \"Building with Maven\" doc for correct Maven options."
+ echo ""
+ exit 1
+}
+
# Parse arguments
while (( "$#" )); do
case $1 in
--hadoop)
- SPARK_HADOOP_VERSION="$2"
- shift
+ echo "Error: '--hadoop' is no longer supported:"
+ echo "Error: use Maven options -Phadoop.version and -Pyarn.version"
+ exit_with_usage
;;
--with-yarn)
- SPARK_YARN=true
+ echo "Error: '--with-yarn' is no longer supported, use Maven option -Pyarn"
+ exit_with_usage
;;
--with-hive)
- SPARK_HIVE=true
+ echo "Error: '--with-hive' is no longer supported, use Maven option -Pyarn"
+ exit_with_usage
;;
--skip-java-test)
SKIP_JAVA_TEST=true
@@ -80,6 +74,12 @@ while (( "$#" )); do
NAME="$2"
shift
;;
+ --help)
+ exit_with_usage
+ ;;
+ *)
+ break
+ ;;
esac
shift
done
@@ -143,14 +143,6 @@ else
echo "Making distribution for Spark $VERSION in $DISTDIR..."
fi
-echo "Hadoop version set to $SPARK_HADOOP_VERSION"
-echo "Release name set to $NAME"
-if [ "$SPARK_YARN" == "true" ]; then
- echo "YARN enabled"
-else
- echo "YARN disabled"
-fi
-
if [ "$SPARK_TACHYON" == "true" ]; then
echo "Tachyon Enabled"
else
@@ -162,33 +154,12 @@ cd $FWDIR
export MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m"
-BUILD_COMMAND="mvn clean package"
-
-# Use special profiles for hadoop versions 0.23.x, 2.2.x, 2.3.x, 2.4.x
-if [[ "$SPARK_HADOOP_VERSION" =~ ^0\.23\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-0.23"; fi
-if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.2\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.2"; fi
-if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.3\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.3"; fi
-if [[ "$SPARK_HADOOP_VERSION" =~ ^2\.4\. ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phadoop-2.4"; fi
-if [[ "$SPARK_HIVE" == "true" ]]; then BUILD_COMMAND="$BUILD_COMMAND -Phive"; fi
-if [[ "$SPARK_YARN" == "true" ]]; then
- # For hadoop versions 0.23.x to 2.1.x, use the yarn-alpha profile
- if [[ "$SPARK_HADOOP_VERSION" =~ ^0\.2[3-9]\. ]] ||
- [[ "$SPARK_HADOOP_VERSION" =~ ^0\.[3-9][0-9]\. ]] ||
- [[ "$SPARK_HADOOP_VERSION" =~ ^1\.[0-9]\. ]] ||
- [[ "$SPARK_HADOOP_VERSION" =~ ^2\.[0-1]\. ]]; then
- BUILD_COMMAND="$BUILD_COMMAND -Pyarn-alpha"
- # For hadoop versions 2.2+, use the yarn profile
- elif [[ "$SPARK_HADOOP_VERSION" =~ ^2.[2-9]. ]]; then
- BUILD_COMMAND="$BUILD_COMMAND -Pyarn"
- fi
- BUILD_COMMAND="$BUILD_COMMAND -Dyarn.version=$SPARK_HADOOP_VERSION"
-fi
-BUILD_COMMAND="$BUILD_COMMAND -Dhadoop.version=$SPARK_HADOOP_VERSION"
-BUILD_COMMAND="$BUILD_COMMAND -DskipTests"
+BUILD_COMMAND="mvn clean package -DskipTests $@"
# Actually build the jar
echo -e "\nBuilding with..."
echo -e "\$ $BUILD_COMMAND\n"
+
${BUILD_COMMAND}
# Make directories