You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2014/03/02 09:35:24 UTC
git commit: Merge the old sbt-launch-lib.bash with the new
sbt-launcher jar downloading logic.
Repository: spark
Updated Branches:
refs/heads/master 6fc76e49c -> 012bd5fbc
Merge the old sbt-launch-lib.bash with the new sbt-launcher jar downloading logic.
This allows developers to pass options (such as -D) to sbt. I also modified the SparkBuild to ensure spark specific properties are propagated to forked test JVMs.
Author: Michael Armbrust <mi...@databricks.com>
Closes #14 from marmbrus/sbtScripts and squashes the following commits:
c008b18 [Michael Armbrust] Merge the old sbt-launch-lib.bash with the new sbt-launcher jar downloading logic.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/012bd5fb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/012bd5fb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/012bd5fb
Branch: refs/heads/master
Commit: 012bd5fbc97dc40bb61e0e2b9cc97ed0083f37f6
Parents: 6fc76e4
Author: Michael Armbrust <mi...@databricks.com>
Authored: Sun Mar 2 00:35:23 2014 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Sun Mar 2 00:35:23 2014 -0800
----------------------------------------------------------------------
LICENSE | 32 ++++++++
project/SparkBuild.scala | 3 +
sbt/sbt | 153 ++++++++++++++++++++++++------------
sbt/sbt-launch-lib.bash | 178 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 315 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/012bd5fb/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 1c166d1..1c1c2c0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -396,3 +396,35 @@ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.
+
+
+========================================================================
+For sbt and sbt-launch-lib.bash in sbt/:
+========================================================================
+
+// Generated from http://www.opensource.org/licenses/bsd-license.php
+Copyright (c) 2011, Paul Phillips.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+ * Neither the name of the author nor the names of its contributors may be
+ used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/spark/blob/012bd5fb/project/SparkBuild.scala
----------------------------------------------------------------------
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index c402415..86340ad 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -23,6 +23,8 @@ import AssemblyKeys._
import scala.util.Properties
import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
+import scala.collection.JavaConversions._
+
// For Sonatype publishing
//import com.jsuereth.pgp.sbtplugin.PgpKeys._
@@ -140,6 +142,7 @@ object SparkBuild extends Build {
fork := true,
javaOptions in Test += "-Dspark.home=" + sparkHome,
javaOptions in Test += "-Dspark.testing=1",
+ javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark").map { case (k,v) => s"-D$k=$v" }.toSeq,
javaOptions += "-Xmx3g",
// Show full stack trace and duration in test cases.
testOptions in Test += Tests.Argument("-oDF"),
http://git-wip-us.apache.org/repos/asf/spark/blob/012bd5fb/sbt/sbt
----------------------------------------------------------------------
diff --git a/sbt/sbt b/sbt/sbt
index 8472dce..3ffa4ed 100755
--- a/sbt/sbt
+++ b/sbt/sbt
@@ -1,51 +1,102 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script launches sbt for this project. If present it uses the system
-# version of sbt. If there is no system version of sbt it attempts to download
-# sbt locally.
-SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
-URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
-URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
-JAR=sbt/sbt-launch-${SBT_VERSION}.jar
-
-# Download sbt launch jar if it hasn't been downloaded yet
-if [ ! -f ${JAR} ]; then
- # Download
- printf "Attempting to fetch sbt\n"
- JAR_DL=${JAR}.part
- if hash curl 2>/dev/null; then
- (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
- elif hash wget 2>/dev/null; then
- (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
- else
- printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
- exit -1
- fi
-fi
-if [ ! -f ${JAR} ]; then
- # We failed to download
- printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
- exit -1
-fi
-printf "Launching sbt from ${JAR}\n"
-java \
- -Xmx1200m -XX:MaxPermSize=350m -XX:ReservedCodeCacheSize=256m \
- -jar ${JAR} \
- "$@"
+#!/usr/bin/env bash
+
+realpath () {
+(
+ TARGET_FILE=$1
+
+ cd $(dirname $TARGET_FILE)
+ TARGET_FILE=$(basename $TARGET_FILE)
+
+ COUNT=0
+ while [ -L "$TARGET_FILE" -a $COUNT -lt 100 ]
+ do
+ TARGET_FILE=$(readlink $TARGET_FILE)
+ cd $(dirname $TARGET_FILE)
+ TARGET_FILE=$(basename $TARGET_FILE)
+ COUNT=$(($COUNT + 1))
+ done
+
+ echo $(pwd -P)/$TARGET_FILE
+)
+}
+
+. $(dirname $(realpath $0))/sbt-launch-lib.bash
+
+
+declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
+declare -r sbt_opts_file=".sbtopts"
+declare -r etc_sbt_opts_file="/etc/sbt/sbtopts"
+
+usage() {
+ cat <<EOM
+Usage: $script_name [options]
+
+ -h | -help print this message
+ -v | -verbose this runner is chattier
+ -d | -debug set sbt log level to debug
+ -no-colors disable ANSI color codes
+ -sbt-create start sbt even if current directory contains no sbt project
+ -sbt-dir <path> path to global settings/plugins directory (default: ~/.sbt)
+ -sbt-boot <path> path to shared boot directory (default: ~/.sbt/boot in 0.11 series)
+ -ivy <path> path to local Ivy repository (default: ~/.ivy2)
+ -mem <integer> set memory options (default: $sbt_mem, which is $(get_mem_opts $sbt_mem))
+ -no-share use all local caches; no sharing
+ -no-global uses global caches, but does not use global ~/.sbt directory.
+ -jvm-debug <port> Turn on JVM debugging, open at the given port.
+ -batch Disable interactive mode
+
+ # sbt version (default: from project/build.properties if present, else latest release)
+ -sbt-version <version> use the specified version of sbt
+ -sbt-jar <path> use the specified jar as the sbt launcher
+ -sbt-rc use an RC version of sbt
+ -sbt-snapshot use a snapshot version of sbt
+
+ # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
+ -java-home <path> alternate JAVA_HOME
+
+ # jvm options and output control
+ JAVA_OPTS environment variable, if unset uses "$java_opts"
+ SBT_OPTS environment variable, if unset uses "$default_sbt_opts"
+ .sbtopts if this file exists in the current directory, it is
+ prepended to the runner args
+ /etc/sbt/sbtopts if this file exists, it is prepended to the runner args
+ -Dkey=val pass -Dkey=val directly to the java runtime
+ -J-X pass option -X directly to the java runtime
+ (-J is stripped)
+ -S-X add -X to sbt's scalacOptions (-J is stripped)
+
+In the case of duplicated or conflicting options, the order above
+shows precedence: JAVA_OPTS lowest, command line options highest.
+EOM
+}
+
+process_my_args () {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
+ -no-share) addJava "$noshare_opts" && shift ;;
+ -no-global) addJava "-Dsbt.global.base=$(pwd)/project/.sbtboot" && shift ;;
+ -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
+ -sbt-dir) require_arg path "$1" "$2" && addJava "-Dsbt.global.base=$2" && shift 2 ;;
+ -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
+ -batch) exec </dev/null && shift ;;
+
+ -sbt-create) sbt_create=true && shift ;;
+
+ *) addResidual "$1" && shift ;;
+ esac
+ done
+
+ # Now, ensure sbt version is used.
+ [[ "${sbt_version}XXX" != "XXX" ]] && addJava "-Dsbt.version=$sbt_version"
+}
+
+loadConfigFile() {
+ cat "$1" | sed '/^\#/d'
+}
+
+# if sbtopts files exist, prepend their contents to $@ so it can be processed by this runner
+[[ -f "$etc_sbt_opts_file" ]] && set -- $(loadConfigFile "$etc_sbt_opts_file") "$@"
+[[ -f "$sbt_opts_file" ]] && set -- $(loadConfigFile "$sbt_opts_file") "$@"
+
+run "$@"
http://git-wip-us.apache.org/repos/asf/spark/blob/012bd5fb/sbt/sbt-launch-lib.bash
----------------------------------------------------------------------
diff --git a/sbt/sbt-launch-lib.bash b/sbt/sbt-launch-lib.bash
new file mode 100755
index 0000000..d65bbdc
--- /dev/null
+++ b/sbt/sbt-launch-lib.bash
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+#
+
+# A library to simplify using the SBT launcher from other packages.
+# Note: This should be used by tools like giter8/conscript etc.
+
+# TODO - Should we merge the main SBT script with this library?
+
+if test -z "$HOME"; then
+ declare -r script_dir="$(dirname $script_path)"
+else
+ declare -r script_dir="$HOME/.sbt"
+fi
+
+declare -a residual_args
+declare -a java_args
+declare -a scalac_args
+declare -a sbt_commands
+declare java_cmd=java
+
+echoerr () {
+ echo 1>&2 "$@"
+}
+vlog () {
+ [[ $verbose || $debug ]] && echoerr "$@"
+}
+dlog () {
+ [[ $debug ]] && echoerr "$@"
+}
+
+acquire_sbt_jar () {
+ SBT_VERSION=`awk -F "=" '/sbt\\.version/ {print $2}' ./project/build.properties`
+ URL1=http://typesafe.artifactoryonline.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+ URL2=http://repo.typesafe.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch.jar
+ JAR=sbt/sbt-launch-${SBT_VERSION}.jar
+
+ sbt_jar=$JAR
+
+ if [[ ! -f "$sbt_jar" ]]; then
+ # Download sbt launch jar if it hasn't been downloaded yet
+ if [ ! -f ${JAR} ]; then
+ # Download
+ printf "Attempting to fetch sbt\n"
+ JAR_DL=${JAR}.part
+ if hash curl 2>/dev/null; then
+ (curl --progress-bar ${URL1} > ${JAR_DL} || curl --progress-bar ${URL2} > ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+ elif hash wget 2>/dev/null; then
+ (wget --progress=bar ${URL1} -O ${JAR_DL} || wget --progress=bar ${URL2} -O ${JAR_DL}) && mv ${JAR_DL} ${JAR}
+ else
+ printf "You do not have curl or wget installed, please install sbt manually from http://www.scala-sbt.org/\n"
+ exit -1
+ fi
+ fi
+ if [ ! -f ${JAR} ]; then
+ # We failed to download
+ printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from http://www.scala-sbt.org/\n"
+ exit -1
+ fi
+ printf "Launching sbt from ${JAR}\n"
+ fi
+}
+
+execRunner () {
+ # print the arguments one to a line, quoting any containing spaces
+ [[ $verbose || $debug ]] && echo "# Executing command line:" && {
+ for arg; do
+ if printf "%s\n" "$arg" | grep -q ' '; then
+ printf "\"%s\"\n" "$arg"
+ else
+ printf "%s\n" "$arg"
+ fi
+ done
+ echo ""
+ }
+
+ exec "$@"
+}
+
+addJava () {
+ dlog "[addJava] arg = '$1'"
+ java_args=( "${java_args[@]}" "$1" )
+}
+addSbt () {
+ dlog "[addSbt] arg = '$1'"
+ sbt_commands=( "${sbt_commands[@]}" "$1" )
+}
+addResidual () {
+ dlog "[residual] arg = '$1'"
+ residual_args=( "${residual_args[@]}" "$1" )
+}
+addDebugger () {
+ addJava "-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
+}
+
+# a ham-fisted attempt to move some memory settings in concert
+# so they need not be dicked around with individually.
+get_mem_opts () {
+ local mem=${1:-2048}
+ local perm=$(( $mem / 4 ))
+ (( $perm > 256 )) || perm=256
+ (( $perm < 1024 )) || perm=1024
+ local codecache=$(( $perm / 2 ))
+
+ echo "-Xms${mem}m -Xmx${mem}m -XX:MaxPermSize=${perm}m -XX:ReservedCodeCacheSize=${codecache}m"
+}
+
+require_arg () {
+ local type="$1"
+ local opt="$2"
+ local arg="$3"
+ if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
+ die "$opt requires <$type> argument"
+ fi
+}
+
+is_function_defined() {
+ declare -f "$1" > /dev/null
+}
+
+process_args () {
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ -h|-help) usage; exit 1 ;;
+ -v|-verbose) verbose=1 && shift ;;
+ -d|-debug) debug=1 && shift ;;
+
+ -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
+ -mem) require_arg integer "$1" "$2" && sbt_mem="$2" && shift 2 ;;
+ -jvm-debug) require_arg port "$1" "$2" && addDebugger $2 && shift 2 ;;
+ -batch) exec </dev/null && shift ;;
+
+ -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
+ -sbt-version) require_arg version "$1" "$2" && sbt_version="$2" && shift 2 ;;
+ -java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && shift 2 ;;
+
+ -D*) addJava "$1" && shift ;;
+ -J*) addJava "${1:2}" && shift ;;
+ *) addResidual "$1" && shift ;;
+ esac
+ done
+
+ is_function_defined process_my_args && {
+ myargs=("${residual_args[@]}")
+ residual_args=()
+ process_my_args "${myargs[@]}"
+ }
+}
+
+run() {
+ # no jar? download it.
+ [[ -f "$sbt_jar" ]] || acquire_sbt_jar "$sbt_version" || {
+ # still no jar? uh-oh.
+ echo "Download failed. Obtain the sbt-launch.jar manually and place it at $sbt_jar"
+ exit 1
+ }
+
+ # process the combined args, then reset "$@" to the residuals
+ process_args "$@"
+ set -- "${residual_args[@]}"
+ argumentCount=$#
+
+ # run sbt
+ execRunner "$java_cmd" \
+ ${SBT_OPTS:-$default_sbt_opts} \
+ $(get_mem_opts $sbt_mem) \
+ ${java_opts} \
+ ${java_args[@]} \
+ -jar "$sbt_jar" \
+ "${sbt_commands[@]}" \
+ "${residual_args[@]}"
+}
+
+runAlternateBoot() {
+ local bootpropsfile="$1"
+ shift
+ addJava "-Dsbt.boot.properties=$bootpropsfile"
+ run $@
+}