You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/07/17 02:31:09 UTC
[12/50] [abbrv] git commit: Merge remote-tracking branch
'origin/pr/662'
Merge remote-tracking branch 'origin/pr/662'
Conflicts:
bin/compute-classpath.sh
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/cd28d9c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/cd28d9c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/cd28d9c1
Branch: refs/heads/master
Commit: cd28d9c14732a91416589bb3dc523533495240e8
Parents: 1889f9f 1107b4d
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Sat Jul 13 19:10:00 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Sat Jul 13 19:10:00 2013 -0700
----------------------------------------------------------------------
bin/compute-classpath.sh | 111 +++++++++++++++++++++++-------------------
bin/start-slave.sh | 3 ++
make-distribution.sh | 39 +++++++++++++++
project/SparkBuild.scala | 2 +-
run | 44 ++++++-----------
spark-shell | 67 +++++++++++++++++++++++--
6 files changed, 185 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/cd28d9c1/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --cc bin/compute-classpath.sh
index 75c58d1,47937a0..bd48b43
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@@ -24,46 -23,70 +24,71 @@@ PYSPARK_DIR="$FWDIR/python
# Build up classpath
CLASSPATH="$SPARK_CLASSPATH"
- CLASSPATH="$CLASSPATH:$FWDIR/conf"
- CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/classes"
- if [ -n "$SPARK_TESTING" ] ; then
- CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/test-classes"
- CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/test-classes"
- fi
- CLASSPATH="$CLASSPATH:$CORE_DIR/src/main/resources"
- CLASSPATH="$CLASSPATH:$REPL_DIR/target/scala-$SCALA_VERSION/classes"
- CLASSPATH="$CLASSPATH:$EXAMPLES_DIR/target/scala-$SCALA_VERSION/classes"
- CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/classes"
- CLASSPATH="$CLASSPATH:$STREAMING_DIR/lib/org/apache/kafka/kafka/0.7.2-spark/*" # <-- our in-project Kafka Jar
- if [ -e "$FWDIR/lib_managed" ]; then
- CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/jars/*"
- CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/bundles/*"
- fi
- CLASSPATH="$CLASSPATH:$REPL_DIR/lib/*"
- # Add the shaded JAR for Maven builds
- if [ -e $REPL_BIN_DIR/target ]; then
- for jar in `find "$REPL_BIN_DIR/target" -name 'spark-repl-*-shaded-hadoop*.jar'`; do
+
+ function dev_classpath {
+ CLASSPATH="$CLASSPATH:$FWDIR/conf"
+ CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/classes"
+ if [ -n "$SPARK_TESTING" ] ; then
+ CLASSPATH="$CLASSPATH:$CORE_DIR/target/scala-$SCALA_VERSION/test-classes"
+ CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/test-classes"
+ fi
+ CLASSPATH="$CLASSPATH:$CORE_DIR/src/main/resources"
+ CLASSPATH="$CLASSPATH:$REPL_DIR/target/scala-$SCALA_VERSION/classes"
+ CLASSPATH="$CLASSPATH:$EXAMPLES_DIR/target/scala-$SCALA_VERSION/classes"
+ CLASSPATH="$CLASSPATH:$STREAMING_DIR/target/scala-$SCALA_VERSION/classes"
+ CLASSPATH="$CLASSPATH:$STREAMING_DIR/lib/org/apache/kafka/kafka/0.7.2-spark/*" # <-- our in-project Kafka Jar
+ if [ -e "$FWDIR/lib_managed" ]; then
+ CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/jars/*"
+ CLASSPATH="$CLASSPATH:$FWDIR/lib_managed/bundles/*"
+ fi
+ CLASSPATH="$CLASSPATH:$REPL_DIR/lib/*"
+ # Add the shaded JAR for Maven builds
+ if [ -e $REPL_BIN_DIR/target ]; then
+ for jar in `find "$REPL_BIN_DIR/target" -name 'spark-repl-*-shaded-hadoop*.jar'`; do
+ CLASSPATH="$CLASSPATH:$jar"
+ done
+ # The shaded JAR doesn't contain examples, so include those separately
+ EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar`
+ CLASSPATH+=":$EXAMPLES_JAR"
+ fi
+ CLASSPATH="$CLASSPATH:$BAGEL_DIR/target/scala-$SCALA_VERSION/classes"
++ CLASSPATH="$CLASSPATH:$MLLIB_DIR/target/scala-$SCALA_VERSION/classes"
+ for jar in `find $PYSPARK_DIR/lib -name '*jar'`; do
CLASSPATH="$CLASSPATH:$jar"
done
- # The shaded JAR doesn't contain examples, so include those separately
- EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar`
- CLASSPATH+=":$EXAMPLES_JAR"
- fi
- CLASSPATH="$CLASSPATH:$BAGEL_DIR/target/scala-$SCALA_VERSION/classes"
- CLASSPATH="$CLASSPATH:$MLLIB_DIR/target/scala-$SCALA_VERSION/classes"
- for jar in `find $PYSPARK_DIR/lib -name '*jar'`; do
- CLASSPATH="$CLASSPATH:$jar"
- done
- # Figure out the JAR file that our examples were packaged into. This includes a bit of a hack
- # to avoid the -sources and -doc packages that are built by publish-local.
- if [ -e "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar ]; then
- # Use the JAR from the SBT build
- export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar`
- fi
- if [ -e "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar ]; then
- # Use the JAR from the Maven build
- export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar`
+ # Figure out the JAR file that our examples were packaged into. This includes a bit of a hack
+ # to avoid the -sources and -doc packages that are built by publish-local.
+ if [ -e "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar ]; then
+ # Use the JAR from the SBT build
+ export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/scala-$SCALA_VERSION/spark-examples"*[0-9T].jar`
+ fi
+ if [ -e "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar ]; then
+ # Use the JAR from the Maven build
+ export SPARK_EXAMPLES_JAR=`ls "$EXAMPLES_DIR/target/spark-examples"*[0-9T].jar`
+ fi
+
+ # Add Scala standard library
+ if [ -z "$SCALA_LIBRARY_PATH" ]; then
+ if [ -z "$SCALA_HOME" ]; then
+ echo "SCALA_HOME is not set" >&2
+ exit 1
+ fi
+ SCALA_LIBRARY_PATH="$SCALA_HOME/lib"
+ fi
+ CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/scala-library.jar"
+ CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/scala-compiler.jar"
+ CLASSPATH="$CLASSPATH:$SCALA_LIBRARY_PATH/jline.jar"
+ }
+
+ function release_classpath {
+ CLASSPATH="$CLASSPATH:$FWDIR/jars/*"
+ }
+
+ if [ -f "$FWDIR/RELEASE" ]; then
+ release_classpath
+ else
+ dev_classpath
fi
# Add hadoop conf dir - else FileSystem.*, etc fail !
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/cd28d9c1/make-distribution.sh
----------------------------------------------------------------------
diff --cc make-distribution.sh
index 0000000,8554758..feb13d5
mode 000000,100755..100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@@ -1,0 -1,38 +1,39 @@@
+ #!/bin/bash
+ #
+ # Script to create a binary distribution for easy deploys of Spark.
+ # The distribution directory defaults to dist/ but can be overridden below.
+ # The distribution contains fat (assembly) jars that include the Scala library,
+ # so it is completely self contained.
+ # It does not contain source or *.class files.
+ #
+ # Recommended deploy/testing procedure (standalone mode):
+ # 1) Rsync / deploy the dist/ dir to one host
+ # 2) cd to deploy dir; ./bin/start-master.sh
+ # 3) Verify master is up by visiting web page, ie http://master-ip:8080. Note the spark:// URL.
+ # 4) ./bin/start-slave.sh 1 <<spark:// URL>>
+ # 5) MASTER="spark://my-master-ip:7077" ./spark-shell
+
+ # Figure out where the Spark framework is installed
+ FWDIR="$(cd `dirname $0`; pwd)"
+ DISTDIR="$FWDIR/dist"
+
+ # Get version from SBT
++export TERM=dumb # Prevents color codes in SBT output
+ VERSION=$($FWDIR/sbt/sbt "show version" | tail -1 | cut -f 2)
+ echo "Making distribution for Spark $VERSION in $DISTDIR..."
+
+ # Build fat JAR
+ $FWDIR/sbt/sbt "repl/assembly"
+
+ # Make directories
+ rm -rf "$DISTDIR"
+ mkdir -p "$DISTDIR/jars"
+ echo "$VERSION" >$DISTDIR/RELEASE
+
+ # Copy jars
+ cp $FWDIR/repl/target/*.jar "$DISTDIR/jars/"
+
+ # Copy other things
+ cp -r "$FWDIR/bin" "$DISTDIR"
+ cp -r "$FWDIR/conf" "$DISTDIR"
-cp "$FWDIR/run" "$FWDIR/spark-shell" "$DISTDIR"
++cp "$FWDIR/run" "$FWDIR/spark-shell" "$DISTDIR"
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/cd28d9c1/project/SparkBuild.scala
----------------------------------------------------------------------
diff --cc project/SparkBuild.scala
index 641d379,6058fb1..bc9214d
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@@ -197,7 -198,7 +197,7 @@@ object SparkBuild extends Build
def replSettings = sharedSettings ++ Seq(
name := "spark-repl",
libraryDependencies <+= scalaVersion("org.scala-lang" % "scala-compiler" % _)
- )
- ) ++ assemblySettings ++ extraAssemblySettings ++ Twirl.settings
++ ) ++ assemblySettings ++ extraAssemblySettings
def examplesSettings = sharedSettings ++ Seq(
name := "spark-examples",