You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/04/16 04:34:50 UTC

git commit: SPARK-1455: Better isolation for unit tests.

Repository: spark
Updated Branches:
  refs/heads/master 07d72fe69 -> 5aaf9836f


SPARK-1455: Better isolation for unit tests.

This is a simple first step towards avoiding running the Hive tests
whenever possible.

Author: Patrick Wendell <pw...@gmail.com>

Closes #420 from pwendell/test-isolation and squashes the following commits:

350c8af [Patrick Wendell] SPARK-1455: Better isolation for unit tests.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5aaf9836
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5aaf9836
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5aaf9836

Branch: refs/heads/master
Commit: 5aaf9836f108d4ef9afe809353ad4d3aed560368
Parents: 07d72fe
Author: Patrick Wendell <pw...@gmail.com>
Authored: Tue Apr 15 19:34:39 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Apr 15 19:34:39 2014 -0700

----------------------------------------------------------------------
 dev/run-tests | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/5aaf9836/dev/run-tests
----------------------------------------------------------------------
diff --git a/dev/run-tests b/dev/run-tests
index 0725b68..6805993 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -24,9 +24,6 @@ cd $FWDIR
 # Remove work directory
 rm -rf ./work
 
-# Fail fast
-set -e
-set -o pipefail
 if test -x "$JAVA_HOME/bin/java"; then
     declare java_cmd="$JAVA_HOME/bin/java"
 else 
@@ -34,7 +31,20 @@ else
 fi
 JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
 [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
-export SPARK_HIVE=true
+
+# Partial solution for SPARK-1455. Only run Hive tests if there are sql changes.
+if [ -n "$AMPLAB_JENKINS" ]; then
+  git fetch origin master:master
+  diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"`
+  if [ -n "$diffs" ]; then
+    echo "Detected changes in SQL. Will run Hive test suite."
+    run_sql_tests=true
+  fi
+fi
+
+# Fail fast
+set -e
+set -o pipefail
 
 echo "========================================================================="
 echo "Running Apache RAT checks"
@@ -49,9 +59,17 @@ dev/scalastyle
 echo "========================================================================="
 echo "Running Spark unit tests"
 echo "========================================================================="
-# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation)
-# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block.
-echo -e "q\n" | sbt/sbt assembly test |  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+# echo "q" is needed because sbt on encountering a build file with failure 
+# (either resolution or compilation) prompts the user for input either q, r, 
+# etc to quit or retry. This echo is there to make it not block.
+echo -e "q\n" | SPARK_HIVE=true sbt/sbt assembly | \
+  grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+
+if [ -n "$run_sql_tests" ]; then
+  echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving" 
+else
+  echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving" 
+fi
 
 echo "========================================================================="
 echo "Running PySpark tests"