You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/04/16 04:34:50 UTC
git commit: SPARK-1455: Better isolation for unit tests.
Repository: spark
Updated Branches:
refs/heads/master 07d72fe69 -> 5aaf9836f
SPARK-1455: Better isolation for unit tests.
This is a simple first step towards avoiding running the Hive tests
whenever possible.
Author: Patrick Wendell <pw...@gmail.com>
Closes #420 from pwendell/test-isolation and squashes the following commits:
350c8af [Patrick Wendell] SPARK-1455: Better isolation for unit tests.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5aaf9836
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5aaf9836
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5aaf9836
Branch: refs/heads/master
Commit: 5aaf9836f108d4ef9afe809353ad4d3aed560368
Parents: 07d72fe
Author: Patrick Wendell <pw...@gmail.com>
Authored: Tue Apr 15 19:34:39 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Tue Apr 15 19:34:39 2014 -0700
----------------------------------------------------------------------
dev/run-tests | 32 +++++++++++++++++++++++++-------
1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5aaf9836/dev/run-tests
----------------------------------------------------------------------
diff --git a/dev/run-tests b/dev/run-tests
index 0725b68..6805993 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -24,9 +24,6 @@ cd $FWDIR
# Remove work directory
rm -rf ./work
-# Fail fast
-set -e
-set -o pipefail
if test -x "$JAVA_HOME/bin/java"; then
declare java_cmd="$JAVA_HOME/bin/java"
else
@@ -34,7 +31,20 @@ else
fi
JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
[ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8."
-export SPARK_HIVE=true
+
+# Partial solution for SPARK-1455. Only run Hive tests if there are sql changes.
+if [ -n "$AMPLAB_JENKINS" ]; then
+ git fetch origin master:master
+ diffs=`git diff --dirstat master | awk '{ print $2; }' | grep "^sql/"`
+ if [ -n "$diffs" ]; then
+ echo "Detected changes in SQL. Will run Hive test suite."
+ run_sql_tests=true
+ fi
+fi
+
+# Fail fast
+set -e
+set -o pipefail
echo "========================================================================="
echo "Running Apache RAT checks"
@@ -49,9 +59,17 @@ dev/scalastyle
echo "========================================================================="
echo "Running Spark unit tests"
echo "========================================================================="
-# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation)
-# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block.
-echo -e "q\n" | sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+# echo "q" is needed because sbt on encountering a build file with failure
+# (either resolution or compilation) prompts the user for input either q, r,
+# etc to quit or retry. This echo is there to make it not block.
+echo -e "q\n" | SPARK_HIVE=true sbt/sbt assembly | \
+ grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
+
+if [ -n "$run_sql_tests" ]; then
+ echo -e "q\n" | SPARK_HIVE=true sbt/sbt test | grep -v -e "info.*Resolving"
+else
+ echo -e "q\n" | sbt/sbt test | grep -v -e "info.*Resolving"
+fi
echo "========================================================================="
echo "Running PySpark tests"