You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/03/30 08:03:10 UTC
git commit: SPARK-1336 Reducing the output of run-tests script.
Repository: spark
Updated Branches:
refs/heads/master 2861b07bb -> df1b9f7b1
SPARK-1336 Reducing the output of run-tests script.
Author: Prashant Sharma <pr...@imaginea.com>
Author: Prashant Sharma <sc...@gmail.com>
Closes #262 from ScrapCodes/SPARK-1336/ReduceVerbosity and squashes the following commits:
87dfa54 [Prashant Sharma] Further reduction in noise and made pyspark tests to fail fast.
811170f [Prashant Sharma] Reducing the ouput of run-tests script.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/df1b9f7b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/df1b9f7b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/df1b9f7b
Branch: refs/heads/master
Commit: df1b9f7b1a07bf8d806695a7684f9d69bf705093
Parents: 2861b07
Author: Prashant Sharma <pr...@imaginea.com>
Authored: Sat Mar 29 23:03:03 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sat Mar 29 23:03:03 2014 -0700
----------------------------------------------------------------------
.gitignore | 1 +
dev/run-tests | 7 +++----
dev/scalastyle | 27 +++++++++++++++++++++++++++
python/run-tests | 19 ++++++++++++-------
4 files changed, 43 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/df1b9f7b/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 3a68abd..cd9f90d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -47,3 +47,4 @@ spark-*-bin.tar.gz
unit-tests.log
/lib/
rat-results.txt
+scalastyle.txt
http://git-wip-us.apache.org/repos/asf/spark/blob/df1b9f7b/dev/run-tests
----------------------------------------------------------------------
diff --git a/dev/run-tests b/dev/run-tests
index 6f115d2..a6fcc40 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -39,18 +39,17 @@ JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*
echo "========================================================================="
echo "Running Apache RAT checks"
echo "========================================================================="
-
dev/check-license
echo "========================================================================="
echo "Running Scala style checks"
echo "========================================================================="
-sbt/sbt clean scalastyle
+dev/scalastyle
echo "========================================================================="
echo "Running Spark unit tests"
echo "========================================================================="
-sbt/sbt assembly test
+sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
echo "========================================================================="
echo "Running PySpark tests"
@@ -64,5 +63,5 @@ echo "========================================================================="
echo "Detecting binary incompatibilites with MiMa"
echo "========================================================================="
./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore
-sbt/sbt mima-report-binary-issues
+sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving"
http://git-wip-us.apache.org/repos/asf/spark/blob/df1b9f7b/dev/scalastyle
----------------------------------------------------------------------
diff --git a/dev/scalastyle b/dev/scalastyle
new file mode 100755
index 0000000..5a18f4d
--- /dev/null
+++ b/dev/scalastyle
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+sbt/sbt clean scalastyle > scalastyle.txt
+ERRORS=$(cat scalastyle.txt | grep -e "error file")
+if test ! -z "$ERRORS"; then
+ echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS"
+ exit 1
+else
+ echo -e "Scalastyle checks passed.\n"
+fi
http://git-wip-us.apache.org/repos/asf/spark/blob/df1b9f7b/python/run-tests
----------------------------------------------------------------------
diff --git a/python/run-tests b/python/run-tests
index a986ac9..b2b60f0 100755
--- a/python/run-tests
+++ b/python/run-tests
@@ -29,8 +29,18 @@ FAILED=0
rm -f unit-tests.log
function run_test() {
- SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a unit-tests.log
+ SPARK_TESTING=0 $FWDIR/bin/pyspark $1 2>&1 | tee -a > unit-tests.log
FAILED=$((PIPESTATUS[0]||$FAILED))
+
+ # Fail and exit on the first test failure.
+ if [[ $FAILED != 0 ]]; then
+ cat unit-tests.log | grep -v "^[0-9][0-9]*" # filter all lines starting with a number.
+ echo -en "\033[31m" # Red
+ echo "Had test failures; see logs."
+ echo -en "\033[0m" # No color
+ exit -1
+ fi
+
}
run_test "pyspark/rdd.py"
@@ -46,12 +56,7 @@ run_test "pyspark/mllib/clustering.py"
run_test "pyspark/mllib/recommendation.py"
run_test "pyspark/mllib/regression.py"
-if [[ $FAILED != 0 ]]; then
- echo -en "\033[31m" # Red
- echo "Had test failures; see logs."
- echo -en "\033[0m" # No color
- exit -1
-else
+if [[ $FAILED == 0 ]]; then
echo -en "\033[32m" # Green
echo "Tests passed."
echo -en "\033[0m" # No color