You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jo...@apache.org on 2015/06/30 08:09:02 UTC
spark git commit: [SPARK-5161] [HOTFIX] Fix bug in Python test
failure reporting
Repository: spark
Updated Branches:
refs/heads/master e6c3f7462 -> 6c5a6db4d
[SPARK-5161] [HOTFIX] Fix bug in Python test failure reporting
This patch fixes a bug introduced in #7031 which can cause Jenkins to incorrectly report a build with failed Python tests as passing if an error occurred while printing the test failure message.
Author: Josh Rosen <jo...@databricks.com>
Closes #7112 from JoshRosen/python-tests-hotfix and squashes the following commits:
c3f2961 [Josh Rosen] Hotfix for bug in Python test failure reporting
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c5a6db4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c5a6db4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c5a6db4
Branch: refs/heads/master
Commit: 6c5a6db4d53d6db8aa3464ea6713cf0d3a3bdfb5
Parents: e6c3f74
Author: Josh Rosen <jo...@databricks.com>
Authored: Mon Jun 29 23:08:51 2015 -0700
Committer: Josh Rosen <jo...@databricks.com>
Committed: Mon Jun 29 23:08:51 2015 -0700
----------------------------------------------------------------------
python/run-tests.py | 35 +++++++++++++++++++++++------------
1 file changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6c5a6db4/python/run-tests.py
----------------------------------------------------------------------
diff --git a/python/run-tests.py b/python/run-tests.py
index aaa35e9..b773765 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -58,22 +58,33 @@ def run_individual_python_test(test_name, pyspark_python):
env = {'SPARK_TESTING': '1', 'PYSPARK_PYTHON': which(pyspark_python)}
LOGGER.debug("Starting test(%s): %s", pyspark_python, test_name)
start_time = time.time()
- per_test_output = tempfile.TemporaryFile()
- retcode = subprocess.Popen(
- [os.path.join(SPARK_HOME, "bin/pyspark"), test_name],
- stderr=per_test_output, stdout=per_test_output, env=env).wait()
+ try:
+ per_test_output = tempfile.TemporaryFile()
+ retcode = subprocess.Popen(
+ [os.path.join(SPARK_HOME, "bin/pyspark"), test_name],
+ stderr=per_test_output, stdout=per_test_output, env=env).wait()
+ except:
+ LOGGER.exception("Got exception while running %s with %s", test_name, pyspark_python)
+ # Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
+ # this code is invoked from a thread other than the main thread.
+ os._exit(1)
duration = time.time() - start_time
# Exit on the first failure.
if retcode != 0:
- with FAILURE_REPORTING_LOCK:
- with open(LOG_FILE, 'ab') as log_file:
+ try:
+ with FAILURE_REPORTING_LOCK:
+ with open(LOG_FILE, 'ab') as log_file:
+ per_test_output.seek(0)
+ log_file.writelines(per_test_output)
per_test_output.seek(0)
- log_file.writelines(per_test_output.readlines())
- per_test_output.seek(0)
- for line in per_test_output:
- if not re.match('[0-9]+', line):
- print(line, end='')
- per_test_output.close()
+ for line in per_test_output:
+ decoded_line = line.decode()
+ if not re.match('[0-9]+', decoded_line):
+ print(decoded_line, end='')
+ per_test_output.close()
+ except:
+ LOGGER.exception("Got an exception while trying to print failed test output")
+ finally:
print_red("\nHad test failures in %s with %s; see logs." % (test_name, pyspark_python))
# Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
# this code is invoked from a thread other than the main thread.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org