You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/08/23 20:58:58 UTC
[3/6] impala git commit: IMPALA-7399: Emit a junit xml report when
trapping errors
IMPALA-7399: Emit a junit xml report when trapping errors
This patch will cause a junitxml file to be emitted in the case of
errors in build scripts. Instead of simply echoing a message to the
console, we set up a trap function that also writes out to a
junit xml report that can be consumed by jenkins.impala.io.
Main things to pay attention to:
- New file that gets sourced by all bash scripts when trapping
within bash scripts:
https://gerrit.cloudera.org/c/11257/1/bin/report_build_error.sh
- Installation of the python lib into impala-python venv for use
from within python files:
https://gerrit.cloudera.org/c/11257/1/bin/impala-python-common.sh
- Change to the generate_junitxml.py file itself, for ease of
https://gerrit.cloudera.org/c/11257/1/lib/python/impala_py_lib/jenkins/generate_junitxml.py
Most of the other changes are to source the new report_build_error.sh
script to set up the trap function.
Change-Id: Idd62045bb43357abc2b89a78afff499149d3c3fc
Reviewed-on: http://gerrit.cloudera.org:8080/11257
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6e5ec22b
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6e5ec22b
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6e5ec22b
Branch: refs/heads/master
Commit: 6e5ec22b1237f1d466c095c96a2fc1cb71ccb2d9
Parents: bb9454f
Author: David Knupp <dk...@cloudera.com>
Authored: Thu Aug 16 17:06:04 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Aug 23 18:33:58 2018 +0000
----------------------------------------------------------------------
bin/clean-cmake.sh | 4 +-
bin/clean.sh | 4 +-
bin/create-test-configuration.sh | 3 +-
bin/create_testdata.sh | 3 +-
bin/distcc/distcc_server_setup.sh | 3 +-
bin/impala-python-common.sh | 3 +-
bin/jenkins/all-tests.sh | 3 +-
bin/jenkins/build-all-flag-combinations.sh | 3 +-
bin/jenkins/build-only.sh | 3 +-
bin/make_impala.sh | 3 +-
bin/report_build_error.sh | 30 +++++++++++++++
bin/run-all-tests.sh | 3 +-
bin/run-backend-tests.sh | 3 +-
bin/start-catalogd.sh | 3 +-
bin/start-impalad.sh | 3 +-
bin/start-statestored.sh | 3 +-
buildall.sh | 4 +-
infra/python/bootstrap_virtualenv.py | 24 ++++++++----
.../impala_py_lib/jenkins/generate_junitxml.py | 40 ++++++++++++--------
shell/make_shell_tarball.sh | 3 +-
testdata/bin/check-schema-diff.sh | 4 ++
testdata/bin/compute-table-stats.sh | 3 +-
testdata/bin/copy-data-sources.sh | 3 +-
testdata/bin/copy-udfs-udas.sh | 3 +-
testdata/bin/create-load-data.sh | 3 +-
testdata/bin/create-table-many-blocks.sh | 3 +-
testdata/bin/generate-load-nested.sh | 3 +-
testdata/bin/kill-all.sh | 3 +-
testdata/bin/kill-hbase.sh | 3 +-
testdata/bin/kill-hive-server.sh | 3 +-
testdata/bin/kill-java-service.sh | 3 +-
testdata/bin/kill-sentry-service.sh | 3 +-
testdata/bin/load-hive-builtins.sh | 3 +-
testdata/bin/load-metastore-snapshot.sh | 3 +-
testdata/bin/load-test-warehouse-snapshot.sh | 3 +-
testdata/bin/run-all.sh | 3 +-
testdata/bin/run-hbase.sh | 3 +-
testdata/bin/run-hive-server.sh | 3 +-
testdata/bin/run-mini-dfs.sh | 3 +-
testdata/bin/run-sentry-service.sh | 3 +-
testdata/bin/setup-hdfs-env.sh | 3 +-
testdata/cluster/admin | 3 +-
testdata/datasets/tpcds/preload | 3 +-
testdata/datasets/tpch/preload | 3 +-
tests/run-custom-cluster-tests.sh | 3 +-
tests/run-process-failure-tests.sh | 3 +-
46 files changed, 159 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/clean-cmake.sh
----------------------------------------------------------------------
diff --git a/bin/clean-cmake.sh b/bin/clean-cmake.sh
index 92415fc..aa1ea96 100755
--- a/bin/clean-cmake.sh
+++ b/bin/clean-cmake.sh
@@ -20,8 +20,8 @@
# Removes artifacts generated by cmake.
set -euo pipefail
-trap 'echo Error in ${0} at line ${LINENO}: $(cd "'${PWD}'" && awk "NR == ${LINENO}" \
- ${0})' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
if [[ -z "${IMPALA_HOME}" || ! -d "${IMPALA_HOME}" ]]; then
echo IMPALA_HOME=${IMPALA_HOME} is not valid. 1>&2
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/clean.sh
----------------------------------------------------------------------
diff --git a/bin/clean.sh b/bin/clean.sh
index 89a991c..d0b7c3b 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -22,8 +22,8 @@
# branch to a non-toolchain branch due to caching in CMake generated files.
set -euo pipefail
-trap 'echo Error in ${0} at line ${LINENO}: $(cd "'${PWD}'" && awk "NR == ${LINENO}" \
- ${0})' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# If the project was never build, no Makefile will exist and thus make clean will fail.
# Combine the make command with the bash noop to always return true.
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/create-test-configuration.sh
----------------------------------------------------------------------
diff --git a/bin/create-test-configuration.sh b/bin/create-test-configuration.sh
index e68af9c..6e799fb 100755
--- a/bin/create-test-configuration.sh
+++ b/bin/create-test-configuration.sh
@@ -22,7 +22,8 @@
# as creation of the Hive metastore.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
CREATE_METASTORE=0
CREATE_SENTRY_POLICY_DB=0
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/create_testdata.sh
----------------------------------------------------------------------
diff --git a/bin/create_testdata.sh b/bin/create_testdata.sh
index cc296bf..5b81d9e 100755
--- a/bin/create_testdata.sh
+++ b/bin/create_testdata.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/distcc/distcc_server_setup.sh
----------------------------------------------------------------------
diff --git a/bin/distcc/distcc_server_setup.sh b/bin/distcc/distcc_server_setup.sh
index 8b5e6a9..6b514e7 100755
--- a/bin/distcc/distcc_server_setup.sh
+++ b/bin/distcc/distcc_server_setup.sh
@@ -39,7 +39,8 @@
# CCACHE_DIR: directory to use for distccd's ccache.
# CCACHE_SIZE: size of ccache, passed to ccache's -M option
set -eu -o pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
if [[ $# != 1 ]]; then
echo "Usage: $0 <allowed IP address range>"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/impala-python-common.sh
----------------------------------------------------------------------
diff --git a/bin/impala-python-common.sh b/bin/impala-python-common.sh
index 29a36a2..501d487 100644
--- a/bin/impala-python-common.sh
+++ b/bin/impala-python-common.sh
@@ -19,7 +19,8 @@
# $IMPALA_HOME/bin/impala-py* executables.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
LD_LIBRARY_PATH+=":$(python "$IMPALA_HOME/infra/python/bootstrap_virtualenv.py" \
--print-ld-library-path)"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/all-tests.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh
index 7917358..1e73722 100644
--- a/bin/jenkins/all-tests.sh
+++ b/bin/jenkins/all-tests.sh
@@ -19,7 +19,8 @@
# Run all Impala tests.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
cd "${IMPALA_HOME}"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/build-all-flag-combinations.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index 8dce06d..200729e 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -25,7 +25,8 @@
# Usage: build-all-flag-combinations.sh [--dryrun]
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
export IMPALA_MAVEN_OPTIONS="-U"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/jenkins/build-only.sh
----------------------------------------------------------------------
diff --git a/bin/jenkins/build-only.sh b/bin/jenkins/build-only.sh
index d14ad6e..e7bdb3f 100644
--- a/bin/jenkins/build-only.sh
+++ b/bin/jenkins/build-only.sh
@@ -19,7 +19,8 @@
# Only run an Impala build.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
cd "${IMPALA_HOME}"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/make_impala.sh
----------------------------------------------------------------------
diff --git a/bin/make_impala.sh b/bin/make_impala.sh
index 8e1807b..f33bd74 100755
--- a/bin/make_impala.sh
+++ b/bin/make_impala.sh
@@ -20,7 +20,8 @@
# Incrementally compiles the frontend and backend.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
: ${IMPALA_TOOLCHAIN=}
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/report_build_error.sh
----------------------------------------------------------------------
diff --git a/bin/report_build_error.sh b/bin/report_build_error.sh
new file mode 100644
index 0000000..295cce0
--- /dev/null
+++ b/bin/report_build_error.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+report_build_error() {
+ ERROR_MSG=$(cd "$PWD" && awk "NR == $1" $0)
+ FILENAME=$(basename -- "$0")
+ echo ERROR in $0 at line $1: $ERROR_MSG
+ $IMPALA_HOME/bin/generate_junitxml.py --step "${FILENAME%.*}" \
+ --error "Error in $0 at line $1: $ERROR_MSG"
+}
+
+setup_report_build_error() {
+ trap 'report_build_error $LINENO' ERR
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/run-all-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-all-tests.sh b/bin/run-all-tests.sh
index 5f6831e..da7090e 100755
--- a/bin/run-all-tests.sh
+++ b/bin/run-all-tests.sh
@@ -22,7 +22,8 @@
# Exit on reference to uninitialized variables and non-zero exit codes
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. "$IMPALA_HOME/bin/set-pythonpath.sh"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/run-backend-tests.sh
----------------------------------------------------------------------
diff --git a/bin/run-backend-tests.sh b/bin/run-backend-tests.sh
index 3bc84c2..16a432f 100755
--- a/bin/run-backend-tests.sh
+++ b/bin/run-backend-tests.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
export GTEST_OUTPUT="xml:$IMPALA_BE_TEST_LOGS_DIR/"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-catalogd.sh
----------------------------------------------------------------------
diff --git a/bin/start-catalogd.sh b/bin/start-catalogd.sh
index a8b7e28..35cf592 100755
--- a/bin/start-catalogd.sh
+++ b/bin/start-catalogd.sh
@@ -21,7 +21,8 @@
# -build_type parameter can be passed to determine the build type to use.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
BUILD_TYPE=latest
CATALOGD_ARGS=""
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-impalad.sh
----------------------------------------------------------------------
diff --git a/bin/start-impalad.sh b/bin/start-impalad.sh
index f052fa2..dacd44c 100755
--- a/bin/start-impalad.sh
+++ b/bin/start-impalad.sh
@@ -21,7 +21,8 @@
# parameter can be passed to determine the build type to use for the impalad instance.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
BUILD_TYPE=latest
IMPALAD_ARGS=""
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/bin/start-statestored.sh
----------------------------------------------------------------------
diff --git a/bin/start-statestored.sh b/bin/start-statestored.sh
index f023810..eb94910 100755
--- a/bin/start-statestored.sh
+++ b/bin/start-statestored.sh
@@ -20,7 +20,8 @@
# Starts up the StateStored with the specified command line arguments.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
BUILD_TYPE=latest
STATESTORED_ARGS=""
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index 59e27ee..339168e 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -18,6 +18,8 @@
# under the License.
set -euo pipefail
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# run buildall.sh -help to see options
ROOT=`dirname "$0"`
@@ -29,8 +31,6 @@ then
exit 1
fi
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
-
# Grab this *before* we source impala-config.sh to see if the caller has
# kerberized environment variables already or not.
NEEDS_RE_SOURCE_NOTE=1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/infra/python/bootstrap_virtualenv.py
----------------------------------------------------------------------
diff --git a/infra/python/bootstrap_virtualenv.py b/infra/python/bootstrap_virtualenv.py
index 1953935..f2ce839 100644
--- a/infra/python/bootstrap_virtualenv.py
+++ b/infra/python/bootstrap_virtualenv.py
@@ -140,29 +140,37 @@ def exec_pip_install(args, cc="no-cc-available", env=None):
# Don't call the virtualenv pip directly, it uses a hashbang to to call the python
# virtualenv using an absolute path. If the path to the virtualenv is very long, the
# hashbang won't work.
- #
+ impala_pip_base_cmd = [os.path.join(ENV_DIR, "bin", "python"),
+ os.path.join(ENV_DIR, "bin", "pip"), "install", "-v"]
+
# Passes --no-binary for IMPALA-3767: without this, Cython (and
# several other packages) fail download.
#
# --no-cache-dir is used to prevent caching of compiled artifacts, which may be built
# with different compilers or settings.
- cmd = [os.path.join(ENV_DIR, "bin", "python"), os.path.join(ENV_DIR, "bin", "pip"),
- "install", "-v", "--no-binary", ":all:", "--no-cache-dir"]
+ third_party_pkg_install_cmd = \
+ impala_pip_base_cmd[:] + ["--no-binary", ":all:", "--no-cache-dir"]
# When using a custom mirror, we also must use the index of that mirror.
if "PYPI_MIRROR" in os.environ:
- cmd.extend(["--index-url", "%s/simple" % os.environ["PYPI_MIRROR"]])
+ third_party_pkg_install_cmd.extend(["--index-url",
+ "%s/simple" % os.environ["PYPI_MIRROR"]])
else:
# Prevent fetching additional packages from the index. If we forget to add a package
# to one of the requirements.txt files, this should trigger an error. However, we will
# still access the index for version/dependency resolution, hence we need to change it
# when using a private mirror.
- cmd.append("--no-index")
+ third_party_pkg_install_cmd.append("--no-index")
- cmd.extend(["--find-links",
+ third_party_pkg_install_cmd.extend(["--find-links",
"file://%s" % urllib.pathname2url(os.path.abspath(DEPS_DIR))])
- cmd.extend(args)
- exec_cmd(cmd, env=env)
+ third_party_pkg_install_cmd.extend(args)
+ exec_cmd(third_party_pkg_install_cmd, env=env)
+
+ # Finally, we want to install the packages from our own internal python lib
+ local_package_install_cmd = impala_pip_base_cmd + \
+ ['-e', os.path.join(os.getenv('IMPALA_HOME'), 'lib', 'python')]
+ exec_cmd(local_package_install_cmd)
def find_file(*paths):
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/lib/python/impala_py_lib/jenkins/generate_junitxml.py
----------------------------------------------------------------------
diff --git a/lib/python/impala_py_lib/jenkins/generate_junitxml.py b/lib/python/impala_py_lib/jenkins/generate_junitxml.py
index 68bedd1..97dcbdd 100755
--- a/lib/python/impala_py_lib/jenkins/generate_junitxml.py
+++ b/lib/python/impala_py_lib/jenkins/generate_junitxml.py
@@ -32,18 +32,30 @@ from datetime import datetime as dt
IMPALA_HOME = os.getenv('IMPALA_HOME', '.')
SCRIPT_NAME, _ = os.path.splitext(os.path.basename(__file__))
+JUNITXML_LOGDIR = os.path.join(os.getenv("IMPALA_LOGS_DIR", "."), 'extra_junit_xml_logs')
class JunitReport(object):
"""A Junit XML style report parseable by Jenkins for reporting build status.
- Generally, a caller who invokes this script doesn't need to do anything
+ Generally, a caller who invokes this script from bash doesn't need to do
more than supply the necessary command line parameters. The JunitReport
class is instantiated using those initial inputs, and a timestamped XML
file is output to the $IMPALA_HOME/logs/extra_junit_xml_logs/.
Log files are timestamped, so they will not overwrite previous files containing
output of the same step.
+
+ For use from within a python script (must be invoked with impala-python), an
+ example might look like:
+
+ >>> from impala_py_lib.jenkins.generate_junitxml import JunitReport
+ >>> report = JunitReport(phase='load_data', step='load_hbase', error_msg='oops')
+ >>> report.tofile()
+
+ For now, the class does not support adding more than one step (analogous to a
+ test case) to the same phase (analogous to a test suite). Each report should
+ be unique for a given junit XML file. This may be enhanced at some point.
"""
def __init__(self, phase, step, error_msg=None, stdout=None, stderr=None,
@@ -124,7 +136,7 @@ class JunitReport(object):
output = ET.SubElement(self.testcase_element, "system-{}".format(output_type))
output.text = JunitReport.get_xml_content(file_or_string)
- def to_file(self, junitxml_logdir='.'):
+ def to_file(self, junitxml_logdir=JUNITXML_LOGDIR):
"""
Create a timestamped XML report file.
@@ -134,6 +146,15 @@ class JunitReport(object):
Return:
junit_log_file: path to the generated file
"""
+ # The equivalent of mkdir -p
+ try:
+ os.makedirs(junitxml_logdir)
+ except OSError as e:
+ if e.errno == errno.EEXIST and os.path.isdir(junitxml_logdir):
+ pass
+ else:
+ raise
+
filename = '{}.{}.xml'.format(
self.testsuite_element.attrib['name'],
self.utc_time.strftime('%Y%m%d_%H_%M_%S')
@@ -226,17 +247,6 @@ def main():
Phase can be repeated in a given test run, but the step leaf node, which is
equivalent to a "test case", must be unique within each phase.
"""
- junitxml_logdir = os.path.join(IMPALA_HOME, 'logs', 'extra_junit_xml_logs')
-
- # The equivalent of mkdir -p
- try:
- os.makedirs(junitxml_logdir)
- except OSError as e:
- if e.errno == errno.EEXIST and os.path.isdir(junitxml_logdir):
- pass
- else:
- raise
-
options = get_options()
junit_report = JunitReport(phase=options.phase,
@@ -246,8 +256,8 @@ def main():
stderr=options.stderr,
elapsed_time=options.time)
- xml_report = junit_report.to_file(junitxml_logdir)
- print("Generated: {}".format(xml_report))
+ junit_log_file = junit_report.to_file()
+ print("Generated: {0}".format(junit_log_file))
if "__main__" == __name__:
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/shell/make_shell_tarball.sh
----------------------------------------------------------------------
diff --git a/shell/make_shell_tarball.sh b/shell/make_shell_tarball.sh
index 637fa7d..dd6a648 100755
--- a/shell/make_shell_tarball.sh
+++ b/shell/make_shell_tarball.sh
@@ -23,7 +23,8 @@
# ${IMPALA_HOME}/shell/build.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
if [ "x${IMPALA_HOME}" == "x" ]; then
echo "\$IMPALA_HOME must be set"
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/check-schema-diff.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/check-schema-diff.sh b/testdata/bin/check-schema-diff.sh
index 4eb8172..dd36b85 100755
--- a/testdata/bin/check-schema-diff.sh
+++ b/testdata/bin/check-schema-diff.sh
@@ -24,6 +24,10 @@
# - 1 implies that the schemas have changed.
set -euo pipefail
+
+# We don't want to generate a junit xml report for errors generated here,
+# since exit code 1 here denotes something useful. So in the case of this
+# script, we don't call setup_report_build_error.
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/compute-table-stats.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/compute-table-stats.sh b/testdata/bin/compute-table-stats.sh
index 63eb0da..08c7595 100755
--- a/testdata/bin/compute-table-stats.sh
+++ b/testdata/bin/compute-table-stats.sh
@@ -20,7 +20,8 @@
# Runs compute table stats over a curated set of Impala test tables.
#
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/copy-data-sources.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/copy-data-sources.sh b/testdata/bin/copy-data-sources.sh
index a1838ce..1782aca 100755
--- a/testdata/bin/copy-data-sources.sh
+++ b/testdata/bin/copy-data-sources.sh
@@ -20,7 +20,8 @@
# This script copies the test data source library into hdfs.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/copy-udfs-udas.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/copy-udfs-udas.sh b/testdata/bin/copy-udfs-udas.sh
index c1a0454..def7812 100755
--- a/testdata/bin/copy-udfs-udas.sh
+++ b/testdata/bin/copy-udfs-udas.sh
@@ -20,7 +20,8 @@
# This script copies udf/uda binaries into hdfs.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
if [ x${JAVA_HOME} == x ]; then
echo JAVA_HOME not set
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/create-load-data.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/create-load-data.sh b/testdata/bin/create-load-data.sh
index 1953daf..74ae248 100755
--- a/testdata/bin/create-load-data.sh
+++ b/testdata/bin/create-load-data.sh
@@ -29,7 +29,8 @@
# bin/load-data.py
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
. ${IMPALA_HOME}/testdata/bin/run-step.sh
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/create-table-many-blocks.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/create-table-many-blocks.sh b/testdata/bin/create-table-many-blocks.sh
index 4c0a57d..2db9bee 100755
--- a/testdata/bin/create-table-many-blocks.sh
+++ b/testdata/bin/create-table-many-blocks.sh
@@ -25,7 +25,8 @@
# blocks/files.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/generate-load-nested.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/generate-load-nested.sh b/testdata/bin/generate-load-nested.sh
index 4986418..cceea1a 100755
--- a/testdata/bin/generate-load-nested.sh
+++ b/testdata/bin/generate-load-nested.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
SHELL_CMD=${IMPALA_HOME}/bin/impala-shell.sh
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-all.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-all.sh b/testdata/bin/kill-all.sh
index 0e8d201..b6c13a6 100755
--- a/testdata/bin/kill-all.sh
+++ b/testdata/bin/kill-all.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# Shutdown Impala if it is alive
${IMPALA_HOME}/bin/start-impala-cluster.py --kill
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-hbase.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-hbase.sh b/testdata/bin/kill-hbase.sh
index 4e5a42f..0a5fcda 100755
--- a/testdata/bin/kill-hbase.sh
+++ b/testdata/bin/kill-hbase.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
DIR=$(dirname "$0")
echo Stopping Hbase
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-hive-server.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-hive-server.sh b/testdata/bin/kill-hive-server.sh
index 0ebd18c..59c44e5 100755
--- a/testdata/bin/kill-hive-server.sh
+++ b/testdata/bin/kill-hive-server.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
DIR=$(dirname "$0")
echo Stopping Hive
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-java-service.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-java-service.sh b/testdata/bin/kill-java-service.sh
index 4cdc1bd..8ae449d 100755
--- a/testdata/bin/kill-java-service.sh
+++ b/testdata/bin/kill-java-service.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
CLASSES=()
EXTRA_SHUTDOWN_TIME_SECS=1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/kill-sentry-service.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-sentry-service.sh b/testdata/bin/kill-sentry-service.sh
index e9ed7aa..ceb8958 100755
--- a/testdata/bin/kill-sentry-service.sh
+++ b/testdata/bin/kill-sentry-service.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
DIR=$(dirname "$0")
echo Stopping Sentry
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-hive-builtins.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/load-hive-builtins.sh b/testdata/bin/load-hive-builtins.sh
index 061d42b..55cc845 100755
--- a/testdata/bin/load-hive-builtins.sh
+++ b/testdata/bin/load-hive-builtins.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-metastore-snapshot.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/load-metastore-snapshot.sh b/testdata/bin/load-metastore-snapshot.sh
index 1760cdd..dd4e136 100755
--- a/testdata/bin/load-metastore-snapshot.sh
+++ b/testdata/bin/load-metastore-snapshot.sh
@@ -22,7 +22,8 @@
# full data load build.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/load-test-warehouse-snapshot.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/load-test-warehouse-snapshot.sh b/testdata/bin/load-test-warehouse-snapshot.sh
index fe5dd2a..da5fb07 100755
--- a/testdata/bin/load-test-warehouse-snapshot.sh
+++ b/testdata/bin/load-test-warehouse-snapshot.sh
@@ -25,7 +25,8 @@
# to backup any data you need before running this script.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/impala-config.sh > /dev/null 2>&1
: ${REMOTE_LOAD:=}
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-all.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh
index 6820e5d..1e14315 100755
--- a/testdata/bin/run-all.sh
+++ b/testdata/bin/run-all.sh
@@ -20,7 +20,8 @@
# Starts up a mini-dfs test cluster and related services
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# If -format is passed, format the mini-dfs cluster.
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-hbase.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-hbase.sh b/testdata/bin/run-hbase.sh
index 1433073..e7d67c9 100755
--- a/testdata/bin/run-hbase.sh
+++ b/testdata/bin/run-hbase.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
CLUSTER_BIN=${IMPALA_HOME}/testdata/bin
HBASE_JAAS_CLIENT=${HBASE_CONF_DIR}/hbase-jaas-client.conf
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-hive-server.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-hive-server.sh b/testdata/bin/run-hive-server.sh
index 2b5a486..fbeba72 100755
--- a/testdata/bin/run-hive-server.sh
+++ b/testdata/bin/run-hive-server.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/set-pythonpath.sh
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-mini-dfs.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-mini-dfs.sh b/testdata/bin/run-mini-dfs.sh
index ea6c519..be63715 100755
--- a/testdata/bin/run-mini-dfs.sh
+++ b/testdata/bin/run-mini-dfs.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
if [[ $# -eq 1 && "$1" == -format ]]; then
SHOULD_FORMAT=true
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/run-sentry-service.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/run-sentry-service.sh b/testdata/bin/run-sentry-service.sh
index 755c382..f49f88b 100755
--- a/testdata/bin/run-sentry-service.sh
+++ b/testdata/bin/run-sentry-service.sh
@@ -18,7 +18,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
. ${IMPALA_HOME}/bin/set-classpath.sh
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/bin/setup-hdfs-env.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/setup-hdfs-env.sh b/testdata/bin/setup-hdfs-env.sh
index 552c48b..4308757 100755
--- a/testdata/bin/setup-hdfs-env.sh
+++ b/testdata/bin/setup-hdfs-env.sh
@@ -18,7 +18,8 @@
# under the License.
#
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
: ${REMOTE_LOAD:=}
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/cluster/admin
----------------------------------------------------------------------
diff --git a/testdata/cluster/admin b/testdata/cluster/admin
index 534bc45..ca438db 100755
--- a/testdata/cluster/admin
+++ b/testdata/cluster/admin
@@ -28,7 +28,8 @@
# TODO: Run each node on its own IP address, e.g. 127.0.0.1, 127.0.0.2, and so on.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
: ${IMPALA_KERBERIZE=}
: ${INCLUDE_YARN=}
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/datasets/tpcds/preload
----------------------------------------------------------------------
diff --git a/testdata/datasets/tpcds/preload b/testdata/datasets/tpcds/preload
index 631a1c2..423b3c8 100755
--- a/testdata/datasets/tpcds/preload
+++ b/testdata/datasets/tpcds/preload
@@ -17,7 +17,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
IMPALA_DATA=${IMPALA_HOME}/testdata/impala-data
TPC_DS_DATA=${IMPALA_DATA}/tpcds
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/testdata/datasets/tpch/preload
----------------------------------------------------------------------
diff --git a/testdata/datasets/tpch/preload b/testdata/datasets/tpch/preload
index 2b0cbb6..619a4d2 100755
--- a/testdata/datasets/tpch/preload
+++ b/testdata/datasets/tpch/preload
@@ -17,7 +17,8 @@
# under the License.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
IMPALA_DATA=${IMPALA_HOME}/testdata/impala-data
TPC_H_DATA=${IMPALA_DATA}/tpch
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/tests/run-custom-cluster-tests.sh
----------------------------------------------------------------------
diff --git a/tests/run-custom-cluster-tests.sh b/tests/run-custom-cluster-tests.sh
index ce161b1..6b77e26 100755
--- a/tests/run-custom-cluster-tests.sh
+++ b/tests/run-custom-cluster-tests.sh
@@ -21,7 +21,8 @@
# clusters will be restarted.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# Disable HEAPCHECK for the process failure tests because they can cause false positives.
# TODO: Combine with run-process-failure-tests.sh
http://git-wip-us.apache.org/repos/asf/impala/blob/6e5ec22b/tests/run-process-failure-tests.sh
----------------------------------------------------------------------
diff --git a/tests/run-process-failure-tests.sh b/tests/run-process-failure-tests.sh
index 9ab531c..db47571 100755
--- a/tests/run-process-failure-tests.sh
+++ b/tests/run-process-failure-tests.sh
@@ -20,7 +20,8 @@
# Runs the Impala process failure tests.
set -euo pipefail
-trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+. $IMPALA_HOME/bin/report_build_error.sh
+setup_report_build_error
# Disable HEAPCHECK for the process failure tests because they can cause false positives.
export HEAPCHECK=