You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2018/07/17 16:53:27 UTC
[7/7] impala git commit: IMPALA-7199: Add scripts to create code
coverage reports
IMPALA-7199: Add scripts to create code coverage reports
gcovr is a python library that uses gcov to generate
code coverage reports. This adds gcovr to the python
dependencies and adds bin/impala-gcovr to provide
easy access to gcovr's command line. gcovr 3.4
supports python 2.6+.
This also adds bin/coverage_helper.sh to provide a
simplified interface to generate reports and zero
coverage counters.
Code coverage data is written out when a program
exits, so it is important to avoid hard kills
to shut down the impalads when generating coverage.
This modifies testdata/bin/kill-all.sh to call
start-impala-cluster.py --kill when shutting down
the minicluster to try to avoid doing a hard kill.
It will still do a hard kill if impala is still
running after the softer kill.
Change-Id: I5b2e0b794c64f9343ec976de7a3f235e54d2badd
Reviewed-on: http://gerrit.cloudera.org:8080/10791
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/7f3f6342
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/7f3f6342
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/7f3f6342
Branch: refs/heads/master
Commit: 7f3f63424f1a82390386ab1a21408d0b820e6c81
Parents: 6887fc2
Author: Joe McDonnell <jo...@cloudera.com>
Authored: Thu May 24 14:58:16 2018 -0700
Committer: Joe McDonnell <jo...@cloudera.com>
Committed: Tue Jul 17 16:45:44 2018 +0000
----------------------------------------------------------------------
bin/coverage_helper.sh | 86 +++++++++++++++++++++++++++++++++
bin/impala-gcovr | 21 ++++++++
infra/python/deps/requirements.txt | 1 +
testdata/bin/kill-all.sh | 3 ++
4 files changed, 111 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/7f3f6342/bin/coverage_helper.sh
----------------------------------------------------------------------
diff --git a/bin/coverage_helper.sh b/bin/coverage_helper.sh
new file mode 100755
index 0000000..dad1498
--- /dev/null
+++ b/bin/coverage_helper.sh
@@ -0,0 +1,86 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -eu -o pipefail
+
+ZERO_COUNTERS_ACTION=0
+REPORT_ACTION=0
+REPORT_DIRECTORY=${IMPALA_HOME}/logs/coverage
+
+function print_usage {
+ echo "coverage_helper.sh - Driver script for coverage"
+ echo "[-zerocounters] : Reset coverage counters"
+ echo "[-report] : Generate a detailed coverage report"
+ echo "[-reportdirectory <directory>] : Output directory for coverage report files"
+}
+
+while [ -n "$*" ]
+do
+ case "$1" in
+ -zerocounters)
+ ZERO_COUNTERS_ACTION=1
+ ;;
+ -report)
+ REPORT_ACTION=1
+ ;;
+ -reportdirectory)
+ REPORT_DIRECTORY="${2-}"
+ shift
+ ;;
+ -help|*)
+ print_usage
+ exit 1
+ ;;
+ esac
+ shift
+done
+
+if [[ ${ZERO_COUNTERS_ACTION} -eq 0 && ${REPORT_ACTION} -eq 0 ]]; then
+ print_usage
+ exit 1
+fi
+
+if pgrep -U "$USER" impalad; then
+ echo "Warning: impalad is running. Coverage counters are only updated when"
+ echo "a program exits. Any report will not include the information from"
+ echo "the running impalad. Similarly, zeroing the counters will not zero"
+ echo "the counters for the running impalad."
+fi
+
+if [ ${REPORT_ACTION} -eq 1 ]; then
+ mkdir -p "${REPORT_DIRECTORY}"
+ rm -f "${REPORT_DIRECTORY}"/index*.html
+ # src/util/bit-packing.inline.h gets lots of hits, so generating a detailed report
+ # for it takes several minutes. Exclude it to keep the execution time down.
+ # gcovr excludes are buggy, so on some environments these excludes won't work.
+ echo "Generating report in directory: ${REPORT_DIRECTORY}"
+ cd "${IMPALA_HOME}"
+ "${IMPALA_HOME}/bin/impala-gcovr" -v -r "${IMPALA_HOME}/be" \
+ --exclude=".*src/benchmarks.*" \
+ --exclude=".*generated-sources/gen-cpp.*" \
+ --exclude=".*src/util/bit-packing.inline.h.*" \
+ --html --html-details -o "${REPORT_DIRECTORY}/index.html" > "${REPORT_DIRECTORY}/gcovr.out"
+fi
+
+if [ ${ZERO_COUNTERS_ACTION} -eq 1 ]; then
+ # The .gcda files contain the counters for coverage. Deleting them zeros the
+ # the counters.
+ echo "Zeroing Counters"
+ find "${IMPALA_HOME}/be" -name "*.gcda" -delete
+fi
http://git-wip-us.apache.org/repos/asf/impala/blob/7f3f6342/bin/impala-gcovr
----------------------------------------------------------------------
diff --git a/bin/impala-gcovr b/bin/impala-gcovr
new file mode 100755
index 0000000..6e8be6c
--- /dev/null
+++ b/bin/impala-gcovr
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source "$(dirname "$0")/impala-python-common.sh"
+exec "$PY_DIR/env/bin/gcovr" "$@"
http://git-wip-us.apache.org/repos/asf/impala/blob/7f3f6342/infra/python/deps/requirements.txt
----------------------------------------------------------------------
diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt
index b0b7f23..a92fdc6 100644
--- a/infra/python/deps/requirements.txt
+++ b/infra/python/deps/requirements.txt
@@ -43,6 +43,7 @@ Flask == 0.10.1
MarkupSafe == 0.23
Werkzeug == 0.11.3
itsdangerous == 0.24
+gcovr == 3.4
kazoo == 2.2.1
ordereddict == 1.1
pexpect == 3.3
http://git-wip-us.apache.org/repos/asf/impala/blob/7f3f6342/testdata/bin/kill-all.sh
----------------------------------------------------------------------
diff --git a/testdata/bin/kill-all.sh b/testdata/bin/kill-all.sh
index 5462162..0e8d201 100755
--- a/testdata/bin/kill-all.sh
+++ b/testdata/bin/kill-all.sh
@@ -20,6 +20,9 @@
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)' ERR
+# Shutdown Impala if it is alive
+${IMPALA_HOME}/bin/start-impala-cluster.py --kill
+
# Kill HBase, then MiniLlama (which includes a MiniDfs, a Yarn RM several NMs).
$IMPALA_HOME/testdata/bin/kill-sentry-service.sh
$IMPALA_HOME/testdata/bin/kill-hive-server.sh