You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/12/10 06:59:54 UTC
[2/5] yetus git commit: YETUS-713. pylint: rewrite to make it easier
to maintain and fix stderr output in brief report
YETUS-713. pylint: rewrite to make it easier to maintain and fix stderr output in brief report
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/8a502dc0
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/8a502dc0
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/8a502dc0
Branch: refs/heads/master
Commit: 8a502dc055b7a133a24bca45eb429f43f29834e8
Parents: 671b88f
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Nov 12 15:24:00 2018 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Sun Dec 9 22:25:43 2018 -0800
----------------------------------------------------------------------
.../src/main/shell/test-patch.d/briefreport.sh | 34 +++---
precommit/src/main/shell/test-patch.d/pylint.sh | 110 ++++++++++---------
2 files changed, 79 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/briefreport.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/briefreport.sh b/precommit/src/main/shell/test-patch.d/briefreport.sh
index e536b97..caeaaf0 100755
--- a/precommit/src/main/shell/test-patch.d/briefreport.sh
+++ b/precommit/src/main/shell/test-patch.d/briefreport.sh
@@ -96,6 +96,7 @@ function briefreport_finalreport
declare hours
declare newtime
declare havelogs=false
+ declare fn
if [[ -z "${BRIEFOUT_REPORTFILE}" ]]; then
return
@@ -103,11 +104,10 @@ function briefreport_finalreport
big_console_header "Writing Brief Text Report to ${BRIEFOUT_REPORTFILE}"
-
if [[ ${result} == 0 ]]; then
- printf "\n\n+1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+ printf '\n\n+1 overall\n\n' > "${BRIEFOUT_REPORTFILE}"
else
- printf "\n\n-1 overall\n\n" > "${BRIEFOUT_REPORTFILE}"
+ printf '\n\n-1 overall\n\n' > "${BRIEFOUT_REPORTFILE}"
fi
i=0
@@ -137,11 +137,14 @@ function briefreport_finalreport
((i=i+1))
done
- tmparray=($(printf "%s\n" "${failed[@]}" | sort -u))
+ #shellcheck disable=SC2207
+ tmparray=($(printf '%s\n' "${failed[@]}" | sort -u))
failed=("${tmparray[@]}")
- tmparray=($(printf "%s\n" "${filtered[@]}" | sort -u))
+ #shellcheck disable=SC2207
+ tmparray=($(printf '%s\n' "${filtered[@]}" | sort -u))
filtered=("${tmparray[@]}")
- tmparray=($(printf "%s\n" "${long[@]}" | sort -u))
+ #shellcheck disable=SC2207
+ tmparray=($(printf '%s\n' "${long[@]}" | sort -u))
long=("${tmparray[@]}")
if [[ ${#failed[@]} -gt 0 ]]; then
@@ -198,11 +201,11 @@ function briefreport_finalreport
{
if [[ -n "${vote// }" ]]; then
echo ""
- printf " %s:\n" "${vote}"
+ printf ' %s:\n' "${vote}"
echo ""
vote=""
fi
- printf " %s\n" "${subs}"
+ printf ' %s\n' "${subs}"
} >> "${BRIEFOUT_REPORTFILE}"
((i=i+1))
done
@@ -231,22 +234,25 @@ function briefreport_finalreport
subs=$(echo "${TP_FOOTER_TABLE[${i}]}" | cut -f2 -d\|)
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
cut -f3 -d\| |
- ${SED} -e "s,@@BASE@@,${PATCH_DIR},g")
- # shellcheck disable=SC2016
- size=$(du -sh "${comment// }" | ${AWK} '{print $1}')
+ "${SED}" -e "s,@@BASE@@,${PATCH_DIR},g")
+ fn="${comment// }"
+ if [[ -f ${fn} ]]; then
+ # shellcheck disable=SC2016
+ size=$(du -sh "${fn}" | "${AWK}" '{print $1}')
+ fi
if [[ -n "${BUILD_URL}" ]]; then
comment=$(echo "${TP_FOOTER_TABLE[${i}]}" |
cut -f3 -d\| |
- ${SED} -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
+ "${SED}" -e "s,@@BASE@@,${BUILD_URL}${BUILD_URL_ARTIFACTS},g")
fi
{
if [[ "${subs}" != "${vote}" ]]; then
echo ""
- printf " %s:\n" "${subs// }"
+ printf ' %s:\n' "${subs// }"
echo ""
vote=${subs}
fi
- printf " %s [%s]\n" "${comment}" "${size}"
+ printf ' %s [%s]\n' "${comment}" "${size}"
} >> "${BRIEFOUT_REPORTFILE}"
fi
((i=i+1))
http://git-wip-us.apache.org/repos/asf/yetus/blob/8a502dc0/precommit/src/main/shell/test-patch.d/pylint.sh
----------------------------------------------------------------------
diff --git a/precommit/src/main/shell/test-patch.d/pylint.sh b/precommit/src/main/shell/test-patch.d/pylint.sh
index facba6c..d5a0848 100755
--- a/precommit/src/main/shell/test-patch.d/pylint.sh
+++ b/precommit/src/main/shell/test-patch.d/pylint.sh
@@ -14,17 +14,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# SHELLDOC-IGNORE
+
add_test_type pylint
PYLINT_TIMER=0
-PYLINT=${PYLINT:-$(which pylint 2>/dev/null)}
+PYLINT=${PYLINT:-$(command -v pylint 2>/dev/null)}
+# backward compatibility, do not use
PYLINT_OPTIONS=${PYLINT_OPTIONS:-}
function pylint_usage
{
yetus_add_option "--pylint=<path>" "path to pylint executable"
- yetus_add_option "--pylint-options=<path>" "pylint options other than output-format and reports"
+ yetus_add_option "--pylint-rcfile=<path>" "pylint configuration file"
}
function pylint_parse_args
@@ -36,7 +39,11 @@ function pylint_parse_args
--pylint=*)
PYLINT=${i#*=}
;;
+ --pylint-rcfile=*)
+ PYLINT_RCFILE=${i#*=}
+ ;;
--pylint-options=*)
+ # backward compatibility
PYLINT_OPTIONS=${i#*=}
;;
esac
@@ -60,52 +67,80 @@ function pylint_precheck
fi
}
-
-function pylint_preapply
+function pylint_executor
{
- local i
- local count
- local pylintStderr=branch-pylint-stderr.txt
+ declare repostatus=$1
+ declare i
+ declare count
+ declare pylintStderr=${repostatus}-pylint-stderr.txt
if ! verify_needed_test pylint; then
return 0
fi
- big_console_header "pylint plugin: ${PATCH_BRANCH}"
+ big_console_header "pylint plugin: ${BUILDMODE}"
start_clock
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${PYLINT_TIMER}"
+
+ # backward compatibility
+ # shellcheck disable=SC2206
+ pylintopts=(${PYLINT_OPTIONS})
+
+ if [[ -n "${PYLINT_RCFILE}" ]] && [[ -f "${PYLINT_RCFILE}" ]]; then
+ pylintopts+=('--rcfile='"${PYLINT_RCFILE}")
+ fi
+
+ pylintops+=('--persistent=n')
+ pylintops+=('--reports=n')
+ pylintops+=('--score=n')
+
echo "Running pylint against identified python scripts."
- pushd "${BASEDIR}" >/dev/null
+ pushd "${BASEDIR}" >/dev/null || return 1
for i in "${CHANGED_FILES[@]}"; do
if [[ ${i} =~ \.py$ && -f ${i} ]]; then
- # shellcheck disable=SC2086
- eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \
- 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/branch-pylint-result.txt"
+ "${PYLINT}" "${pylintopts[@]}" --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' "${i}" \
+ 2>>"${PATCH_DIR}/${pylintStderr}" | "${AWK}" '1<NR' >> "${PATCH_DIR}/${repostatus}-pylint-result.txt"
fi
done
if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then
- count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
+ count=$("${GREP}" -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
if [[ ${count} -gt 0 ]]; then
- add_footer_table pylint "${PATCH_BRANCH} stderr: @@BASE@@/${pylintStderr}"
+ add_vote_table -1 pylint "Error running pylint. Please check pylint stderr files."
+ add_footer_table pylint "@@BASE@@/${pylintStderr}"
return 1
fi
fi
rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null
- popd >/dev/null
+ popd >/dev/null || return 1
+ return 0
+}
+
+
+function pylint_preapply
+{
+ declare retval
+
+ if ! verify_needed_test pylint; then
+ return 0
+ fi
+
+ pylint_executor "branch"
+ retval=$?
+
# keep track of how much as elapsed for us already
PYLINT_TIMER=$(stop_clock)
- return 0
+ return ${retval}
}
function pylint_postapply
{
- declare i
- declare count
declare numPrepatch
declare numPostpatch
declare diffPostpatch
- declare pylintStderr=patch-pylint-stderr.txt
declare fixedpatch
declare statstring
@@ -113,46 +148,19 @@ function pylint_postapply
return 0
fi
- big_console_header "pylint plugin: ${BUILDMODE}"
-
- start_clock
-
- # add our previous elapsed to our new timer
- # by setting the clock back
- offset_clock "${PYLINT_TIMER}"
-
- echo "Running pylint against identified python scripts."
- # we re-check this in case one has been added
- pushd "${BASEDIR}" >/dev/null
- for i in "${CHANGED_FILES[@]}"; do
- if [[ ${i} =~ \.py$ && -f ${i} ]]; then
- # shellcheck disable=SC2086
- eval "${PYLINT} ${PYLINT_OPTIONS} --msg-template='{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}' --reports=n ${i}" \
- 2>>${PATCH_DIR}/${pylintStderr} | ${AWK} '1<NR' >> "${PATCH_DIR}/patch-pylint-result.txt"
- fi
- done
- if [[ -f ${PATCH_DIR}/${pylintStderr} ]]; then
- count=$(${GREP} -Evc "^(No config file found|Using config file)" "${PATCH_DIR}/${pylintStderr}")
- if [[ ${count} -gt 0 ]]; then
- add_vote_table -1 pylint "Something bad seems to have happened in running pylint. Please check pylint stderr files."
- add_footer_table pylint "${BUILDMODEMSG} stderr: @@BASE@@/${pylintStderr}"
- return 1
- fi
- fi
- rm "${PATCH_DIR}/${pylintStderr}" 2>/dev/null
- popd >/dev/null
+ pylint_executor patch
# shellcheck disable=SC2016
- PYLINT_VERSION=$(${PYLINT} --version 2>/dev/null | ${GREP} pylint | ${AWK} '{print $NF}')
+ PYLINT_VERSION=$("${PYLINT}" --version 2>/dev/null | "${GREP}" pylint | "${AWK}" '{print $NF}')
add_footer_table pylint "v${PYLINT_VERSION%,}"
calcdiffs "${PATCH_DIR}/branch-pylint-result.txt" \
"${PATCH_DIR}/patch-pylint-result.txt" \
pylint > "${PATCH_DIR}/diff-patch-pylint.txt"
- numPrepatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/branch-pylint-result.txt")
- numPostpatch=$(${GREP} -c "^.*:.*: \[.*\] " "${PATCH_DIR}/patch-pylint-result.txt")
+ numPrepatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/branch-pylint-result.txt")
+ numPostpatch=$("${GREP}" -c '^.*:.*: \[.*\] ' "${PATCH_DIR}/patch-pylint-result.txt")
# Exclude Pylint messages from the information category to avoid false positives (see YETUS-309).
- diffPostpatch=$(${GREP} -c "^.*:.*: \[[^I].*\] " "${PATCH_DIR}/diff-patch-pylint.txt")
+ diffPostpatch=$("${GREP}" -c '^.*:.*: \[[^I].*\] ' "${PATCH_DIR}/diff-patch-pylint.txt")
((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))