You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2021/11/16 15:06:55 UTC
[yetus] branch main updated: YETUS-1136. detect-secrets is crashing (#235)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 477eeec YETUS-1136. detect-secrets is crashing (#235)
477eeec is described below
commit 477eeecec11d18b3f4e10bb25b1b5987970f2deb
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Tue Nov 16 07:06:48 2021 -0800
YETUS-1136. detect-secrets is crashing (#235)
---
.yetus/detsecrets-ignored-hashes.txt | 1 +
asf-site-src/Dockerfile | 2 +-
.../precommit/plugins/detsecrets.html.md | 7 ++---
precommit/src/main/shell/plugins.d/detsecrets.sh | 36 +++++++++++-----------
precommit/src/main/shell/plugins.d/github.sh | 2 +-
precommit/src/main/shell/plugins.d/gitlab.sh | 10 +++---
.../src/main/shell/test-patch-docker/Dockerfile | 5 ++-
7 files changed, 31 insertions(+), 32 deletions(-)
diff --git a/.yetus/detsecrets-ignored-hashes.txt b/.yetus/detsecrets-ignored-hashes.txt
index 5a25105..feff2e4 100644
--- a/.yetus/detsecrets-ignored-hashes.txt
+++ b/.yetus/detsecrets-ignored-hashes.txt
@@ -23,3 +23,4 @@ d8e18255635efa84e79bb9eaf2ffad3371a95288
301c198f635f0ec01ed5046598bff06f13313a17
5dfd74f05dcaf3b8d23bb351915be66b4d2628a4
1a1ce1ac1c87cc884ba59133f405f0d45b201d1f
+3b3c84f10a63cb8e95b93ea5f46e02d885153f5d
diff --git a/asf-site-src/Dockerfile b/asf-site-src/Dockerfile
index 6ce0d39..99f44bc 100644
--- a/asf-site-src/Dockerfile
+++ b/asf-site-src/Dockerfile
@@ -50,7 +50,7 @@ RUN echo "export GPG_TTY=\$(tty)" >> /root/.bashrc
RUN groupadd --non-unique -g "${GROUP_ID}" "${USER_NAME}" \
&& useradd -l -g "${GROUP_ID}" -u "${USER_ID}" -k /root -m "${USER_NAME}"
# hadolint disable=DL3059,SC2039,SC1117
-RUN echo -e "${USER_NAME}\\tALL=NOPASSWD: ALL" > "/etc/sudoers.d/yetus-build-${USER_ID}"
+RUN echo -e "${USER_NAME}\\tALL=NOPASSWD: ALL" > "/etc/sudoers.d/yetus-build-${USER_ID}" # pragma: allowlist secret
ENV HOME /home/${USER_NAME}
WORKDIR /home/${USER_NAME}
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md
index 05dbd5b..b4e63bd 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md
@@ -27,9 +27,7 @@ Test
# Description
-Runs Yelp's [detect-secrets](https://github.com/Yelp/detect-secrets) or
-IBM's forked [detect-secrets](https://github.com/IBM/detect-secrets). If `--detsecrets-baseline` is
-provided, it will effectively use that as an exception file in addition to the normal exception handling.
+Runs IBM's version of [detect-secrets](https://github.com/IBM/detect-secrets).
NOTE: This test also requires a working Python 3.4+ interpreter available on the path. It will be called first
as `python3` and secondarily as `python`.
@@ -50,7 +48,8 @@ None
# Docker Notes
-Currently, the Yelp version is provided but that may change in the future depending upon how the project shakes out.
+While the original project came from Yelp, the IBM version is provided due to instability with the Yelp version.
+That may change in the future depending upon how the project shakes out.
# Developer Notes
diff --git a/precommit/src/main/shell/plugins.d/detsecrets.sh b/precommit/src/main/shell/plugins.d/detsecrets.sh
index d9d38f2..3c5e9f2 100755
--- a/precommit/src/main/shell/plugins.d/detsecrets.sh
+++ b/precommit/src/main/shell/plugins.d/detsecrets.sh
@@ -27,7 +27,6 @@ DETSECRETS_FILES='' #regex of files to ignore
DETSECRETS_LINES='' #regex of lines to ignore
DETSECRETS_SECRETS='' #regex of secrets to ignore
DETSECRETS_HASHFILE='.yetus/detsecrets-ignored-hashes.txt'
-DETSECRETS_OLD='false'
function detsecrets_usage
{
@@ -82,10 +81,6 @@ function detsecrets_precheck
# shellcheck disable=SC2016
DETSECRETS_VERSION=$("${DETSECRETS}" --version 2>/dev/null| "${AWK}" '{print $NF}')
-
- if [[ ${DETSECRETS_VERSION} =~ /^0 ]]; then
- DETSECRETS_OLD='true'
- fi
}
function detsecrets_calcdiffs
@@ -98,6 +93,14 @@ function detsecrets_calcdiffs
function detsecrets_convert_json_to_flat
{
declare repostatus=$1
+ declare filename
+ declare tmpfile
+
+ tmpfile="${PATCH_DIR}/detsecrets.$$"
+
+ for filename in "${CHANGED_FILES[@]}"; do
+ echo "${filename}" >> "${tmpfile}"
+ done
if [[ -f "${PATCH_DIR}/excluded.txt" ]]; then
stripcmd=("${GREP}" "-v" "-f" "${PATCH_DIR}/excluded.txt")
@@ -113,8 +116,11 @@ function detsecrets_convert_json_to_flat
"${pythonexec}" "${BINDIR}/plugins.d/detsecrets_parse.py" \
"${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
"${DETSECRETS_HASHFILE}" \
+ | "${GREP}" "-f" "${tmpfile}" \
| "${stripcmd[@]}" \
> "${PATCH_DIR}/${repostatus}-detsecrets-result.txt"
+
+ rm "${tmpfile}"
}
function detsecrets_executor
@@ -140,7 +146,7 @@ function detsecrets_executor
echo "Running detect-secrets against source tree."
pushd "${BASEDIR}" >/dev/null || return 1
- detsecretsopts=()
+ detsecretsopts=(--all-files)
if [[ -n "${DETSECRETS_FILES}" ]]; then
detsecretsopts=("${detsecretsopts[@]}" "--exclude-files" "${DETSECRETS_FILES}")
@@ -149,22 +155,16 @@ function detsecrets_executor
if [[ -n "${DETSECRETS_LINES}" ]]; then
detsecretsopts=("${detsecretsopts[@]}" "--exclude-lines" "${DETSECRETS_LINES}")
fi
+
if [[ -n "${DETSECRETS_SECRETS}" ]]; then
detsecretsopts=("${detsecretsopts[@]}" "--exclude-secrets" "${DETSECRETS_SECRETS}")
fi
- if [[ ${DETSECRETS_OLD} == 'false' ]]; then
- "${DETSECRETS}" "${detsecretsopts[@]}" scan \
- --all-files \
- "${detsecretsopts[@]}" \
- > "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
- 2>"${PATCH_DIR}/${detsecretsStderr}"
- else
- "${DETSECRETS}" "${detsecretsopts[@]}" scan \
- "${detsecretsopts[@]}" \
- > "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
- 2>"${PATCH_DIR}/${detsecretsStderr}"
- fi
+ "${DETSECRETS}" scan \
+ "${detsecretsopts[@]}" \
+ "${BASEDIR}" \
+ > "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
+ 2>"${PATCH_DIR}/${detsecretsStderr}"
detsecrets_convert_json_to_flat "${repostatus}"
diff --git a/precommit/src/main/shell/plugins.d/github.sh b/precommit/src/main/shell/plugins.d/github.sh
index 39a1df2..05192c1 100755
--- a/precommit/src/main/shell/plugins.d/github.sh
+++ b/precommit/src/main/shell/plugins.d/github.sh
@@ -185,7 +185,7 @@ function github_brute_force_repo_on_remote
function github_initialize
{
if [[ -n "${GITHUB_TOKEN}" ]]; then
- GITHUB_AUTH=(-H "Authorization: token ${GITHUB_TOKEN}")
+ GITHUB_AUTH=(-H "Authorization: token ${GITHUB_TOKEN}") # pragma: allowlist secret
fi
GITHUB_REPO=${GITHUB_REPO:-${GITHUB_REPO_DEFAULT}}
diff --git a/precommit/src/main/shell/plugins.d/gitlab.sh b/precommit/src/main/shell/plugins.d/gitlab.sh
index 2736640..18b5b1d 100755
--- a/precommit/src/main/shell/plugins.d/gitlab.sh
+++ b/precommit/src/main/shell/plugins.d/gitlab.sh
@@ -222,9 +222,9 @@ function gitlab_locate_mr_patch
echo "${GITLAB_BASE_URL}/${GITLAB_REPO}/merge_requests/${input}"
if [[ -n "${GITLAB_TOKEN}" ]]; then
- gitlabauth="Private-Token: ${GITLAB_TOKEN}"
+ gitlabauth="Private-Token: ${GITLAB_TOKEN}" # pragma: allowlist secret
else
- gitlabauth="X-ignore-me: fake"
+ gitlabauth="X-ignore-me: fake" # pragma: allowlist secret
fi
# Let's merge the MR JSON for later use
@@ -287,9 +287,9 @@ function gitlab_locate_sha_patch
GITLAB_COMMITSHA=${input#GLSHA:}
if [[ -n "${GITLAB_TOKEN}" ]]; then
- gitlabauth="Private-Token: ${GITLAB_TOKEN}"
+ gitlabauth="Private-Token: ${GITLAB_TOKEN}" # pragma: allowlist secret
else
- gitlabauth="X-ignore-me: fake"
+ gitlabauth="X-ignore-me: fake" # pragma: allowlist secret
fi
# Let's merge the MR JSON for later use
@@ -383,7 +383,7 @@ function gitlab_write_comment
} > "${restfile}"
if [[ -n "${GITLAB_TOKEN}" ]]; then
- gitlabauth="Private-Token: ${GITLAB_TOKEN}"
+ gitlabauth="Private-Token: ${GITLAB_TOKEN}" # pragma: allowlist secret
else
echo "Gitlab Plugin: no credentials provided to write a comment."
return 0
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index fdb6c59..88823ac 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -315,7 +315,6 @@ ARG PY3_ANSIBLE_VERSION=4.7.0
ARG PY3_ANSIBLELINT_VERSION=5.2.1
ARG PY3_ASTROID_VERSION=2.8.0
ARG PY3_CODESPELL_VERSION=2.1.0
-ARG PY3_DETECT_SECRETS=1.0.3
ARG PY3_DOCKER_COMPOSE=1.29.2
ARG PY3_PYLINT_VERSION=2.11.1
ARG PY3_YAMLLINT_VERSION=1.26.3
@@ -347,11 +346,11 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
ansible-lint==$PY3_ANSIBLELINT_VERSION \
astroid==$PY3_ASTROID_VERSION \
codespell==$PY3_CODESPELL_VERSION \
- detect-secrets==$PY3_DETECT_SECRETS \
docker-compose==$PY3_DOCKER_COMPOSE \
pylint==$PY3_PYLINT_VERSION \
yamllint==$PY3_YAMLLINT_VERSION \
-
+ && pip3 install --no-cache-dir -v \
+ git+https://github.com/IBM/detect-secrets@0.13.1+ibm.46.dss \
&& rm -rf /root/.cache \
&& mv /usr/local/bin/pylint /usr/local/bin/pylint3 \
&& ln -s /usr/local/bin/pylint3 /usr/local/bin/pylint \