You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@yetus.apache.org by aw...@apache.org on 2021/11/01 06:34:55 UTC
[yetus] branch main updated: YETUS-1123. Add support for
detect-secrets (#230)
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/yetus.git
The following commit(s) were added to refs/heads/main by this push:
new 2a90d10 YETUS-1123. Add support for detect-secrets (#230)
2a90d10 is described below
commit 2a90d10ef02796e63b50a370240082f53b86c122
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Sun Oct 31 23:34:50 2021 -0700
YETUS-1123. Add support for detect-secrets (#230)
---
.yetus/detsecrets-ignored-hashes.txt | 25 +++
.../in-progress/precommit/index.html.md | 1 +
.../precommit/plugins/detsecrets.html.md | 57 ++++++
precommit/src/main/shell/plugins.d/detsecrets.sh | 228 +++++++++++++++++++++
.../src/main/shell/plugins.d/detsecrets_parse.py | 58 ++++++
.../src/main/shell/test-patch-docker/Dockerfile | 4 +-
6 files changed, 371 insertions(+), 2 deletions(-)
diff --git a/.yetus/detsecrets-ignored-hashes.txt b/.yetus/detsecrets-ignored-hashes.txt
new file mode 100644
index 0000000..5a25105
--- /dev/null
+++ b/.yetus/detsecrets-ignored-hashes.txt
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+d2df5f21ba6acb0c2d48b94519e9551d37769900
+51de2b835bd35a67eb32dbcd3d77d4b96e5aa39d
+827d9fff5a87952085163f1f1d3cb58f51a7909b
+f17f2d6f8527bb25a0a1bb7e73f319eaae2a3d78
+114678a8310a403fac03b8a9bbd3fa62b4c6a521
+d8e18255635efa84e79bb9eaf2ffad3371a95288
+50d23ceafd8a071f5f82bd2e13425995279b95ee
+301c198f635f0ec01ed5046598bff06f13313a17
+5dfd74f05dcaf3b8d23bb351915be66b4d2628a4
+1a1ce1ac1c87cc884ba59133f405f0d45b201d1f
diff --git a/asf-site-src/source/documentation/in-progress/precommit/index.html.md b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
index 885d53d..2320ee8 100644
--- a/asf-site-src/source/documentation/in-progress/precommit/index.html.md
+++ b/asf-site-src/source/documentation/in-progress/precommit/index.html.md
@@ -139,6 +139,7 @@ Language Support, Licensing, and more:
* [checkmake](plugins/checkmake)
* [checkstyle](plugins/checkstyle)
* [codespell](plugins/codespell)
+* [detect-secrets](plugins/detsecrets)
* [golangci-lint](plugins/golangcilint)
* [hadolint](plugins/hadolint)
* [jshint](plugins/jshint)
diff --git a/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md b/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md
new file mode 100644
index 0000000..05dbd5b
--- /dev/null
+++ b/asf-site-src/source/documentation/in-progress/precommit/plugins/detsecrets.html.md
@@ -0,0 +1,57 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Name
+
+detsecrets
+
+# Category
+
+Test
+
+# Description
+
+Runs Yelp's [detect-secrets](https://github.com/Yelp/detect-secrets) or
+IBM's forked [detect-secrets](https://github.com/IBM/detect-secrets). If `--detsecrets-baseline` is
+provided, it will effectively use that as an exception file in addition to the normal exception handling.
+
+NOTE: This test also requires a working Python 3.4+ interpreter available on the path. It will be called first
+as `python3` and secondarily as `python`.
+
+# Environment Variables
+
+None
+
+# Options
+
+| Option | Notes |
+|:---------|:------|
+| `--detsecrets=<file>` | Location of the `detect-secrets` binary if it is not on the path. Default is 'detect-secrets'. |
+| `--detsecrets-files=<regex>` | Regex of files to ignore. |
+| `--detsecrets-hashes-to-ignore=<file>` | Filename of a list of hashes to ignore Default is .yetus/detsecrets-ignored-hashes.txt' |
+| `--detsecrets-lines=<regex>` | Regex of lines to ignore. |
+| `--detsecrets-secrets=<regex>` | Regex of secrets to ignore. |
+
+# Docker Notes
+
+Currently, the Yelp version is provided but that may change in the future depending upon how the project shakes out.
+
+# Developer Notes
+
+None
diff --git a/precommit/src/main/shell/plugins.d/detsecrets.sh b/precommit/src/main/shell/plugins.d/detsecrets.sh
new file mode 100755
index 0000000..d9d38f2
--- /dev/null
+++ b/precommit/src/main/shell/plugins.d/detsecrets.sh
@@ -0,0 +1,228 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SHELLDOC-IGNORE
+
+add_test_type detsecrets
+
+DETSECRETS_TIMER=0
+
+DETSECRETS=${DETSECRETS:-$(command -v detect-secrets 2>/dev/null)}
+
+# why are these command line options instead of reading from a file?
+DETSECRETS_FILES='' #regex of files to ignore
+DETSECRETS_LINES='' #regex of lines to ignore
+DETSECRETS_SECRETS='' #regex of secrets to ignore
+DETSECRETS_HASHFILE='.yetus/detsecrets-ignored-hashes.txt'
+DETSECRETS_OLD='false'
+
+function detsecrets_usage
+{
+ yetus_add_option "--detsecrets=<file>" "Filename of the detect-secrets executable (default: ${DETSECRETS})"
+ yetus_add_option "--detsecrets-files=<regex>" "Regex of files to ignore"
+ yetus_add_option "--detsecrets-hashes-to-ignore=<file>" "Filename of a list of hashes to ignore (default: ${DETSECRETS_HASHFILE})"
+ yetus_add_option "--detsecrets-lines=<regex>" "Regex of lines to ignore"
+ yetus_add_option "--detsecrets-secrets=<regex>" "Regex of secrets to ignore"
+}
+
+function detsecrets_parse_args
+{
+ local i
+
+ for i in "$@"; do
+ case ${i} in
+ --detsecrets=*)
+ delete_parameter "${i}"
+ DETSECRETS=${i#*=}
+ ;;
+ --detsecrets-files=*)
+ delete_parameter "${i}"
+ DETSECRETS_FILES=${i#*=}
+ ;;
+ --detsecrets-hashes-to-ignore=*)
+ delete_parameter "${i}"
+ DETSECRETS_HASHFILE=${i#*=}
+ ;;
+ --detsecrets-lines=*)
+ delete_parameter "${i}"
+ DETSECRETS_LINES=${i#*=}
+ ;;
+ --detsecrets-secrets=*)
+ delete_parameter "${i}"
+ DETSECRETS_SECRETS=${i#*=}
+ ;;
+ esac
+ done
+}
+
+function detsecrets_filefilter
+{
+ add_test detsecrets
+}
+
+function detsecrets_precheck
+{
+ if ! verify_command "detect-secrets" "${DETSECRETS}"; then
+ add_vote_table_v2 0 detsecrets "" "detect-secrets was not available."
+ delete_test detsecrets
+ fi
+
+ # shellcheck disable=SC2016
+ DETSECRETS_VERSION=$("${DETSECRETS}" --version 2>/dev/null| "${AWK}" '{print $NF}')
+
+ if [[ ${DETSECRETS_VERSION} =~ /^0 ]]; then
+ DETSECRETS_OLD='true'
+ fi
+}
+
+function detsecrets_calcdiffs
+{
+ # should be able to use column since our detsecrets-parse turns
+ # our output into file:line:hash:error, where hash will be unique
+ column_calcdiffs "$@"
+}
+
+function detsecrets_convert_json_to_flat
+{
+ declare repostatus=$1
+
+ if [[ -f "${PATCH_DIR}/excluded.txt" ]]; then
+ stripcmd=("${GREP}" "-v" "-f" "${PATCH_DIR}/excluded.txt")
+ else
+ stripcmd=("cat")
+ fi
+
+ # rip apart the detect-secrets json and make it a colon delimited file
+ # to make it easier to parse. Theoretically, python or python3 should
+ # be available on the path since detect-secrets also needs it.
+ pythonexec=$(command -v python3) || pythonexec=$(command -v python)
+
+ "${pythonexec}" "${BINDIR}/plugins.d/detsecrets_parse.py" \
+ "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
+ "${DETSECRETS_HASHFILE}" \
+ | "${stripcmd[@]}" \
+ > "${PATCH_DIR}/${repostatus}-detsecrets-result.txt"
+}
+
+function detsecrets_executor
+{
+ declare repostatus=$1
+ declare i
+ declare count
+ declare detsecretsStderr=${repostatus}-detsecrets-stderr.txt
+ declare -a detsecretsopts
+
+ if ! verify_needed_test detsecrets; then
+ return 0
+ fi
+
+ big_console_header "detsecrets plugin: ${BUILDMODE}"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${DETSECRETS_TIMER}"
+
+ echo "Running detect-secrets against source tree."
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ detsecretsopts=()
+
+ if [[ -n "${DETSECRETS_FILES}" ]]; then
+ detsecretsopts=("${detsecretsopts[@]}" "--exclude-files" "${DETSECRETS_FILES}")
+ fi
+
+ if [[ -n "${DETSECRETS_LINES}" ]]; then
+ detsecretsopts=("${detsecretsopts[@]}" "--exclude-lines" "${DETSECRETS_LINES}")
+ fi
+ if [[ -n "${DETSECRETS_SECRETS}" ]]; then
+ detsecretsopts=("${detsecretsopts[@]}" "--exclude-secrets" "${DETSECRETS_SECRETS}")
+ fi
+
+ if [[ ${DETSECRETS_OLD} == 'false' ]]; then
+ "${DETSECRETS}" "${detsecretsopts[@]}" scan \
+ --all-files \
+ "${detsecretsopts[@]}" \
+ > "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
+ 2>"${PATCH_DIR}/${detsecretsStderr}"
+ else
+ "${DETSECRETS}" "${detsecretsopts[@]}" scan \
+ "${detsecretsopts[@]}" \
+ > "${PATCH_DIR}/${repostatus}-detsecrets-result.json" \
+ 2>"${PATCH_DIR}/${detsecretsStderr}"
+ fi
+
+ detsecrets_convert_json_to_flat "${repostatus}"
+
+ if [[ -f ${PATCH_DIR}/${detsecretsStderr} ]]; then
+ # shellcheck disable=SC2016
+ count=$(wc -l "${PATCH_DIR}/${detsecretsStderr}" | "${AWK}" '{print $1}')
+ if [[ ${count} -gt 0 ]]; then
+ add_vote_table_v2 -1 detsecrets "@@BASE@@/${detsecretsStderr}" "Error running detsecrets. Please check detsecrets stderr files."
+ return 1
+ fi
+ fi
+ rm "${PATCH_DIR}/${detsecretsStderr}" 2>/dev/null
+ popd >/dev/null || return 1
+ return 0
+}
+
+function detsecrets_preapply
+{
+ declare retval
+
+ if ! verify_needed_test detsecrets; then
+ return 0
+ fi
+
+ detsecrets_executor "branch"
+ retval=$?
+
+ # keep track of how much as elapsed for us already
+ DETSECRETS_TIMER=$(stop_clock)
+ return ${retval}
+}
+
+function detsecrets_postapply
+{
+ if ! verify_needed_test detsecrets; then
+ return 0
+ fi
+
+ detsecrets_executor patch
+
+ # shellcheck disable=SC2016
+ DETSECRETS_VERSION=$("${DETSECRETS}" --version 2>/dev/null | "${GREP}" detsecrets | "${AWK}" '{print $NF}')
+ add_version_data detsecrets "${DETSECRETS_VERSION%,}"
+
+
+ root_postlog_compare \
+ detsecrets \
+ "${PATCH_DIR}/branch-detsecrets-result.txt" \
+ "${PATCH_DIR}/patch-detsecrets-result.txt"
+}
+
+function detsecrets_precompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ detsecrets_preapply
+ else
+ detsecrets_postapply
+ fi
+}
diff --git a/precommit/src/main/shell/plugins.d/detsecrets_parse.py b/precommit/src/main/shell/plugins.d/detsecrets_parse.py
new file mode 100755
index 0000000..38d8cc1
--- /dev/null
+++ b/precommit/src/main/shell/plugins.d/detsecrets_parse.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+''' helper app for detect-secrets to take the json and make it colon delimited '''
+
+import json
+import logging
+import pathlib
+import sys
+
+hashdict = []
+
+inputfile = sys.argv[1]
+inputpath = pathlib.Path(inputfile).resolve()
+
+if len(sys.argv) == 3:
+ hashfile = sys.argv[2]
+ hashpath = pathlib.Path(hashfile).resolve()
+ if hashpath.exists():
+ with open(hashpath, encoding='utf-8') as filein:
+ while True:
+ line = filein.readline()
+ if not line:
+ break
+ if line.startswith('#'):
+ continue
+ hashdict.append(line.strip())
+
+if not inputpath.exists() or not inputpath.is_file():
+ logging.error('%s does not exist or is not a file.', inputpath)
+ sys.exit(1)
+
+with open(inputfile, encoding='utf-8') as filein:
+ rawdata = filein.read()
+
+jsondata = json.loads(rawdata)
+
+for filename, results in sorted(jsondata['results'].items(),
+ key=lambda x: x[0]):
+ for result in results:
+ linenumber = result['line_number']
+ resulttype = result['type']
+ hashsecret = result['hashed_secret']
+ if hashsecret in hashdict:
+ continue
+ print(f'{filename}:{linenumber}:{hashsecret}:{resulttype}')
diff --git a/precommit/src/main/shell/test-patch-docker/Dockerfile b/precommit/src/main/shell/test-patch-docker/Dockerfile
index 8805e02..9842b92 100644
--- a/precommit/src/main/shell/test-patch-docker/Dockerfile
+++ b/precommit/src/main/shell/test-patch-docker/Dockerfile
@@ -313,6 +313,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
######
ARG PY3_ASTROID_VERSION=2.8.0
ARG PY3_CODESPELL_VERSION=2.1.0
+ARG PY3_DETECT_SECRETS=1.0.3
ARG PY3_DOCKER_COMPOSE=1.29.2
ARG PY3_PYLINT_VERSION=2.11.1
ARG PY3_YAMLLINT_VERSION=1.26.3
@@ -324,13 +325,11 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
python3-cryptography \
python3-dateutil \
python3-dev \
- python3-dev \
python3-isort \
python3-dockerpty \
python3-nacl \
python3-pyrsistent \
python3-setuptools \
- python3-setuptools \
python3-singledispatch \
python3-six \
python3-wheel \
@@ -344,6 +343,7 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
&& pip3 install --no-cache-dir -v \
astroid==$PY3_ASTROID_VERSION \
codespell==$PY3_CODESPELL_VERSION \
+ detect-secrets==$PY3_DETECT_SECRETS \
docker-compose==$PY3_DOCKER_COMPOSE \
pylint==$PY3_PYLINT_VERSION \
yamllint==$PY3_YAMLLINT_VERSION \