You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2018/08/24 00:06:21 UTC
yetus git commit: YETUS-640. add hadolint support
Repository: yetus
Updated Branches:
refs/heads/master 5a2c02552 -> 7615f35f9
YETUS-640. add hadolint support
Signed-off-by: Jack Bearden <ja...@jackbearden.com>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/7615f35f
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/7615f35f
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/7615f35f
Branch: refs/heads/master
Commit: 7615f35f9835fbe144697b6c89e313d70316e99e
Parents: 5a2c025
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jul 10 16:06:02 2018 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Aug 23 17:05:44 2018 -0700
----------------------------------------------------------------------
.hadolint.yaml | 17 +++
precommit/test-patch-docker/Dockerfile | 87 ++++++++----
precommit/test-patch.d/hadolint.sh | 204 ++++++++++++++++++++++++++++
3 files changed, 284 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/.hadolint.yaml
----------------------------------------------------------------------
diff --git a/.hadolint.yaml b/.hadolint.yaml
new file mode 100644
index 0000000..6645586
--- /dev/null
+++ b/.hadolint.yaml
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ignored:
+ - DL3008
http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile
index 2f54bda..8494d5c 100644
--- a/precommit/test-patch-docker/Dockerfile
+++ b/precommit/test-patch-docker/Dockerfile
@@ -26,6 +26,7 @@
FROM ubuntu:xenial
WORKDIR /root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
@@ -41,7 +42,9 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
pinentry-curses \
pkg-config \
rsync \
- software-properties-common
+ software-properties-common \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
###
# Set the locale
@@ -58,16 +61,20 @@ ENV LC_ALL en_US.UTF-8
####
# OpenJDK 8
####
-RUN apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
#######
# OpenJDK 9
# w/workaround for
# https://bugs.launchpad.net/ubuntu/+source/openjdk-9/+bug/1593191
#######
-RUN apt-get -o Dpkg::Options::="--force-overwrite" \
+RUN apt-get -q update && apt-get -o Dpkg::Options::="--force-overwrite" \
-q install --no-install-recommends -y \
- openjdk-9-jdk-headless
+ openjdk-9-jdk-headless \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
#######
# Set default Java
@@ -78,72 +85,91 @@ RUN apt-get -o Dpkg::Options::="--force-overwrite" \
#
RUN update-java-alternatives --set java-1.8.0-openjdk-amd64
RUN update-alternatives --get-selections | grep -i jdk | \
- while read line; do \
- alternative=$(echo $line | awk '{print $1}'); \
- path=$(echo $line | awk '{print $3}'); \
- newpath=$(echo $path | sed -e 's/java-9/java-8/'); \
- update-alternatives --set $alternative $newpath; \
+ while read -r line; do \
+ alternative=$(echo "$line" | awk '{print $1}'); \
+ path=$(echo "$line" | awk '{print $3}'); \
+ newpath=${path//java-9/java-8}; \
+ update-alternatives --set "$alternative" "$newpath"; \
done
####
# Install ant
####
-RUN apt-get -q install --no-install-recommends -y ant
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y ant \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
####
# Install GNU automake, GNU make, and related
####
-RUN apt-get -q install --no-install-recommends -y autoconf automake libtool make
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y autoconf automake libtool make \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
####
# Install bats (TAP-capable unit testing for shell scripts)
####
-RUN apt-get -q install --no-install-recommends -y bats
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y bats \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
####
# Install cmake
####
-RUN apt-get -q install --no-install-recommends -y cmake
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y cmake \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
######
# Install findbugs
######
-RUN apt-get -q install --no-install-recommends -y findbugs
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
ENV FINDBUGS_HOME /usr
####
# Install GNU C/C++
####
-RUN apt-get -q install --no-install-recommends -y g++ gcc libc-dev
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y g++ gcc libc-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
######
# Install maven
######
-RUN apt-get -q install --no-install-recommends -y maven
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y maven \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
######
# Install perl
######
-RUN apt-get -q install --no-install-recommends -y perl libperl-critic-perl
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y perl libperl-critic-perl \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
######
# Install python and pylint
######
-RUN apt-get -q install --no-install-recommends -y python \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y python \
python2.7 \
python-pip \
python-pkg-resources \
python-setuptools \
- python-wheel
-RUN pip2 install --upgrade pip==9.0.3
-RUN pip install -v pylint==1.9.2
+ python-wheel \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+RUN pip2 install -v pylint==1.9.2
####
# Install ruby, rubocop, and ruby-lint
###
RUN echo 'gem: --no-rdoc --no-ri' >> /root/.gemrc
-RUN apt-get -q install --no-install-recommends -y ruby ruby-dev
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y ruby ruby-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
RUN gem install rake
RUN gem install rubocop
RUN gem install ruby-lint
@@ -152,8 +178,19 @@ RUN gem install ruby-lint
# Install shellcheck (shell script lint)
####
RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellcheck \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+###
+# Install hadolint
+####
+RUN curl -L -s -S \
+ https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
+ -o /bin/hadolint && \
+ chmod a+rx /bin/hadolint && \
+ shasum -a 512 /bin/hadolint | \
+ awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
####
# YETUS CUT HERE
@@ -161,4 +198,6 @@ RUN apt-get -q update && apt-get -q install --no-install-recommends -y shellchec
# include other requirements not needed by your development
# (but not build) environment
###
-RUN apt-get -q install --no-install-recommends -y sudo vim
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y sudo vim \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
http://git-wip-us.apache.org/repos/asf/yetus/blob/7615f35f/precommit/test-patch.d/hadolint.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.d/hadolint.sh b/precommit/test-patch.d/hadolint.sh
new file mode 100755
index 0000000..2392e29
--- /dev/null
+++ b/precommit/test-patch.d/hadolint.sh
@@ -0,0 +1,204 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no public APIs here
+# SHELLDOC-IGNORE
+
+add_test_type hadolint
+
+HADOLINT_TIMER=0
+HADOLINT=${HADOLINT:-$(command -v hadolint 2>/dev/null)}
+
+# files that are going to get hadolint'd
+HADOLINT_CHECKFILES=()
+
+function hadolint_filefilter
+{
+ declare filename=$1
+
+ if [[ ${filename} =~ Dockerfile$ ]]; then
+ add_test hadolint
+ yetus_add_array_element HADOLINT_CHECKFILES "${filename}"
+ fi
+}
+
+function hadolint_precheck
+{
+ declare langs
+
+ if ! verify_command "hadolint" "${HADOLINT}"; then
+ add_vote_table 0 hadolint "hadolint was not available."
+ delete_test hadolint
+ fi
+
+ if [[ -z "${LANG}" ]]; then
+ langs=$(locale -a)
+ if [[ ${langs} =~ C.UTF-8 ]]; then
+ yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing C.UTF-8."
+ export LANG=C.UTF-8
+ export LC_ALL=C.UTF-8
+ elif [[ ${langs} =~ en_US.UTF-8 ]]; then
+ yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing en_US.UTF-8."
+ export LANG=en_US.UTF-8
+ export LC_ALL=en_US.UTF-8
+ else
+ for i in ${langs}; do
+ if [[ "${i}" =~ UTF-8 ]]; then
+ yetus_error "WARNING: hadolint needs UTF-8 locale support. Forcing ${i}."
+ export LANG="${i}"
+ export LC_ALL="${i}"
+ break
+ fi
+ done
+ fi
+ fi
+
+ if [[ ! "${LANG}" =~ UTF-8 ]]; then
+ yetus_error "WARNING: hadolint may fail without UTF-8 locale setting."
+ fi
+}
+
+function hadolint_logic
+{
+ declare repostatus=$1
+ declare i
+
+ pushd "${BASEDIR}" >/dev/null || return 1
+
+ for i in "${HADOLINT_CHECKFILES[@]}"; do
+ if [[ -f "${i}" ]]; then
+ echo " * ${i}"
+ "${HADOLINT}" "${i}" >> "${PATCH_DIR}/${repostatus}-hadolint-result.txt"
+ fi
+ done
+ popd > /dev/null || return 1
+}
+
+function hadolint_preapply
+{
+ if ! verify_needed_test hadolint; then
+ return 0
+ fi
+
+ big_console_header "hadolint plugin: ${PATCH_BRANCH}"
+
+ start_clock
+
+ hadolint_logic branch
+
+ # keep track of how much as elapsed for us already
+ HADOLINT_TIMER=$(stop_clock)
+ return 0
+}
+
+## filename:line\sCODE Text
+function hadolint_calcdiffs
+{
+ declare branch=$1
+ declare patch=$2
+ declare tmp=${PATCH_DIR}/pl.$$.${RANDOM}
+ declare j
+
+ # first, pull out just the errors
+ # shellcheck disable=SC2016
+ ${AWK} -F: '{print $NF}' "${branch}" | cut -d' ' -f2- > "${tmp}.branch"
+
+ # shellcheck disable=SC2016
+ ${AWK} -F: '{print $NF}' "${patch}" | cut -d' ' -f2- > "${tmp}.patch"
+
+ ${DIFF} --unchanged-line-format="" \
+ --old-line-format="" \
+ --new-line-format="%dn " \
+ "${tmp}.branch" \
+ "${tmp}.patch" > "${tmp}.lined"
+
+ # now, pull out those lines of the raw output
+ # shellcheck disable=SC2013
+ for j in $(cat "${tmp}.lined"); do
+ # shellcheck disable=SC2086
+ head -${j} "${patch}" | tail -1
+ done
+
+ rm "${tmp}.branch" "${tmp}.patch" "${tmp}.lined" 2>/dev/null
+}
+
+function hadolint_postapply
+{
+ declare i
+ declare numPrepatch
+ declare numPostpatch
+ declare diffPostpatch
+ declare fixedpatch
+ declare statstring
+
+ if ! verify_needed_test hadolint; then
+ return 0
+ fi
+
+ big_console_header "hadolint plugin: ${BUILDMODE}"
+
+ start_clock
+
+ # add our previous elapsed to our new timer
+ # by setting the clock back
+ offset_clock "${HADOLINT_TIMER}"
+
+ hadolint_logic patch
+
+ calcdiffs \
+ "${PATCH_DIR}/branch-hadolint-result.txt" \
+ "${PATCH_DIR}/patch-hadolint-result.txt" \
+ hadolint \
+ > "${PATCH_DIR}/diff-patch-hadolint.txt"
+
+ # shellcheck disable=SC2016
+ numPrepatch=$(wc -l "${PATCH_DIR}/branch-hadolint-result.txt" | ${AWK} '{print $1}')
+
+ # shellcheck disable=SC2016
+ numPostpatch=$(wc -l "${PATCH_DIR}/patch-hadolint-result.txt" | ${AWK} '{print $1}')
+
+ # shellcheck disable=SC2016
+ diffPostpatch=$(wc -l "${PATCH_DIR}/diff-patch-hadolint.txt" | ${AWK} '{print $1}')
+
+
+ ((fixedpatch=numPrepatch-numPostpatch+diffPostpatch))
+
+ statstring=$(generic_calcdiff_status "${numPrepatch}" "${numPostpatch}" "${diffPostpatch}" )
+
+ if [[ ${diffPostpatch} -gt 0 ]] ; then
+ add_vote_table -1 hadolint "${BUILDMODEMSG} ${statstring}"
+ add_footer_table hadolint "@@BASE@@/diff-patch-hadolint.txt"
+ bugsystem_linecomments "hadolint" "${PATCH_DIR}/diff-patch-hadolint.txt"
+ return 1
+ elif [[ ${fixedpatch} -gt 0 ]]; then
+ add_vote_table +1 hadolint "${BUILDMODEMSG} ${statstring}"
+ return 0
+ fi
+
+ add_vote_table +1 hadolint "There were no new hadolint issues."
+ return 0
+}
+
+function hadolint_postcompile
+{
+ declare repostatus=$1
+
+ if [[ "${repostatus}" = branch ]]; then
+ hadolint_preapply
+ else
+ hadolint_postapply
+ fi
+}