You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2015/11/17 00:34:31 UTC
[1/2] yetus git commit: Revert "Multiple docker fixes: (trial)"
Repository: yetus
Updated Branches:
refs/heads/YETUS-83 fa1232897 -> 2ae51492c
Revert "Multiple docker fixes: (trial)"
This reverts commit 72645f469d641576fab94aaac9428ef8c7e90824.
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/60ef0563
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/60ef0563
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/60ef0563
Branch: refs/heads/YETUS-83
Commit: 60ef056370d7cca1cb3c6ca891f92ea2f01d48e5
Parents: fa12328
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Nov 16 15:33:21 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Nov 16 15:33:21 2015 -0800
----------------------------------------------------------------------
.../documentation/latest/precommit-advanced.md | 4 +-
precommit/core.d/00-yetuslib.sh | 2 +-
precommit/core.d/01-common.sh | 19 ++--
precommit/core.d/patchfiles.sh | 5 -
precommit/test-patch-docker/Dockerfile | 99 --------------------
precommit/test-patch-docker/Dockerfile-endstub | 2 -
.../test-patch-docker/Dockerfile-startstub | 91 ++++++++++++++++++
.../test-patch-docker/launch-test-patch.sh | 14 +--
.../test-patch-docker/test-patch-docker.sh | 10 +-
precommit/test-patch.sh | 87 +++--------------
10 files changed, 119 insertions(+), 214 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/asf-site-src/source/documentation/latest/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/latest/precommit-advanced.md b/asf-site-src/source/documentation/latest/precommit-advanced.md
index dda9546..638f7d2 100644
--- a/asf-site-src/source/documentation/latest/precommit-advanced.md
+++ b/asf-site-src/source/documentation/latest/precommit-advanced.md
@@ -27,9 +27,9 @@ test-patch
# Docker Support
-By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance.
+By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java.
-The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker.
+The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker.
NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume).
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/core.d/00-yetuslib.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/00-yetuslib.sh b/precommit/core.d/00-yetuslib.sh
index 9b6185a..69b7915 100755
--- a/precommit/core.d/00-yetuslib.sh
+++ b/precommit/core.d/00-yetuslib.sh
@@ -125,4 +125,4 @@ function yetus_abs
return 0
fi
return 1
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh
index 5e7cb60..2110f75 100755
--- a/precommit/core.d/01-common.sh
+++ b/precommit/core.d/01-common.sh
@@ -45,8 +45,6 @@ function common_defaults
PATCH_METHODS=("gitapply" "patchcmd")
#shellcheck disable=SC2034
PATCH_LEVEL=0
- #shellcheck disable=SC2034
- PATCH_SYSTEM=""
PROJECT_NAME=yetus
RESULT=0
#shellcheck disable=SC2034
@@ -436,19 +434,22 @@ function importplugins
files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
fi
- if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then
- yetus_error "ERROR: Can't find ${PERSONALITY} to import."
- unset PERSONALITY
- fi
-
if [[ -z ${PERSONALITY}
&& -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
&& ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
- yetus_debug "Using project personality."
PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
fi
- if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then
+ if [[ -n ${PERSONALITY} ]]; then
+ if [[ ! -f ${PERSONALITY} ]]; then
+ if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
+ && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
+ PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
+ else
+ yetus_debug "Can't find ${PERSONALITY} to import."
+ return
+ fi
+ fi
yetus_debug "Importing ${PERSONALITY}"
# shellcheck disable=SC1090
. "${PERSONALITY}"
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/core.d/patchfiles.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh
index 7154a58..7bb2f8b 100755
--- a/precommit/core.d/patchfiles.sh
+++ b/precommit/core.d/patchfiles.sh
@@ -97,7 +97,6 @@ function locate_patch
# it's a declarely provided file
if [[ -f ${PATCH_OR_ISSUE} ]]; then
patchfile="${PATCH_OR_ISSUE}"
- PATCH_SYSTEM=generic
else
# run through the bug systems. maybe they know?
for bugsys in ${BUGSYSTEMS}; do
@@ -105,7 +104,6 @@ function locate_patch
"${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
if [[ $? == 0 ]]; then
gotit=true
- PATCH_SYSTEM=${bugsys}
fi
fi
done
@@ -117,12 +115,9 @@ function locate_patch
yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}."
cleanup_and_exit 1
fi
- PATCH_SYSTEM=generic
fi
fi
- yetus_debug "Determined patch system to be ${PATCH_SYSTEM}"
-
if [[ ! -f "${PATCH_DIR}/patch"
&& -f "${patchfile}" ]]; then
cp "${patchfile}" "${PATCH_DIR}/patch"
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile
deleted file mode 100644
index a839057..0000000
--- a/precommit/test-patch-docker/Dockerfile
+++ /dev/null
@@ -1,99 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:trusty
-
-WORKDIR /root
-
-######
-# Install common dependencies from packages
-######
-RUN apt-get update && apt-get install --no-install-recommends -y \
- git curl ant make maven \
- cmake gcc g++ protobuf-compiler \
- build-essential libtool \
- zlib1g-dev pkg-config libssl-dev \
- snappy libsnappy-dev \
- bzip2 libbz2-dev \
- libjansson-dev \
- fuse libfuse-dev \
- libcurl4-openssl-dev \
- python python2.7 pylint \
- ruby \
- openjdk-7-jdk \
- libperl-critic-perl
-
-# Fixing the Apache commons / Maven dependency problem under Ubuntu:
-# See http://wiki.apache.org/commons/VfsProblems
-RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
-
-#######
-# Oracle Java
-#######
-
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository -y ppa:webupd8team/java
-RUN apt-get update
-
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java7-installer
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
-
-######
-# Install findbugs
-######
-RUN mkdir -p /opt/findbugs && \
- curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
- -o /opt/findbugs.tar.gz && \
- tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
-
-####
-# Install shellcheck
-####
-RUN apt-get install -y cabal-install
-RUN cabal update && cabal install shellcheck --global
-
-####
-# Install rubocop
-###
-RUN gem install rubocop
-
-####
-# Install ruby-lint
-###
-RUN gem install ruby-lint
-
-####
-# Install bats
-####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get update
-RUN apt-get install -y bats
-
-###
-# Set the locale
-###
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch-docker/Dockerfile-endstub
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile-endstub b/precommit/test-patch-docker/Dockerfile-endstub
index ce76491..aa0463e 100644
--- a/precommit/test-patch-docker/Dockerfile-endstub
+++ b/precommit/test-patch-docker/Dockerfile-endstub
@@ -15,7 +15,5 @@
# limitations under the License.
ADD launch-test-patch.sh /testptch/launch-test-patch.sh
-RUN mkdir /testptch/extras
-RUN chmod a+rwx /testptch/extras
RUN chmod a+rx /testptch/launch-test-patch.sh
CMD /testptch/launch-test-patch.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile-startstub b/precommit/test-patch-docker/Dockerfile-startstub
new file mode 100644
index 0000000..d191285
--- /dev/null
+++ b/precommit/test-patch-docker/Dockerfile-startstub
@@ -0,0 +1,91 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:trusty
+
+WORKDIR /root
+
+######
+# Install common dependencies from packages
+######
+RUN apt-get update && apt-get install --no-install-recommends -y \
+ git curl ant make maven \
+ cmake gcc g++ protobuf-compiler \
+ build-essential libtool \
+ zlib1g-dev pkg-config libssl-dev \
+ snappy libsnappy-dev \
+ bzip2 libbz2-dev \
+ libjansson-dev \
+ fuse libfuse-dev \
+ libcurl4-openssl-dev \
+ python python2.7 pylint \
+ ruby \
+ openjdk-7-jdk \
+ libperl-critic-perl
+
+# Fixing the Apache commons / Maven dependency problem under Ubuntu:
+# See http://wiki.apache.org/commons/VfsProblems
+RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+
+#######
+# Oracle Java
+#######
+
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:webupd8team/java
+RUN apt-get update
+
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java7-installer
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java8-installer
+
+######
+# Install findbugs
+######
+RUN mkdir -p /opt/findbugs && \
+ curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+ -o /opt/findbugs.tar.gz && \
+ tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
+ENV FINDBUGS_HOME /opt/findbugs
+
+####
+# Install shellcheck
+####
+RUN apt-get install -y cabal-install
+RUN cabal update && cabal install shellcheck --global
+
+####
+# Install rubocop
+###
+RUN gem install rubocop
+
+####
+# Install ruby-lint
+###
+RUN gem install ruby-lint
+
+####
+# Install bats
+####
+RUN add-apt-repository -y ppa:duggan/bats
+RUN apt-get update
+RUN apt-get install -y bats
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch-docker/launch-test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/launch-test-patch.sh b/precommit/test-patch-docker/launch-test-patch.sh
index 67c7dfd..ac976ad 100755
--- a/precommit/test-patch-docker/launch-test-patch.sh
+++ b/precommit/test-patch-docker/launch-test-patch.sh
@@ -38,16 +38,6 @@ TESTPATCHMODE=${TESTPATCHMODE/--docker }
cd "${BASEDIR}"
PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
-# if patch system is generic, then it's either a local
-# patch file or was in some other way not pulled from a bug
-# system. So we need to rescue it and then tell
-# test-patch where to find it.
-if [[ "${PATCH_SYSTEM}" = generic ]]; then
- cp -p "${PATCH_DIR}/patch" /testptch/extras/patch
- patchfile="/testptch/extras/patch"
-fi
-
-
cd "${PATCH_DIR}/precommit/"
#shellcheck disable=SC2086
"${PATCH_DIR}/precommit/test-patch.sh" \
@@ -56,5 +46,5 @@ cd "${PATCH_DIR}/precommit/"
--basedir="${BASEDIR}" \
--patch-dir="${PATCH_DIR}" \
--java-home="${JAVA_HOME}" \
- --user-plugins="${PATCH_DIR}/precommit/user-plugins" \
- ${patchfile}
+ --personality="${PATCH_DIR}/precommit/personality/provided.sh" \
+ --user-plugins="${PATCH_DIR}/precommit/user-plugins"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch-docker/test-patch-docker.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/test-patch-docker.sh b/precommit/test-patch-docker/test-patch-docker.sh
index ae31556..ca910ff 100755
--- a/precommit/test-patch-docker/test-patch-docker.sh
+++ b/precommit/test-patch-docker/test-patch-docker.sh
@@ -37,7 +37,7 @@ function yetus_debug
function dockercmd
{
yetus_debug "docker $*"
- "${DOCKERCMD}" "$@"
+ docker "$@"
}
## @description Handle command line arguments
@@ -54,9 +54,6 @@ function parse_args
--debug)
YETUS_SHELL_SCRIPT_DEBUG=true
;;
- --dockercmd=*)
- DOCKERCMD=${i#*=}
- ;;
--dockerversion=*)
DOCKER_VERSION=${i#*=}
;;
@@ -70,9 +67,6 @@ function parse_args
--patch-dir=*)
PATCH_DIR=${i#*=}
;;
- --patchsystem=*)
- PATCH_SYSTEM=${i#*=}
- ;;
--project=*)
PROJECT_NAME=${i#*=}
;;
@@ -373,7 +367,6 @@ PatchSpecificDocker
--env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
--env=JAVA_HOME="${JAVA_HOME}" \
--env=PATCH_DIR=/testptch/patchprocess \
- --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \
--env=PROJECT_NAME="${PROJECT_NAME}" \
--env=TESTPATCHMODE="${TESTPATCHMODE}" \
"test-patch-tp-${PROJECT_NAME}-${DID}"
@@ -387,7 +380,6 @@ PatchSpecificDocker
--env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
--env=JAVA_HOME="${JAVA_HOME}" \
--env=PATCH_DIR="${PATCH_DIR}" \
- --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \
--env=PROJECT_NAME="${PROJECT_NAME}" \
--env=TESTPATCHMODE="${TESTPATCHMODE}" \
"test-patch-tp-${PROJECT_NAME}-${DID}"
http://git-wip-us.apache.org/repos/asf/yetus/blob/60ef0563/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index cb3160a..1847848 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -56,7 +56,7 @@ function setup_defaults
{
common_defaults
- DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile"
+ DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub"
HOW_TO_CONTRIBUTE="https://yetus.apache.org/documentation/latest/precommit-patchnames"
INSTANCE=${RANDOM}
RELOCATE_PATCH_DIR=false
@@ -677,7 +677,6 @@ function yetus_usage
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
echo "--docker Spawn a docker container"
- echo "--dockercmd=<file> Command to use as docker executable (default: docker from path)"
echo "--dockerfile=<file> Dockerfile fragment to use as the base"
echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)"
echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)"
@@ -763,9 +762,6 @@ function parse_args
--docker)
DOCKERSUPPORT=true
;;
- --dockercmd=*)
- DOCKERCMD=${i#*=}
- ;;
--dockerfile=*)
DOCKERFILE=${i#*=}
;;
@@ -916,12 +912,6 @@ function parse_args
GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt"
- if [[ -n "${REEXECPERSONALITY}"
- && -f "${PATCH_DIR}/precommit/personality/provided.sh" ]]; then
- PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh"
- fi
-
- DOCKERFILE=$(yetus_abs "${DOCKERFILE}")
}
## @description Locate the build file for a given directory
@@ -1373,11 +1363,9 @@ function apply_patch_file
## @replaceable no
function copytpbits
{
- declare dockerdir
- declare dockfile
- declare person
- declare lines
-
+ local dockerdir
+ local dockfile
+ local person
# we need to copy/consolidate all the bits that might have changed
# that are considered part of test-patch. This *might* break
# things that do off-path includes, but there isn't much we can
@@ -1428,16 +1416,10 @@ function copytpbits
(
echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}"
echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}"
- lines=$(${GREP} -n 'YETUS CUT HERE' ${DOCKERFILE} | cut -f1 -d:)
- if [[ -z "${lines}" ]]; then
- cat "${DOCKERFILE}"
- else
- head -n "${lines}" "${DOCKERFILE}"
- fi
+ cat "${DOCKERFILE}"
# make sure we put some space between, just in case last
# line isn't an empty line or whatever
printf "\n\n"
- echo "### TEST_PATCH_PRIVATE: start test-patch-bootstrap"
cat "${BINDIR}/test-patch-docker/Dockerfile-endstub"
printf "\n\n"
@@ -1448,34 +1430,6 @@ function copytpbits
popd >/dev/null
}
-## @description Verify docker exists
-## @audience private
-## @stability evolving
-## @replaceable no
-## @returns 1 if docker not found
-## @returns 0 if docker is found
-function dockerverify
-{
- declare pathdocker
-
- if [[ -z "${DOCKERCMD}" ]]; then
- pathdocker=$(which docker 2>/dev/null)
-
- if [[ ! -f "${pathdocker}" ]]; then
- yetus_error "Docker cannot be found."
- return 1
- fi
- DOCKERCMD="${pathdocker}"
- fi
-
- if [[ ! -x "${DOCKERCMD}" ]];then
- yetus_error "Docker command ${DOCKERCMD} is not executable."
- return 1
- fi
- return 0
-}
-
-
## @description If this patches actually patches test-patch.sh, then
## @description run with the patched version for the test.
## @audience private
@@ -1496,10 +1450,6 @@ function check_reexec
return
fi
- # determine if the patch hits
- # any test-patch sensitive bits
- # if so, we need to copy the universe
- # after patching it (copy=true)
for testdir in "${BINDIR}" \
"${PERSONALITY}" \
"${USER_PLUGIN_DIR}" \
@@ -1511,14 +1461,6 @@ function check_reexec
fi
done
- if [[ "${DOCKERSUPPORT}" = true ]]; then
- dockerverify
- if [[ $? == 1 ]]; then
- yetus_error "Docker not found or not executable. Disabling Docker mode."
- DOCKERSUPPORT=false
- fi
- fi
-
if [[ ${copy} == true ]]; then
big_console_header "precommit patch detected"
@@ -1547,8 +1489,8 @@ function check_reexec
fi
if [[ ${DOCKERSUPPORT} == true
- && ${copy} == false ]]; then
- big_console_header "Re-execing under Docker"
+ && ${copy} == false ]]; then
+ big_console_header "Re-execing under Docker"
fi
# copy our universe
@@ -1571,21 +1513,17 @@ function check_reexec
if [[ -n "${BUILD_URL}" ]]; then
TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}"
fi
-
- if [[ -f "${PERSONALITY}" ]]; then
- TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}"
- fi
-
TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}"
TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}"
TESTPATCHMODE="--tpinstance=${INSTANCE} ${TESTPATCHMODE}"
+ TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}"
TESTPATCHMODE="--plugins=${ENABLED_PLUGINS// /,} ${TESTPATCHMODE}"
TESTPATCHMODE=" ${TESTPATCHMODE}"
export TESTPATCHMODE
patchdir=$(relative_dir "${PATCH_DIR}")
- if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
+ if [[ ${TP_SHELL_SCRIPT_DEBUG} = true ]]; then
debugflag="--debug"
fi
@@ -1593,12 +1531,11 @@ function check_reexec
#shellcheck disable=SC2093
exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \
${debugflag} \
- --dockercmd="${DOCKERCMD}" \
--dockerversion="${dockerversion}" \
--java-home="${JAVA_HOME}" \
--patch-dir="${patchdir}" \
- --patchsystem="${PATCH_SYSTEM}" \
- --project="${PROJECT_NAME}" \
+ --project="${PROJECT_NAME}"
+
else
# if we aren't doing docker, then just call ourselves
@@ -2627,7 +2564,7 @@ function prechecks
declare plugin
declare result=0
- for plugin in ${BUILDTOOL} ${NEEDEDTESTS} ${TESTFORMATS}; do
+ for plugin in ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do
verify_patchdir_still_exists
if declare -f ${plugin}_precheck >/dev/null 2>&1; then
[2/2] yetus git commit: YETUS-123-05. Various Docker mode fixes:
plug-in issues, local patch file problems, availability,
& Dockerfile re-usability
Posted by aw...@apache.org.
YETUS-123-05. Various Docker mode fixes: plug-in issues, local patch file problems, availability, & Dockerfile re-usability
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/2ae51492
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/2ae51492
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/2ae51492
Branch: refs/heads/YETUS-83
Commit: 2ae51492c6363d4afc57edda3a2ac2289c4a4e13
Parents: 60ef056
Author: Allen Wittenauer <aw...@apache.org>
Authored: Thu Nov 12 08:21:21 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Nov 16 15:34:07 2015 -0800
----------------------------------------------------------------------
.../documentation/latest/precommit-advanced.md | 4 +-
precommit/core.d/01-common.sh | 19 +-
precommit/core.d/docker.sh | 408 +++++++++++++++++++
precommit/core.d/patchfiles.sh | 5 +
precommit/test-patch-docker/Dockerfile | 99 +++++
precommit/test-patch-docker/Dockerfile-endstub | 2 +
.../test-patch-docker/Dockerfile-startstub | 91 -----
.../test-patch-docker/launch-test-patch.sh | 14 +-
.../test-patch-docker/test-patch-docker.sh | 392 ------------------
precommit/test-patch.sh | 94 ++++-
10 files changed, 612 insertions(+), 516 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/asf-site-src/source/documentation/latest/precommit-advanced.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/latest/precommit-advanced.md b/asf-site-src/source/documentation/latest/precommit-advanced.md
index 638f7d2..dda9546 100644
--- a/asf-site-src/source/documentation/latest/precommit-advanced.md
+++ b/asf-site-src/source/documentation/latest/precommit-advanced.md
@@ -27,9 +27,9 @@ test-patch
# Docker Support
-By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself into a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java.
+By default, test-patch runs in the same shell where it was launched. It can alternatively use Docker to launch itself in a container. This is particularly useful if running under a QA environment that does not provide all the necessary binaries. For example, if the patch requires a newer version of Java than what is installed on a Jenkins instance.
-The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. However be aware that test-patch will copy this file and append its necessary hooks to re-launch itself prior to executing docker.
+The `--docker` parameter tells test-patch to run in Docker mode. The `--dockerfile` parameter allows one to provide a custom Dockerfile. The Dockerfile should contain all of the necessary binaries and tooling needed to run the test. test-patch will copy this file up until the text "YETUS CUT HERE" to a different directory and then append its necessary hooks to re-launch itself prior to executing docker.
NOTE: If you are using Boot2Docker, you must use directories under /Users (OSX) or C:\Users (Windows) as the base and patchprocess directories (specified by the --basedir and --patch-dir options respectively), because automatically mountable directories are limited to them. See [the Docker documentation](https://docs.docker.com/userguide/dockervolumes/#mount-a-host-directory-as-a-data-volume).
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/core.d/01-common.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/01-common.sh b/precommit/core.d/01-common.sh
index 2110f75..5e7cb60 100755
--- a/precommit/core.d/01-common.sh
+++ b/precommit/core.d/01-common.sh
@@ -45,6 +45,8 @@ function common_defaults
PATCH_METHODS=("gitapply" "patchcmd")
#shellcheck disable=SC2034
PATCH_LEVEL=0
+ #shellcheck disable=SC2034
+ PATCH_SYSTEM=""
PROJECT_NAME=yetus
RESULT=0
#shellcheck disable=SC2034
@@ -434,22 +436,19 @@ function importplugins
files=("${files[@]}" ${USER_PLUGIN_DIR}/*.sh)
fi
+ if [[ -n ${PERSONALITY} && ! -f ${PERSONALITY} ]]; then
+ yetus_error "ERROR: Can't find ${PERSONALITY} to import."
+ unset PERSONALITY
+ fi
+
if [[ -z ${PERSONALITY}
&& -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
&& ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
+ yetus_debug "Using project personality."
PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
fi
- if [[ -n ${PERSONALITY} ]]; then
- if [[ ! -f ${PERSONALITY} ]]; then
- if [[ -f "${BINDIR}/personality/${PROJECT_NAME}.sh"
- && ${LOAD_SYSTEM_PLUGINS} = "true" ]]; then
- PERSONALITY="${BINDIR}/personality/${PROJECT_NAME}.sh"
- else
- yetus_debug "Can't find ${PERSONALITY} to import."
- return
- fi
- fi
+ if [[ -n ${PERSONALITY} && -f ${PERSONALITY} ]]; then
yetus_debug "Importing ${PERSONALITY}"
# shellcheck disable=SC1090
. "${PERSONALITY}"
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh
new file mode 100755
index 0000000..d5b0d34
--- /dev/null
+++ b/precommit/core.d/docker.sh
@@ -0,0 +1,408 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER_ID=${RANDOM}
+DOCKER_DESTRUCTIVE=true
+
+## @description Run docker with some arguments, and
+## @description optionally send to debug
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function dockercmd
+{
+ yetus_debug "docker $*"
+ "${DOCKERCMD}" "$@"
+}
+
+## @description Stop and delete all defunct containers
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_stop_exited_containers
+{
+ declare line
+ declare id
+ declare value
+ declare size
+ declare exitfn="${PATCH_DIR}/dsec.$$"
+
+ big_console_header "Removing stopped/exited containers"
+
+ echo "Docker containers in exit state:"
+
+ dockercmd ps -a | ${GREP} Exited > "${exitfn}"
+ if [[ ! -s "${exitfn}" ]]; then
+ return
+ fi
+
+ # stop *all* containers that are in exit state for
+ # more than > 8 hours
+ while read -r line; do
+ id=$(echo "${line}" | cut -f1 -d' ')
+ value=$(echo "${line}" | cut -f2 -d' ')
+ size=$(echo "${line}" | cut -f3 -d' ')
+
+ if [[ ${size} =~ day
+ || ${size} =~ week
+ || ${size} =~ month
+ || ${size} =~ year ]]; then
+ echo "Removing docker ${id}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rm "${id}"
+ else
+ echo docker rm "${id}"
+ fi
+ fi
+
+ if [[ ${size} =~ hours
+ && ${value} -gt 8 ]]; then
+ echo "Removing docker ${id}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rm "${id}"
+ else
+ echo docker rm "${id}"
+ fi
+ fi
+ done < <(
+ cat "${exitfn}" \
+ | ${SED} -e 's,ago,,g' \
+ | ${AWK} '{print $1" "$(NF - 2)" "$(NF - 1)}')
+ rm "${exitfn}"
+}
+
+## @description Remove all containers that are not
+## @description are not running + older than 1 day
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_rm_old_containers
+{
+ declare line
+ declare id
+ declare value
+ declare size
+ declare running
+
+ big_console_header "Removing old containers"
+
+ while read -r line; do
+ id=$(echo "${line}" | cut -f1 -d, )
+ running=$(echo "${line}" | cut -f2 -d, )
+ stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. )
+
+ if [[ ${running} = true ]]; then
+ yetus_debug "${id} is still running, skipping."
+ continue
+ fi
+
+ # believe it or not, date is not even close to standardized...
+ if [[ $(uname -s) == Linux ]]; then
+
+ # GNU date
+ stoptime=$(date -d "${stoptime}" "+%s")
+ else
+
+ # BSD date
+ stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s")
+ fi
+
+ curtime=$(date "+%s")
+ ((difftime = curtime - stoptime))
+ if [[ ${difftime} -gt 86400 ]]; then
+ echo "Removing docker ${id}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rm "${id}"
+ else
+ echo docker rm "${id}"
+ fi
+ fi
+ done < <(
+ # see https://github.com/koalaman/shellcheck/issues/375
+ # shellcheck disable=SC2046
+ dockercmd inspect \
+ -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \
+ $(dockercmd ps -qa) 2>/dev/null)
+}
+
+## @description Remove untagged/unu${SED} images
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_remove_untagged_images
+{
+
+ big_console_header "Removing untagged images"
+
+ # this way is a bit more compatible with older docker versions
+ dockercmd images | tail -n +2 | ${AWK} '$1 == "<none>" {print $3}' | \
+ xargs --no-run-if-empty docker rmi
+}
+
+## @description Remove defunct tagged images
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_remove_old_tagged_images
+{
+ declare line
+ declare id
+ declare created
+
+ big_console_header "Removing old tagged images"
+
+ while read -r line; do
+ id=$(echo "${line}" | ${AWK} '{print $1":"$2}')
+ created=$(echo "${line}" | ${AWK} '{print $5}')
+
+ if [[ ${created} =~ week
+ || ${created} =~ month
+ || ${created} =~ year ]]; then
+ echo "Removing docker image ${id}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rmi "${id}"
+ else
+ echo docker rmi "${id}"
+ fi
+ fi
+
+ if [[ ${id} =~ yetus/${PROJECT_NAME}:date
+ || ${id} =~ test-patch- ]]; then
+ if [[ ${created} =~ day
+ || ${created} =~ hours ]]; then
+ echo "Removing docker image ${id}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rmi "${id}"
+ else
+ echo docker rmi "${id}"
+ fi
+ fi
+ fi
+ done < <(dockercmd images)
+}
+
+## @description Performance docker maintenance on Jenkins
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_cleanup_apache_jenkins
+{
+ echo "=========================="
+ echo "Docker Images:"
+ dockercmd images
+ echo "=========================="
+ echo "Docker Containers:"
+ dockercmd ps -a
+ echo "=========================="
+
+ docker_stop_exited_containers
+
+ docker_rm_old_containers
+
+ docker_remove_untagged_images
+
+ docker_remove_old_tagged_images
+}
+
+## @description Clean up our old images u${SED} for patch testing
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_cleanup_yetus_images
+{
+ declare images
+ declare imagecount
+ declare rmimage
+ declare rmi
+
+ # we always want to leave at least one of our images
+ # so that the whole thing doesn't have to be rebuilt.
+ # This also let's us purge any old images so that
+ # we can get fresh stuff sometimes
+ images=$(dockercmd images | ${GREP} "yetus/${PROJECT_NAME}" | ${GREP} tp | ${AWK} '{print $1":"$2}') 2>&1
+
+ # shellcheck disable=SC2086
+ imagecount=$(echo ${images} | tr ' ' '\n' | wc -l)
+ ((imagecount = imagecount - 1 ))
+
+ # shellcheck disable=SC2086
+ rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount})
+ for rmi in ${rmimage}
+ do
+ echo "Removing image ${rmi}"
+ if [[ "${DOCKER_DESTRUCTIVE}" = true ]]; then
+ dockercmd rmi "${rmi}"
+ else
+ echo docker rmi "${rmi}"
+ fi
+ done
+}
+
+## @description Perform pre-run maintenance to free up
+## @description resources. With --jenkins, it is a lot
+## @description more destructive.
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_cleanup
+{
+ if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
+ docker_cleanup_apache_jenkins
+ fi
+
+ docker_cleanup_yetus_images
+}
+
+## @description Deterine the user name and user id of the user
+## @description that the docker container should use
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_determine_user
+{
+ # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some
+ # ... person ... sets it to an account that doesn't actually exist.
+ # so instead, we need to try and override it with something that's
+ # probably close to reality.
+ if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
+ USER=$(id | cut -f2 -d\( | cut -f1 -d\))
+ fi
+
+ if [[ "$(uname -s)" == "Linux" ]]; then
+ USER_NAME=${SUDO_USER:=$USER}
+ USER_ID=$(id -u "${USER_NAME}")
+ GROUP_ID=$(id -g "${USER_NAME}")
+ else # boot2docker uid and gid
+ USER_NAME=${USER}
+ USER_ID=1000
+ GROUP_ID=50
+ fi
+}
+
+## @description Determine the revision of a dockerfile
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_getfilerev
+{
+ ${GREP} 'YETUS_PRIVATE: gitrev=' \
+ "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \
+ | cut -f2 -d=
+}
+
+## @description Start a test patch docker container
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_run_image
+{
+ declare dockerfilerev
+ declare baseimagename
+ declare patchimagename="yetus/${PROJECT_NAME}:tp-${DOCKER_ID}"
+
+ dockerfilerev=$(docker_getfilerev)
+
+ baseimagename="yetus/${PROJECT_NAME}:${dockerfilerev}"
+
+ # make a base image, if it isn't available
+ big_console_header "Building base image: ${baseimagename}"
+ dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker"
+
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: Docker failed to build image."
+ add_vote_table -1 docker "Docker failed to build ${baseimagename}."
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+
+ big_console_header "Building patch image: ${patchimagename}"
+ # using the base image, make one that is patch specific
+ dockercmd build -t "${patchimagename}" - <<PatchSpecificDocker
+FROM ${baseimagename}
+RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
+RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME}
+RUN chown -R ${USER_NAME} /home/${USER_NAME}
+ENV HOME /home/${USER_NAME}
+USER ${USER_NAME}
+PatchSpecificDocker
+
+ if [[ $? != 0 ]]; then
+ yetus_error "ERROR: Docker failed to build image."
+ add_vote_table -1 docker "Docker failed to build ${patchimagename}."
+ bugsystem_finalreport 1
+ cleanup_and_exit 1
+ fi
+
+ if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then
+ extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt")
+ else
+ extraargs=""
+ fi
+
+ if [[ ${PATCH_DIR} =~ ^/ ]]; then
+ exec "${DOCKERCMD}" run --rm=true -i \
+ ${extraargs} \
+ -v "${PWD}:/testptch/${PROJECT_NAME}" \
+ -v "${PATCH_DIR}:/testptch/patchprocess" \
+ -u "${USER_NAME}" \
+ -w "/testptch/${PROJECT_NAME}" \
+ --env=BASEDIR="/testptch/${PROJECT_NAME}" \
+ --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
+ --env=JAVA_HOME="${JAVA_HOME}" \
+ --env=PATCH_DIR=/testptch/patchprocess \
+ --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \
+ --env=PROJECT_NAME="${PROJECT_NAME}" \
+ --env=TESTPATCHMODE="${TESTPATCHMODE}" \
+ "${patchimagename}"
+ else
+ exec "${DOCKERCMD}" run --rm=true -i \
+ ${extraargs} \
+ -v "${PWD}:/testptch/${PROJECT_NAME}" \
+ -u "${USER_NAME}" \
+ -w "/testptch/${PROJECT_NAME}" \
+ --env=BASEDIR="/testptch/${PROJECT_NAME}" \
+ --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
+ --env=JAVA_HOME="${JAVA_HOME}" \
+ --env=PATCH_DIR="${PATCH_DIR}" \
+ --env=PATCH_SYSTEM="${PATCH_SYSTEM}" \
+ --env=PROJECT_NAME="${PROJECT_NAME}" \
+ --env=TESTPATCHMODE="${TESTPATCHMODE}" \
+ "${patchimagename}"
+ fi
+}
+
+## @description Switch over to a Docker container
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param args
+function docker_handler
+{
+ docker_cleanup
+ docker_determine_user
+ docker_run_image
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/core.d/patchfiles.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/patchfiles.sh b/precommit/core.d/patchfiles.sh
index 7bb2f8b..7154a58 100755
--- a/precommit/core.d/patchfiles.sh
+++ b/precommit/core.d/patchfiles.sh
@@ -97,6 +97,7 @@ function locate_patch
# it's a declarely provided file
if [[ -f ${PATCH_OR_ISSUE} ]]; then
patchfile="${PATCH_OR_ISSUE}"
+ PATCH_SYSTEM=generic
else
# run through the bug systems. maybe they know?
for bugsys in ${BUGSYSTEMS}; do
@@ -104,6 +105,7 @@ function locate_patch
"${bugsys}_locate_patch" "${PATCH_OR_ISSUE}" "${PATCH_DIR}/patch"
if [[ $? == 0 ]]; then
gotit=true
+ PATCH_SYSTEM=${bugsys}
fi
fi
done
@@ -115,9 +117,12 @@ function locate_patch
yetus_error "ERROR: Unsure how to process ${PATCH_OR_ISSUE}."
cleanup_and_exit 1
fi
+ PATCH_SYSTEM=generic
fi
fi
+ yetus_debug "Determined patch system to be ${PATCH_SYSTEM}"
+
if [[ ! -f "${PATCH_DIR}/patch"
&& -f "${patchfile}" ]]; then
cp "${patchfile}" "${PATCH_DIR}/patch"
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch-docker/Dockerfile
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile b/precommit/test-patch-docker/Dockerfile
new file mode 100644
index 0000000..ee92c33
--- /dev/null
+++ b/precommit/test-patch-docker/Dockerfile
@@ -0,0 +1,99 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:trusty
+
+WORKDIR /root
+
+######
+# Install common dependencies from packages
+######
+RUN apt-get update && apt-get install --no-install-recommends -y \
+ git curl ant make maven \
+ cmake gcc g++ protobuf-compiler \
+ build-essential libtool \
+ zlib1g-dev pkg-config libssl-dev \
+ snappy libsnappy-dev \
+ bzip2 libbz2-dev \
+ libjansson-dev \
+ fuse libfuse-dev \
+ libcurl4-openssl-dev \
+ python python2.7 pylint \
+ ruby \
+ openjdk-7-jdk \
+ libperl-critic-perl
+
+# Fixing the Apache commons / Maven dependency problem under Ubuntu:
+# See http://wiki.apache.org/commons/VfsProblems
+RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
+
+#######
+# Oracle Java
+#######
+
+RUN apt-get install -y software-properties-common
+RUN add-apt-repository -y ppa:webupd8team/java
+RUN apt-get update
+
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java7-installer
+
+# Auto-accept the Oracle JDK license
+RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
+RUN apt-get install -y oracle-java8-installer
+
+######
+# Install findbugs
+######
+RUN mkdir -p /opt/findbugs && \
+ curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
+ -o /opt/findbugs.tar.gz && \
+ tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
+ENV FINDBUGS_HOME /opt/findbugs
+
+####
+# Install shellcheck
+####
+RUN apt-get install -y cabal-install
+RUN cabal update && cabal install shellcheck --global
+
+####
+# Install rubocop
+###
+RUN gem install rubocop --no-ri --no-rdoc
+
+####
+# Install ruby-lint
+###
+RUN gem install ruby-lint --no-ri --no-rdoc
+
+####
+# Install bats
+####
+RUN add-apt-repository -y ppa:duggan/bats
+RUN apt-get update
+RUN apt-get install -y bats
+
+###
+# Set the locale
+###
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch-docker/Dockerfile-endstub
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile-endstub b/precommit/test-patch-docker/Dockerfile-endstub
index aa0463e..ce76491 100644
--- a/precommit/test-patch-docker/Dockerfile-endstub
+++ b/precommit/test-patch-docker/Dockerfile-endstub
@@ -15,5 +15,7 @@
# limitations under the License.
ADD launch-test-patch.sh /testptch/launch-test-patch.sh
+RUN mkdir /testptch/extras
+RUN chmod a+rwx /testptch/extras
RUN chmod a+rx /testptch/launch-test-patch.sh
CMD /testptch/launch-test-patch.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch-docker/Dockerfile-startstub
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/Dockerfile-startstub b/precommit/test-patch-docker/Dockerfile-startstub
deleted file mode 100644
index d191285..0000000
--- a/precommit/test-patch-docker/Dockerfile-startstub
+++ /dev/null
@@ -1,91 +0,0 @@
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:trusty
-
-WORKDIR /root
-
-######
-# Install common dependencies from packages
-######
-RUN apt-get update && apt-get install --no-install-recommends -y \
- git curl ant make maven \
- cmake gcc g++ protobuf-compiler \
- build-essential libtool \
- zlib1g-dev pkg-config libssl-dev \
- snappy libsnappy-dev \
- bzip2 libbz2-dev \
- libjansson-dev \
- fuse libfuse-dev \
- libcurl4-openssl-dev \
- python python2.7 pylint \
- ruby \
- openjdk-7-jdk \
- libperl-critic-perl
-
-# Fixing the Apache commons / Maven dependency problem under Ubuntu:
-# See http://wiki.apache.org/commons/VfsProblems
-RUN cd /usr/share/maven/lib && ln -s ../../java/commons-lang.jar .
-
-#######
-# Oracle Java
-#######
-
-RUN apt-get install -y software-properties-common
-RUN add-apt-repository -y ppa:webupd8team/java
-RUN apt-get update
-
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java7-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java7-installer
-
-# Auto-accept the Oracle JDK license
-RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | sudo /usr/bin/debconf-set-selections
-RUN apt-get install -y oracle-java8-installer
-
-######
-# Install findbugs
-######
-RUN mkdir -p /opt/findbugs && \
- curl -L https://sourceforge.net/projects/findbugs/files/findbugs/3.0.1/findbugs-noUpdateChecks-3.0.1.tar.gz/download \
- -o /opt/findbugs.tar.gz && \
- tar xzf /opt/findbugs.tar.gz --strip-components 1 -C /opt/findbugs
-ENV FINDBUGS_HOME /opt/findbugs
-
-####
-# Install shellcheck
-####
-RUN apt-get install -y cabal-install
-RUN cabal update && cabal install shellcheck --global
-
-####
-# Install rubocop
-###
-RUN gem install rubocop
-
-####
-# Install ruby-lint
-###
-RUN gem install ruby-lint
-
-####
-# Install bats
-####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get update
-RUN apt-get install -y bats
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch-docker/launch-test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/launch-test-patch.sh b/precommit/test-patch-docker/launch-test-patch.sh
index ac976ad..67c7dfd 100755
--- a/precommit/test-patch-docker/launch-test-patch.sh
+++ b/precommit/test-patch-docker/launch-test-patch.sh
@@ -38,6 +38,16 @@ TESTPATCHMODE=${TESTPATCHMODE/--docker }
cd "${BASEDIR}"
PATCH_DIR=$(cd -P -- "${PATCH_DIR}" >/dev/null && pwd -P)
+# if patch system is generic, then it's either a local
+# patch file or was in some other way not pulled from a bug
+# system. So we need to rescue it and then tell
+# test-patch where to find it.
+if [[ "${PATCH_SYSTEM}" = generic ]]; then
+ cp -p "${PATCH_DIR}/patch" /testptch/extras/patch
+ patchfile="/testptch/extras/patch"
+fi
+
+
cd "${PATCH_DIR}/precommit/"
#shellcheck disable=SC2086
"${PATCH_DIR}/precommit/test-patch.sh" \
@@ -46,5 +56,5 @@ cd "${PATCH_DIR}/precommit/"
--basedir="${BASEDIR}" \
--patch-dir="${PATCH_DIR}" \
--java-home="${JAVA_HOME}" \
- --personality="${PATCH_DIR}/precommit/personality/provided.sh" \
- --user-plugins="${PATCH_DIR}/precommit/user-plugins"
\ No newline at end of file
+ --user-plugins="${PATCH_DIR}/precommit/user-plugins" \
+ ${patchfile}
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch-docker/test-patch-docker.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch-docker/test-patch-docker.sh b/precommit/test-patch-docker/test-patch-docker.sh
deleted file mode 100755
index ca910ff..0000000
--- a/precommit/test-patch-docker/test-patch-docker.sh
+++ /dev/null
@@ -1,392 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DID=${RANDOM}
-
-## @description Print a message to stderr if --debug is turned on
-## @audience private
-## @stability stable
-## @replaceable no
-## @param string
-function yetus_debug
-{
- if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
- echo "[$(date) DEBUG]: $*" 1>&2
- fi
-}
-
-## @description Run docker with some arguments, and
-## @description optionally send to debug
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function dockercmd
-{
- yetus_debug "docker $*"
- docker "$@"
-}
-
-## @description Handle command line arguments
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function parse_args
-{
- local i
-
- for i in "$@"; do
- case ${i} in
- --debug)
- YETUS_SHELL_SCRIPT_DEBUG=true
- ;;
- --dockerversion=*)
- DOCKER_VERSION=${i#*=}
- ;;
- --help|-help|-h|help|--h|--\?|-\?|\?)
- yetus_usage
- exit 0
- ;;
- --java-home=*)
- JAVA_HOME=${i#*=}
- ;;
- --patch-dir=*)
- PATCH_DIR=${i#*=}
- ;;
- --project=*)
- PROJECT_NAME=${i#*=}
- ;;
- *)
- ;;
- esac
- done
-}
-
-## @description Stop and delete all defunct containers
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function stop_exited_containers
-{
- local line
- local id
- local value
- local size
-
- echo "Docker containers in exit state:"
-
- dockercmd ps -a | grep Exited
-
- # stop *all* containers that are in exit state for
- # more than > 8 hours
- while read -r line; do
- id=$(echo "${line}" | cut -f1 -d' ')
- value=$(echo "${line}" | cut -f2 -d' ')
- size=$(echo "${line}" | cut -f3 -d' ')
-
- if [[ ${size} =~ day
- || ${size} =~ week
- || ${size} =~ month
- || ${size} =~ year ]]; then
- echo "Removing docker ${id}"
- dockercmd rm "${id}"
- fi
-
- if [[ ${size} =~ hours
- && ${value} -gt 8 ]]; then
- echo "Removing docker ${id}"
- dockercmd rm "${id}"
- fi
- done < <(
- dockercmd ps -a \
- | grep Exited \
- | sed -e 's,ago,,g' \
- | awk '{print $1" "$(NF - 2)" "$(NF - 1)}')
-}
-
-## @description Remove all containers that are not
-## @description are not running + older than 1 day
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function rm_old_containers
-{
- declare line
- declare id
- declare value
- declare size
- declare running
-
- while read -r line; do
- id=$(echo "${line}" | cut -f1 -d, )
- running=$(echo "${line}" | cut -f2 -d, )
- stoptime=$(echo "${line}" | cut -f3 -d, | cut -f1 -d. )
-
- if [[ ${running} = true ]]; then
- yetus_debug "${id} is still running, skipping."
- continue
- fi
-
- # believe it or not, date is not even close to standardized...
- if [[ $(uname -s) == Linux ]]; then
-
- # GNU date
- stoptime=$(date -d "${stoptime}" "+%s")
- else
-
- # BSD date
- stoptime=$(date -j -f "%Y-%m-%dT%H:%M:%S" "${stoptime}" "+%s")
- fi
-
- curtime=$(date "+%s")
- ((difftime = curtime - stoptime))
- if [[ ${difftime} -gt 86400 ]]; then
- echo "Removing docker ${id}"
- dockercmd rm "${id}"
- fi
- done < <(
- # see https://github.com/koalaman/shellcheck/issues/375
- # shellcheck disable=SC2046
- dockercmd inspect \
- -f '{{.Id}},{{.State.Running}},{{.State.FinishedAt}}' \
- $(dockercmd ps -qa) 2>/dev/null)
-}
-
-## @description Remove untagged/unused images
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function remove_untagged_images
-{
- # this way is a bit more compatible with older docker versions
- dockercmd images | tail -n +2 | awk '$1 == "<none>" {print $3}' | \
- xargs --no-run-if-empty docker rmi
-}
-
-## @description Remove defunct tagged images
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function remove_old_tagged_images
-{
- local line
- local id
- local created
-
- while read -r line; do
- id=$(echo "${line}" | awk '{print $1}')
- created=$(echo "${line}" | awk '{print $5}')
-
- if [[ ${created} =~ week
- || ${created} =~ month
- || ${created} =~ year ]]; then
- echo "Removing docker image ${id}"
- dockercmd rmi "${id}"
- fi
-
- if [[ ${id} =~ test-patch-base-${PROJECT_NAME}-date ]]; then
- if [[ ${created} =~ day
- || ${created} =~ hours ]]; then
- echo "Removing docker image ${id}"
- dockercmd rmi "${id}"
- fi
- fi
- done < <(dockercmd images)
-
-}
-
-## @description Performance docker maintenance on Jenkins
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function cleanup_apache_jenkins_docker
-{
- echo "=========================="
- echo "Docker Images:"
- dockercmd images
- echo "=========================="
- echo "Docker Containers:"
- dockercmd ps -a
- echo "=========================="
-
- stop_exited_containers
-
- rm_old_containers
-
- remove_untagged_images
-
- remove_old_tagged_images
-}
-
-## @description Clean up our old images used for patch testing
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function cleanup_test_patch_images
-{
- local images
- local imagecount
- local rmimage
- local rmi
-
- # we always want to leave at least one of our images
- # so that the whole thing doesn't have to be rebuilt.
- # This also let's us purge any old images so that
- # we can get fresh stuff sometimes
- images=$(dockercmd images | grep --color=none "test-patch-tp-${PROJECT_NAME}" | awk '{print $1}') 2>&1
-
- # shellcheck disable=SC2086
- imagecount=$(echo ${images} | tr ' ' '\n' | wc -l)
- ((imagecount = imagecount - 1 ))
-
- # shellcheck disable=SC2086
- rmimage=$(echo ${images} | tr ' ' '\n' | tail -${imagecount})
- for rmi in ${rmimage}
- do
- echo "Removing image ${rmi}"
- dockercmd rmi "${rmi}"
- done
-}
-
-## @description Perform pre-run maintenance to free up
-## @description resources. With --jenkins, it is a lot
-## @description more destructive.
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function cleanup
-{
- if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
- cleanup_apache_jenkins_docker
- fi
-
- cleanup_test_patch_images
-}
-
-## @description Deterine the user name and user id of the user
-## @description that the docker container should use
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function determine_user
-{
- # On the Apache Jenkins hosts, $USER is pretty much untrustable beacuse some
- # ... person ... sets it to an account that doesn't actually exist.
- # so instead, we need to try and override it with something that's
- # probably close to reality.
- if [[ ${TESTPATCHMODE} =~ jenkins ]]; then
- USER=$(id | cut -f2 -d\( | cut -f1 -d\))
- fi
-
- if [[ "$(uname -s)" == "Linux" ]]; then
- USER_NAME=${SUDO_USER:=$USER}
- USER_ID=$(id -u "${USER_NAME}")
- GROUP_ID=$(id -g "${USER_NAME}")
- else # boot2docker uid and gid
- USER_NAME=${USER}
- USER_ID=1000
- GROUP_ID=50
- fi
-}
-
-## @description Determine the revision of a dockerfile
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function getdockerfilerev
-{
- grep 'TEST_PATCH_PRIVATE: gitrev=' \
- "${PATCH_DIR}/precommit/test-patch-docker/Dockerfile" \
- | cut -f2 -d=
-}
-
-## @description Start a test patch docker container
-## @audience private
-## @stability evolving
-## @replaceable no
-## @param args
-function run_image
-{
- local dockerfilerev
- local baseimagename
-
- dockerfilerev=$(getdockerfilerev)
-
- baseimagename="test-patch-base-${PROJECT_NAME}-${dockerfilerev}"
-
- # make a base image, if it isn't available
- dockercmd build -t "${baseimagename}" "${PATCH_DIR}/precommit/test-patch-docker"
-
- # using the base image, make one that is patch specific
- dockercmd build -t "test-patch-tp-${PROJECT_NAME}-${DID}" - <<PatchSpecificDocker
-FROM ${baseimagename}
-RUN groupadd --non-unique -g ${GROUP_ID} ${USER_NAME}
-RUN useradd -g ${GROUP_ID} -u ${USER_ID} -m ${USER_NAME}
-RUN chown -R ${USER_NAME} /home/${USER_NAME}
-ENV HOME /home/${USER_NAME}
-USER ${USER_NAME}
-PatchSpecificDocker
-
- if [[ -f "${PATCH_DIR}/buildtool-docker-params.txt" ]]; then
- extraargs=$(cat "${PATCH_DIR}/buildtool-docker-params.txt")
- else
- extraargs=""
- fi
-
- if [[ ${PATCH_DIR} =~ ^/ ]]; then
- dockercmd run --rm=true -i \
- ${extraargs} \
- -v "${PWD}:/testptch/${PROJECT_NAME}" \
- -v "${PATCH_DIR}:/testptch/patchprocess" \
- -u "${USER_NAME}" \
- -w "/testptch/${PROJECT_NAME}" \
- --env=BASEDIR="/testptch/${PROJECT_NAME}" \
- --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
- --env=JAVA_HOME="${JAVA_HOME}" \
- --env=PATCH_DIR=/testptch/patchprocess \
- --env=PROJECT_NAME="${PROJECT_NAME}" \
- --env=TESTPATCHMODE="${TESTPATCHMODE}" \
- "test-patch-tp-${PROJECT_NAME}-${DID}"
- else
- dockercmd run --rm=true -i \
- ${extraargs} \
- -v "${PWD}:/testptch/${PROJECT_NAME}" \
- -u "${USER_NAME}" \
- -w "/testptch/${PROJECT_NAME}" \
- --env=BASEDIR="/testptch/${PROJECT_NAME}" \
- --env=DOCKER_VERSION="${DOCKER_VERSION} Image:${baseimagename}" \
- --env=JAVA_HOME="${JAVA_HOME}" \
- --env=PATCH_DIR="${PATCH_DIR}" \
- --env=PROJECT_NAME="${PROJECT_NAME}" \
- --env=TESTPATCHMODE="${TESTPATCHMODE}" \
- "test-patch-tp-${PROJECT_NAME}-${DID}"
- fi
-}
-
-parse_args "$@"
-cleanup
-determine_user
-run_image
http://git-wip-us.apache.org/repos/asf/yetus/blob/2ae51492/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index 1847848..7420a05 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -56,7 +56,7 @@ function setup_defaults
{
common_defaults
- DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile-startstub"
+ DOCKERFILE="${BINDIR}/test-patch-docker/Dockerfile"
HOW_TO_CONTRIBUTE="https://yetus.apache.org/documentation/latest/precommit-patchnames"
INSTANCE=${RANDOM}
RELOCATE_PATCH_DIR=false
@@ -677,6 +677,7 @@ function yetus_usage
echo "--debug If set, then output some extra stuff to stderr"
echo "--dirty-workspace Allow the local git workspace to have uncommitted changes"
echo "--docker Spawn a docker container"
+ echo "--dockercmd=<file> Command to use as docker executable (default: docker from path)"
echo "--dockerfile=<file> Dockerfile fragment to use as the base"
echo "--java-home=<path> Set JAVA_HOME (In Docker mode, this should be local to the image)"
echo "--linecomments=<bug> Only write line comments to this comma delimited list (defaults to bugcomments)"
@@ -762,6 +763,9 @@ function parse_args
--docker)
DOCKERSUPPORT=true
;;
+ --dockercmd=*)
+ DOCKERCMD=${i#*=}
+ ;;
--dockerfile=*)
DOCKERFILE=${i#*=}
;;
@@ -912,6 +916,12 @@ function parse_args
GITDIFFCONTENT="${PATCH_DIR}/gitdiffcontent.txt"
GITUNIDIFFLINES="${PATCH_DIR}/gitdiffunilines.txt"
+ if [[ -n "${REEXECPERSONALITY}"
+ && -f "${PATCH_DIR}/precommit/personality/provided.sh" ]]; then
+ PERSONALITY="${PATCH_DIR}/precommit/personality/provided.sh"
+ fi
+
+ DOCKERFILE=$(yetus_abs "${DOCKERFILE}")
}
## @description Locate the build file for a given directory
@@ -1363,9 +1373,11 @@ function apply_patch_file
## @replaceable no
function copytpbits
{
- local dockerdir
- local dockfile
- local person
+ declare dockerdir
+ declare dockfile
+ declare person
+ declare lines
+
# we need to copy/consolidate all the bits that might have changed
# that are considered part of test-patch. This *might* break
# things that do off-path includes, but there isn't much we can
@@ -1414,12 +1426,18 @@ function copytpbits
gitfilerev="date${gitfilerev}"
fi
(
- echo "### TEST_PATCH_PRIVATE: dockerfile=${DOCKERFILE}"
- echo "### TEST_PATCH_PRIVATE: gitrev=${gitfilerev}"
- cat "${DOCKERFILE}"
+ echo "### YETUS_PRIVATE: dockerfile=${DOCKERFILE}"
+ echo "### YETUS_PRIVATE: gitrev=${gitfilerev}"
+ lines=$(${GREP} -n 'YETUS CUT HERE' ${DOCKERFILE} | cut -f1 -d:)
+ if [[ -z "${lines}" ]]; then
+ cat "${DOCKERFILE}"
+ else
+ head -n "${lines}" "${DOCKERFILE}"
+ fi
# make sure we put some space between, just in case last
# line isn't an empty line or whatever
printf "\n\n"
+ echo "### YETUS_PRIVATE: start test-patch-bootstrap"
cat "${BINDIR}/test-patch-docker/Dockerfile-endstub"
printf "\n\n"
@@ -1430,6 +1448,34 @@ function copytpbits
popd >/dev/null
}
+## @description Verify docker exists
+## @audience private
+## @stability evolving
+## @replaceable no
+## @returns 1 if docker not found
+## @returns 0 if docker is found
+function dockerverify
+{
+ declare pathdocker
+
+ if [[ -z "${DOCKERCMD}" ]]; then
+ pathdocker=$(which docker 2>/dev/null)
+
+ if [[ ! -f "${pathdocker}" ]]; then
+ yetus_error "Docker cannot be found."
+ return 1
+ fi
+ DOCKERCMD="${pathdocker}"
+ fi
+
+ if [[ ! -x "${DOCKERCMD}" ]];then
+ yetus_error "Docker command ${DOCKERCMD} is not executable."
+ return 1
+ fi
+ return 0
+}
+
+
## @description If this patches actually patches test-patch.sh, then
## @description run with the patched version for the test.
## @audience private
@@ -1450,6 +1496,10 @@ function check_reexec
return
fi
+ # determine if the patch hits
+ # any test-patch sensitive bits
+ # if so, we need to copy the universe
+ # after patching it (copy=true)
for testdir in "${BINDIR}" \
"${PERSONALITY}" \
"${USER_PLUGIN_DIR}" \
@@ -1461,6 +1511,14 @@ function check_reexec
fi
done
+ if [[ "${DOCKERSUPPORT}" = true ]]; then
+ dockerverify
+ if [[ $? == 1 ]]; then
+ yetus_error "Docker not found or not executable. Disabling Docker mode."
+ DOCKERSUPPORT=false
+ fi
+ fi
+
if [[ ${copy} == true ]]; then
big_console_header "precommit patch detected"
@@ -1489,8 +1547,8 @@ function check_reexec
fi
if [[ ${DOCKERSUPPORT} == true
- && ${copy} == false ]]; then
- big_console_header "Re-execing under Docker"
+ && ${copy} == false ]]; then
+ big_console_header "Re-execing under Docker"
fi
# copy our universe
@@ -1513,29 +1571,27 @@ function check_reexec
if [[ -n "${BUILD_URL}" ]]; then
TESTPATCHMODE="--build-url=${BUILD_URL} ${TESTPATCHMODE}"
fi
+
+ if [[ -f "${PERSONALITY}" ]]; then
+ TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}"
+ fi
+
TESTPATCHMODE="--tpglobaltimer=${GLOBALTIMER} ${TESTPATCHMODE}"
TESTPATCHMODE="--tpreexectimer=${TIMER} ${TESTPATCHMODE}"
TESTPATCHMODE="--tpinstance=${INSTANCE} ${TESTPATCHMODE}"
- TESTPATCHMODE="--tpperson=${PERSONALITY} ${TESTPATCHMODE}"
TESTPATCHMODE="--plugins=${ENABLED_PLUGINS// /,} ${TESTPATCHMODE}"
TESTPATCHMODE=" ${TESTPATCHMODE}"
export TESTPATCHMODE
patchdir=$(relative_dir "${PATCH_DIR}")
- if [[ ${TP_SHELL_SCRIPT_DEBUG} = true ]]; then
+ if [[ "${YETUS_SHELL_SCRIPT_DEBUG}" = true ]]; then
debugflag="--debug"
fi
cd "${BASEDIR}"
#shellcheck disable=SC2093
- exec bash "${PATCH_DIR}/precommit/test-patch-docker/test-patch-docker.sh" \
- ${debugflag} \
- --dockerversion="${dockerversion}" \
- --java-home="${JAVA_HOME}" \
- --patch-dir="${patchdir}" \
- --project="${PROJECT_NAME}"
-
+ docker_handler
else
# if we aren't doing docker, then just call ourselves
@@ -2564,7 +2620,7 @@ function prechecks
declare plugin
declare result=0
- for plugin in ${BUILDTOOL} ${TESTTYPES} ${TESTFORMATS}; do
+ for plugin in ${BUILDTOOL} ${NEEDEDTESTS} ${TESTFORMATS}; do
verify_patchdir_still_exists
if declare -f ${plugin}_precheck >/dev/null 2>&1; then