You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2017/06/30 18:34:43 UTC
[2/2] yetus git commit: YETUS-523: create a separate tool for docker
cleanup
YETUS-523: create a separate tool for docker cleanup
Signed-off-by: Sean Busbey <bu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/yetus/repo
Commit: http://git-wip-us.apache.org/repos/asf/yetus/commit/10d4d13c
Tree: http://git-wip-us.apache.org/repos/asf/yetus/tree/10d4d13c
Diff: http://git-wip-us.apache.org/repos/asf/yetus/diff/10d4d13c
Branch: refs/heads/master
Commit: 10d4d13cc95a814eac97a976a8de525531ac986a
Parents: eabc378
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jun 23 08:59:40 2017 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jun 30 11:34:27 2017 -0700
----------------------------------------------------------------------
.../in-progress/precommit-docker-cleanup.md | 60 ++++++
build.sh | 7 +-
precommit/core.d/docker.sh | 25 +--
precommit/docker-cleanup.sh | 185 +++++++++++++++++++
precommit/test-patch.sh | 2 +-
5 files changed, 265 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
----------------------------------------------------------------------
diff --git a/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
new file mode 100644
index 0000000..9df7524
--- /dev/null
+++ b/asf-site-src/source/documentation/in-progress/precommit-docker-cleanup.md
@@ -0,0 +1,60 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+docker-cleanup
+==============
+
+`docker-cleanup` is a command to perform precommit's Docker cleanup functionality outside of patch and build testing.
+
+# Usage
+
+Executing `docker-cleanup` will peform the same Docker image as the `--robot` option.
+
+```bash
+$ docker-cleanup
+
+...
+ Removing old images
+...
+Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600
+Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8
+...
+ Docker Container Maintenance
+...
+
+```
+
+
+`docker-cleanup` also supports the `--sentinel` mode to kill and remove stale running containers:
+
+```bash
+$ docker-cleanup --sentinel
+
+...
+ Removing old images
+...
+Untagged: hadoop/createrelease:3.0.0-alpha3-SNAPSHOT_10600
+Deleted: sha256:1a96c79a0a9ab538c6c7765dc908eca3e689270b778d6ae2add558e89792a7d8
+...
+
+ Docker Container Maintenance
+...
+Attempting to remove docker container /big_kowalevski [5ffd22a56ebcfe38d72b9078e0e7133ab6dc054115a4804e504c910bdbdeea45]
+...
+```
http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/build.sh
----------------------------------------------------------------------
diff --git a/build.sh b/build.sh
index d9fbe89..6783596 100755
--- a/build.sh
+++ b/build.sh
@@ -214,8 +214,11 @@ exec "\$(dirname -- "\${BASH_SOURCE-0}")/../lib/release-doc-maker/releasedocmake
EOF
chmod +x "${bin_tarball}/bin/releasedocmaker"
-for utility in shelldocs/shelldocs.py precommit/qbt.sh \
- precommit/smart-apply-patch.sh precommit/test-patch.sh
+for utility in shelldocs/shelldocs.py \
+ precommit/docker-cleanup.sh \
+ precommit/qbt.sh \
+ precommit/smart-apply-patch.sh \
+ precommit/test-patch.sh
do
wrapper=${utility##*/}
wrapper=${wrapper%.*}
http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/core.d/docker.sh
----------------------------------------------------------------------
diff --git a/precommit/core.d/docker.sh b/precommit/core.d/docker.sh
index 863b045..124aa0b 100755
--- a/precommit/core.d/docker.sh
+++ b/precommit/core.d/docker.sh
@@ -22,6 +22,7 @@ DOCKERFILE_DEFAULT="${BINDIR}/test-patch-docker/Dockerfile"
DOCKERFAIL="fallback,continue,fail"
DOCKERSUPPORT=false
DOCKER_ENABLE_PRIVILEGED=true
+DOCKER_CLEANUP_CMD=false
declare -a DOCKER_EXTRAARGS
@@ -43,11 +44,15 @@ DOCKER_IMAGE_PURGE=604800
## @replaceable no
function docker_usage
{
- yetus_add_option "--docker" "Spawn a docker container"
+ if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
+ yetus_add_option "--docker" "Spawn a docker container"
+ fi
yetus_add_option "--dockercmd=<file>" "Command to use as docker executable (default: '${DOCKERCMD}')"
- yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')"
- yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})"
- yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')"
+ if [[ "${DOCKER_CLEANUP_CMD}" == false ]]; then
+ yetus_add_option "--dockerfile=<file>" "Dockerfile fragment to use as the base (default: '${DOCKERFILE_DEFAULT}')"
+ yetus_add_option "--dockeronfail=<list>" "If Docker fails, determine fallback method order (default: ${DOCKERFAIL})"
+ yetus_add_option "--dockerprivd=<bool>" "Run docker in privileged mode (default: '${DOCKER_ENABLE_PRIVILEGED}')"
+ fi
yetus_add_option "--dockerdelrep" "In Docker mode, only report image/container deletions, not act on them"
}
@@ -129,8 +134,7 @@ function docker_initialize
DOCKERFAIL=${DOCKERFAIL//fail/3}
DOCKERFAIL=${DOCKERFAIL//[[:blank:]]/}
- docker_exeverify
- if [[ $? != 0 ]]; then
+ if ! docker_exeverify; then
if [[ "${DOCKERFAIL}" =~ ^12
|| "${DOCKERFAIL}" =~ ^2 ]]; then
add_vote_table 0 docker "Docker command '${DOCKERCMD}' not found/broken. Disabling docker."
@@ -205,9 +209,7 @@ function docker_exeverify
return 1
fi
- ${DOCKERCMD} info >/dev/null 2>&1
-
- if [[ $? != 0 ]]; then
+ if ! ${DOCKERCMD} info >/dev/null 2>&1; then
yetus_error "Docker is not functioning properly. Daemon down/unreachable?"
return 1
fi
@@ -487,12 +489,13 @@ function docker_version
{
declare vertype=$1
declare val
-
+ declare ret
# new version command
val=$(dockercmd version --format "{{.${vertype}.Version}}" 2>/dev/null)
+ ret=$?
- if [[ $? != 0 ]];then
+ if [[ ${ret} != 0 ]];then
# old version command
val=$(dockercmd version | ${GREP} "${vertype} version" | cut -f2 -d: | tr -d ' ')
fi
http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/docker-cleanup.sh
----------------------------------------------------------------------
diff --git a/precommit/docker-cleanup.sh b/precommit/docker-cleanup.sh
new file mode 100755
index 0000000..4d18d21
--- /dev/null
+++ b/precommit/docker-cleanup.sh
@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# no shelldocs required from this file
+# SHELLDOC-IGNORE
+
+# Make sure that bash version meets the pre-requisite
+
+if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 3 ]] \
+ || [[ "${BASH_VERSINFO[0]}" -eq 3 && "${BASH_VERSINFO[1]}" -lt 2 ]]; then
+ echo "bash v3.2+ is required. Sorry."
+ exit 1
+fi
+
+this="${BASH_SOURCE-$0}"
+BINDIR=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
+BINNAME=${this##*/}
+BINNAME=${BINNAME%.sh}
+#shellcheck disable=SC2034
+STARTINGDIR=$(pwd)
+#shellcheck disable=SC2034
+USER_PARAMS=("$@")
+#shellcheck disable=SC2034
+QATESTMODE=false
+
+# dummy functions
+function add_vote_table
+{
+ true
+}
+
+function add_footer_table
+{
+ true
+}
+
+function bugsystem_finalreport
+{
+ true
+}
+
+## @description import core library routines
+## @audience private
+## @stability evolving
+function import_core
+{
+ declare filename
+
+ for filename in "${BINDIR}/core.d"/*; do
+ # shellcheck disable=SC1091
+ # shellcheck source=core.d/01-common.sh
+ . "${filename}"
+ done
+}
+
+## @description import plugins then remove the stuff we don't need
+## @audience public
+## @stability stable
+## @replaceable no
+function import_and_clean
+{
+ importplugins
+ yetus_debug "Removing BUILDTOOLS, TESTTYPES, and TESTFORMATS from installed plug-in list"
+ unset BUILDTOOLS
+ unset TESTTYPES
+ unset TESTFORMATS
+
+ #shellcheck disable=SC2034
+ DOCKER_CLEANUP_CMD=true
+ #shellcheck disable=SC2034
+ DOCKERSUPPORT=true
+ #shellcheck disable=SC2034
+ ROBOT=true
+ #shellcheck disable=SC2034
+ DOCKERFAIL="fail"
+}
+
+## @description Clean the filesystem as appropriate and then exit
+## @audience private
+## @stability evolving
+## @replaceable no
+## @param runresult
+function cleanup_and_exit
+{
+ local result=$1
+
+ if [[ ${PATCH_DIR} =~ ^/tmp/yetus
+ && -d ${PATCH_DIR} ]]; then
+ rm -rf "${PATCH_DIR}"
+ fi
+
+ # shellcheck disable=SC2086
+ exit ${result}
+}
+
+## @description Setup the default global variables
+## @audience public
+## @stability stable
+## @replaceable no
+function setup_defaults
+{
+ common_defaults
+}
+
+## @description Interpret the command line parameters
+## @audience private
+## @stability stable
+## @replaceable no
+## @param $@
+## @return May exit on failure
+function parse_args
+{
+ common_args "$@"
+}
+
+## @description Print the usage information
+## @audience public
+## @stability stable
+## @replaceable no
+function yetus_usage
+{
+ import_and_clean
+
+ echo "${BINNAME} [OPTIONS]"
+
+ yetus_add_option "--debug" "If set, then output some extra stuff to stderr"
+ yetus_add_option "--sentinel" "A very aggressive robot (auto: --robot)"
+ docker_usage
+
+ yetus_generic_columnprinter "${YETUS_OPTION_USAGE[@]}"
+ yetus_reset_usage
+}
+
+## @description Large display for the user console
+## @audience public
+## @stability stable
+## @replaceable no
+## @param string
+## @return large chunk of text
+function big_console_header
+{
+ local text="$*"
+ local spacing=$(( (75+${#text}) /2 ))
+ printf "\n\n"
+ echo "============================================================================"
+ echo "============================================================================"
+ printf "%*s\n" ${spacing} "${text}"
+ echo "============================================================================"
+ echo "============================================================================"
+ printf "\n\n"
+}
+
+trap "cleanup_and_exit 1" HUP INT QUIT TERM
+
+import_core
+
+setup_defaults
+
+parse_args "$@"
+
+import_and_clean
+
+parse_args_plugins "$@"
+
+docker_initialize
+plugins_initialize
+
+docker_cleanup
+RESULT=$?
+
+cleanup_and_exit ${RESULT}
http://git-wip-us.apache.org/repos/asf/yetus/blob/10d4d13c/precommit/test-patch.sh
----------------------------------------------------------------------
diff --git a/precommit/test-patch.sh b/precommit/test-patch.sh
index a632f5f..75dbbe4 100755
--- a/precommit/test-patch.sh
+++ b/precommit/test-patch.sh
@@ -2994,7 +2994,7 @@ function initialize
BUGLINECOMMENTS=${BUGLINECOMMENTS:-${BUGCOMMENTS}}
- # we need to do this BEFORE plugins initalize
+ # we need to do this BEFORE plugins initialize
# because they may change what they do based upon
# docker support
# note that docker support still isn't guaranteed