You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@yetus.apache.org by aw...@apache.org on 2019/01/09 23:39:34 UTC
[yetus] 01/04: YETUS-764. test-patch running in a pipeline isn't
getting killed
This is an automated email from the ASF dual-hosted git repository.
aw pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yetus.git
commit 8f31a4c8c88a314e7c76fa212a9abb75a1c98c94
Author: Allen Wittenauer <aw...@apache.org>
AuthorDate: Sun Jan 6 10:42:49 2019 -0800
YETUS-764. test-patch running in a pipeline isn't getting killed
Signed-off-by: Sean Busbey <bu...@apache.org>
---
Jenkinsfile | 22 ++++++++++++++++--
.../documentation/in-progress/precommit-robots.md | 27 ++++++++++++++++++++--
precommit/src/main/shell/core.d/docker.sh | 12 ++++++----
precommit/src/main/shell/test-patch.sh | 8 +++++++
4 files changed, 60 insertions(+), 9 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 78a623a..aee1009 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,9 +254,27 @@ ${FILE,path="out/brief.txt"}
}
}
- // Jenkins pipeline jobs fill slaves on PRs without this :(
+ // Jenkins has issues in some configurations sending
+ // signals to child processes. Additionally, Github Branch
+ // Source plug-in will quickly fill Jenkins build hosts
+ // with PR directories. This cleanup stanze kills
+ // any left over processes/containers and frees disk space
+ // on exit
cleanup() {
- deleteDir()
+ script {
+ sh '''
+ if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
+ echo "test-patch process appears to still be running: killing"
+ kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
+ sleep 10
+ fi
+ if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
+ echo "test-patch container appears to still be running: killing"
+ docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
+ fi
+ '''
+ deleteDir()
+ }
}
}
}
diff --git a/asf-site-src/source/documentation/in-progress/precommit-robots.md b/asf-site-src/source/documentation/in-progress/precommit-robots.md
index faa487c..9c226ac 100644
--- a/asf-site-src/source/documentation/in-progress/precommit-robots.md
+++ b/asf-site-src/source/documentation/in-progress/precommit-robots.md
@@ -113,12 +113,35 @@ pipeline {
JAVA_HOME = '/usr/lib/jvm/java-8-openjdk-amd64'
}
- ...
-
}
```
+ Experience has shown that certain Jenkins + Java + OS combinations have problems sending signals to child processes. In the case of Apache Yetus, this may result in aborted or workflows that timeout not being properly killed. `test-patch` will write two files in the patch directory that may be helpful to combat this situation if it applies to your particular configuration. `pidfile.txt` contains the master `test-patch` process id and `cidfile.txt` contains the docker container id. T [...]
+
+ ```groovy
+ post {
+ cleanup() {
+ script {
+ sh '''
+ if [ -f "${env.PATCH_DIR}/pidfile.txt" ]; then
+ kill `cat "${env.PATCH_DIR}/pidfile.txt"` || true
+ sleep 5
+ fi
+ if [ -f "${env.PATCH_DIR}/cidfile.txt" ]; then
+ docker kill `cat "${env.PATCH_DIR}/cidfile.txt"` || true
+ sleep 5
+ fi
+ '''
+ ...
+ deletedir()
+ }
+ }
+ }
+ ```
+
+
+
See also
* See also the source tree's `Jenkinsfile` for some tips and tricks.
* [precommit-admin](precommit-admin), for special utilities built for Jenkins.
diff --git a/precommit/src/main/shell/core.d/docker.sh b/precommit/src/main/shell/core.d/docker.sh
index d419bc0..d298d19 100755
--- a/precommit/src/main/shell/core.d/docker.sh
+++ b/precommit/src/main/shell/core.d/docker.sh
@@ -717,7 +717,7 @@ function docker_run_image
# make the kernel prefer to kill us if we run out of RAM
DOCKER_EXTRAARGS+=("--oom-score-adj" "500")
- DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile")
+ DOCKER_EXTRAARGS+=("--cidfile=${PATCH_DIR}/cidfile.txt")
if [[ "${DOCKER_IN_DOCKER}" == true ]]; then
if [[ -e "${DOCKER_SOCKET}" ]]; then
@@ -735,8 +735,7 @@ function docker_run_image
DOCKER_EXTRAARGS+=(--name "${containername}")
- trap 'docker_signal_handler' SIGTERM
- trap 'docker_signal_handler' SIGINT
+ trap 'docker_signal_handler' SIGTERM SIGINT SIGHUP
if [[ ${PATCH_DIR} =~ ^/ ]]; then
dockercmd run --rm=true -i \
@@ -758,6 +757,7 @@ function docker_run_image
printf '\n\n'
echo "Cleaning up docker image used for testing."
dockercmd rmi "${patchimagename}" > /dev/null
+ rm "${PATCH_DIR}/cidfile.txt"
cleanup_and_exit ${retval}
}
@@ -769,10 +769,12 @@ function docker_signal_handler
{
declare cid
- cid=$(cat "${PATCH_DIR}/cidfile")
+ cid=$(cat "${PATCH_DIR}/cidfile.txt")
yetus_error "ERROR: Caught signal. Killing docker container:"
- dockercmd kill "${cid}"
+ echo "ERROR: Caught signal. Killing docker container: ${cid}" > "${PATCH_DIR}/signal.log"
+ dockercmd kill "${cid}" | tee -a "${PATCH_DIR}/signal.log"
+ rm "${PATCH_DIR}/cidfile.txt"
yetus_error "ERROR: Exiting."
cleanup_and_exit 143 # 128 + 15 -- SIGTERM
}
diff --git a/precommit/src/main/shell/test-patch.sh b/precommit/src/main/shell/test-patch.sh
index 4ea5cb1..b361289 100755
--- a/precommit/src/main/shell/test-patch.sh
+++ b/precommit/src/main/shell/test-patch.sh
@@ -2470,6 +2470,10 @@ function cleanup_and_exit
big_console_header "Finished build."
fi
+ if [[ "${DOCKERMODE}" != true ]]; then
+ rm "${PATCH_DIR}/pidfile.txt"
+ fi
+
# shellcheck disable=SC2086
exit ${result}
}
@@ -3210,6 +3214,10 @@ function initialize
docker_initialize
fi
+ if [[ "${DOCKERMODE}" != true ]]; then
+ echo "$$" > "${PATCH_DIR}/pidfile.txt"
+ fi
+
plugins_initialize
if [[ ${RESULT} != 0 ]]; then
cleanup_and_exit 1