You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/07/26 21:40:37 UTC
[impala] 07/08: IMPALA-8789: Add helper to initiate graceful
shutdown
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8d4ba5d146da8685adb71d0fb49b0ca73f00a8d2
Author: Lars Volker <lv...@cloudera.com>
AuthorDate: Wed Jul 24 13:51:15 2019 -0700
IMPALA-8789: Add helper to initiate graceful shutdown
This change adds a helper script to initiate graceful daemon shutdown
via the signaling mechanism. It also includes that helper script in the
docker containers.
Testing: This change adds a test to verify that the script works as
expected. In addition, I manually verified that the script gets added to
the containers and that calling it inside the container will cause a
shutdown as expected.
Change-Id: I877483a385cd0747f69b82a6488de203a4029599
Reviewed-on: http://gerrit.cloudera.org:8080/13912
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/graceful_shutdown_backends.sh | 58 +++++++++++++++++++++++++++
docker/CMakeLists.txt | 1 +
docker/setup_build_context.py | 2 +
tests/custom_cluster/test_restart_services.py | 18 +++++++++
4 files changed, 79 insertions(+)
diff --git a/bin/graceful_shutdown_backends.sh b/bin/graceful_shutdown_backends.sh
new file mode 100755
index 0000000..c1410f4
--- /dev/null
+++ b/bin/graceful_shutdown_backends.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script will send a signal to all impalad processes as identified by their name to
+# initiate a graceful shutdown. It will then wait for a configurable amount time to allow
+# the daemons to shutdown. The script will not forcefully terminate them, which is left to
+# the orchestration framework.
+
+set -euxo pipefail
+
+export IMPALA_HOME=${IMPALA_HOME:-"/opt/impala"}
+LOG_FILE="$IMPALA_HOME/logs/shutdown.log"
+
+function LOG() {
+ echo $* | tee -a $LOG_FILE
+}
+
+# Default grace timeout is the same as the default for Impala (-shutdown_grace_period_s),
+# plus some additional safety buffer.
+GRACE_TIMEOUT=$((120 + 10))
+if [[ $# -ge 1 ]]; then
+ GRACE_TIMEOUT=$1
+fi
+
+LOG "Initiating graceful shutdown."
+for pid in $(pidof impalad); do
+ LOG "Sending signal to daemon with pid $pid"
+ kill -SIGRTMIN $pid
+done
+
+LOG "Waiting for daemons to exit, up to $GRACE_TIMEOUT s."
+for ((i=0; i<$GRACE_TIMEOUT; ++i)); do
+ pids=$(pidof impalad || true)
+ if [[ -z "$pids" ]]; then
+ LOG "All daemons have exited after $i s."
+ break
+ else
+ LOG "Daemons still running after $i s: $pids"
+ fi
+ sleep 1
+done
+
+LOG "Graceful shutdown process complete."
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 81952e5..6700706 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -37,6 +37,7 @@ add_custom_target(impala_base_image
COMMAND cd ${IMPALA_BASE_BUILD_CONTEXT_DIR} && tar cvh . | docker build -t impala_base -
DEPENDS impala_base_build_context ${CMAKE_SOURCE_DIR}/docker/impala_base/Dockerfile
DEPENDS ${CMAKE_SOURCE_DIR}/docker/daemon_entrypoint.sh
+ DEPENDS ${CMAKE_SOURCE_DIR}/bin/graceful_shutdown_backends.sh
COMMENT "Building Impala base docker image."
VERBATIM
)
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index b953522..47d630a 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -83,3 +83,5 @@ for jar in glob.glob(os.path.join(IMPALA_HOME, "fe/target/impala-frontend-*.jar"
os.symlink(os.path.join(IMPALA_HOME, "www"), os.path.join(OUTPUT_DIR, "www"))
# Scripts
symlink_file_into_dir(os.path.join(IMPALA_HOME, "docker/daemon_entrypoint.sh"), BIN_DIR)
+symlink_file_into_dir(os.path.join(IMPALA_HOME, "bin/graceful_shutdown_backends.sh"),
+ BIN_DIR)
diff --git a/tests/custom_cluster/test_restart_services.py b/tests/custom_cluster/test_restart_services.py
index 781ec5f..5108324 100644
--- a/tests/custom_cluster/test_restart_services.py
+++ b/tests/custom_cluster/test_restart_services.py
@@ -16,6 +16,7 @@
# under the License.
import logging
+import os
import pytest
import psutil
import re
@@ -23,6 +24,7 @@ import signal
import socket
import time
+from subprocess import check_call
from tests.common.environ import build_flavor_timeout
from time import sleep
@@ -544,3 +546,19 @@ class TestGracefulShutdown(CustomClusterTestSuite, HS2TestSuite):
self.assert_impalad_log_contains('INFO', "Shutdown signal received.",
NUM_SIGNALS_TO_SEND)
assert impalad.is_running(), "Impalad process should still be running."
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--shutdown_grace_period_s={grace_period} \
+ --hostname={hostname}".format(grace_period=IDLE_SHUTDOWN_GRACE_PERIOD_S,
+ hostname=socket.gethostname()), cluster_size=1)
+ def test_graceful_shutdown_script(self):
+ impalad = psutil.Process(self.cluster.impalads[0].get_pid())
+ script = os.path.join(os.environ['IMPALA_HOME'], 'bin',
+ 'graceful_shutdown_backends.sh')
+ start_time = time.time()
+ check_call([script, str(self.IDLE_SHUTDOWN_GRACE_PERIOD_S)])
+ LOG.info("Waiting for impalad to exit {0}".format(start_time))
+ impalad.wait()
+ shutdown_duration = time.time() - start_time
+ assert shutdown_duration <= self.IDLE_SHUTDOWN_GRACE_PERIOD_S + 10