You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by de...@apache.org on 2022/08/30 01:06:46 UTC
[hive] branch master updated: HIVE-22193: Graceful shutdown HiveServer2(addendum) (#3530)
This is an automated email from the ASF dual-hosted git repository.
dengzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 74c4aa91509 HIVE-22193: Graceful shutdown HiveServer2(addendum) (#3530)
74c4aa91509 is described below
commit 74c4aa91509a39e4fcaaccf6c44d0fe258beeecd
Author: dengzh <de...@gmail.com>
AuthorDate: Tue Aug 30 09:06:33 2022 +0800
HIVE-22193: Graceful shutdown HiveServer2(addendum) (#3530)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 5 ++-
.../hive/common/util/ShutdownHookManager.java | 46 ++++++++++++++++++----
.../hive/service/server/TestGracefulStopHS2.java | 3 +-
.../apache/hive/service/server/HiveServer2.java | 9 +++--
4 files changed, 49 insertions(+), 14 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index f96f3859ff3..9452507d035 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -4412,8 +4412,9 @@ public class HiveConf extends Configuration {
"credential providers for jobs run using Tez, MR execution engines."),
HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT("hive.server2.graceful.stop.timeout", "1800s",
new TimeValidator(TimeUnit.SECONDS),
- "Maximum time waiting for the current live operations being finished before shutting down HiveServer2 gracefully.\n" +
- " With value less than or equal to 0, it does not check for the operations regardless of state to shut down HiveServer2."),
+ "Maximum time waiting for live queries being finished and stopping HiveServer2. "
+ + "With value not greater than 30s(the overhead to stop HiveServer2), it will not wait for the live queries to be done, "
+ + "instead call stop directly to shutdown HiveServer2 gracefully"),
HIVE_MOVE_FILES_THREAD_COUNT("hive.mv.files.thread", 15, new SizeValidator(0L, true, 1024L, true), "Number of threads"
+ " used to move files in move task. Set it to 0 to disable multi-threaded file moves. This parameter is also used by"
+ " MSCK to check tables."),
diff --git a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
index 6083fd52bb6..6b32410af6b 100644
--- a/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
+++ b/common/src/java/org/apache/hive/common/util/ShutdownHookManager.java
@@ -41,8 +41,17 @@ public class ShutdownHookManager {
static final private Logger LOG = LoggerFactory.getLogger(ShutdownHookManager.class.getName());
+ // The graceful shutdown hook's priority must be higher than other hooks'
+ public static final int GRACEFUL_SHUTDOWN_HOOK_PRIORITY = 1000;
+
+ // The higher the priority the earlier will run. ShutdownHooks with same priority run
+ // in a non-deterministic order.
+ public static final int DEFAULT_SHUTDOWN_HOOK_PRIORITY = FileSystem.SHUTDOWN_HOOK_PRIORITY;
+
+ public static final int DELETE_ON_EXIT_HOOK_PRIORITY = -1;
+
static {
- MGR.addShutdownHook(DELETE_ON_EXIT_HOOK, -1);
+ MGR.addShutdownHook(DELETE_ON_EXIT_HOOK, DELETE_ON_EXIT_HOOK_PRIORITY);
}
/**
@@ -50,7 +59,26 @@ public class ShutdownHookManager {
* @param shutdownHook - shutdown hook
*/
public static void addShutdownHook(Runnable shutdownHook) {
- addShutdownHook(shutdownHook, FileSystem.SHUTDOWN_HOOK_PRIORITY);
+ addShutdownHook(shutdownHook, DEFAULT_SHUTDOWN_HOOK_PRIORITY);
+ }
+
+ /**
+ * Adds shutdown hook with a priority and default timeout (30s)
+ * @param shutdownHook shutdown hook
+ * @param priority priority of the shutdownHook
+ */
+ public static void addShutdownHook(Runnable shutdownHook, int priority) {
+ addShutdownHook(shutdownHook, priority, 30);
+ }
+
+ /**
+ * Adds a server's graceful shutdown hook with the highest priority
+ * and the given timeout, so the hook runs earlier than other kinds of hooks.
+ * @param shutdownHook shutdown hook
+ * @param timeout timeout of the shutdownHook
+ */
+ public static void addGracefulShutDownHook(Runnable shutdownHook, long timeout) {
+ addShutdownHook(shutdownHook, GRACEFUL_SHUTDOWN_HOOK_PRIORITY, timeout);
}
/**
@@ -60,12 +88,16 @@ public class ShutdownHookManager {
*
* @param shutdownHook shutdownHook <code>Runnable</code>
* @param priority priority of the shutdownHook.
+ * @param timeout timeout of the shutdownHook.
*/
- public static void addShutdownHook(Runnable shutdownHook, int priority) {
- if (priority < 0) {
- throw new IllegalArgumentException("Priority should be greater than or equal to zero");
+ private static void addShutdownHook(Runnable shutdownHook, int priority, long timeout) {
+ if (priority < 0 || priority > GRACEFUL_SHUTDOWN_HOOK_PRIORITY) {
+ throw new IllegalArgumentException("Priority should be ranged between 0 and " +
+ GRACEFUL_SHUTDOWN_HOOK_PRIORITY);
+ }
+ if (!isShutdownInProgress()) {
+ MGR.addShutdownHook(shutdownHook, priority, timeout, TimeUnit.SECONDS);
}
- MGR.addShutdownHook(shutdownHook, priority, 30, TimeUnit.SECONDS);
}
/**
@@ -85,7 +117,7 @@ public class ShutdownHookManager {
* FALSE otherwise (including when shutdownHook == null)
*/
public static boolean removeShutdownHook(Runnable shutdownHook) {
- if (shutdownHook == null) {
+ if (shutdownHook == null || isShutdownInProgress()) {
return false;
}
return MGR.removeShutdownHook(shutdownHook);
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java
index 3f1383a03d9..164a63315e4 100755
--- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/TestGracefulStopHS2.java
@@ -20,7 +20,6 @@ package org.apache.hive.service.server;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.jdbc.miniHS2.MiniHS2;
-import org.apache.hive.service.Service;
import org.apache.hive.service.cli.HiveSQLException;
import org.apache.hive.service.cli.session.SessionManager;
import org.junit.AfterClass;
@@ -53,7 +52,7 @@ public class TestGracefulStopHS2 {
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false);
conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_LOGGING_OPERATION_ENABLED, false);
conf.setBoolVar(HiveConf.ConfVars.HIVESTATSCOLAUTOGATHER, false);
- conf.setTimeVar(HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, 40, TimeUnit.SECONDS);
+ conf.setTimeVar(HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, 60, TimeUnit.SECONDS);
MiniHS2.Builder builder = new MiniHS2.Builder().withConf(conf).cleanupLocalDirOnStartup(false);
miniHS2 = builder.build();
miniHS2.start(new HashMap<>());
diff --git a/service/src/java/org/apache/hive/service/server/HiveServer2.java b/service/src/java/org/apache/hive/service/server/HiveServer2.java
index e48b981dad6..8acf013b298 100644
--- a/service/src/java/org/apache/hive/service/server/HiveServer2.java
+++ b/service/src/java/org/apache/hive/service/server/HiveServer2.java
@@ -439,7 +439,10 @@ public class HiveServer2 extends CompositeService {
}
// Add a shutdown hook for catching SIGTERM & SIGINT
- ShutdownHookManager.addShutdownHook(() -> graceful_stop());
+ long timeout = HiveConf.getTimeVar(getHiveConf(),
+ HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, TimeUnit.SECONDS);
+ // Extra time for releasing the resources if timeout sets to 0
+ ShutdownHookManager.addGracefulShutDownHook(() -> graceful_stop(), timeout == 0 ? 30 : timeout);
}
private void logCompactionParameters(HiveConf hiveConf) {
@@ -918,9 +921,9 @@ public class HiveServer2 extends CompositeService {
public synchronized void graceful_stop() {
try {
decommission();
+ // Need 30s for stop() to release server's resources
long maxTimeForWait = HiveConf.getTimeVar(getHiveConf(),
- HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, TimeUnit.MILLISECONDS);
-
+ HiveConf.ConfVars.HIVE_SERVER2_GRACEFUL_STOP_TIMEOUT, TimeUnit.MILLISECONDS) - 30000;
long timeout = maxTimeForWait, startTime = System.currentTimeMillis();
try {
// The service should be started before when reaches here, as decommissioning would throw