You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pr...@apache.org on 2015/05/07 03:21:28 UTC
[34/52] [abbrv] hive git commit: HIVE-9508: MetaStore client socket
connection should have a lifetime (Thiruvel Thirumoolan reviewed by Vaibhav
Gumashta)
HIVE-9508: MetaStore client socket connection should have a lifetime (Thiruvel Thirumoolan reviewed by Vaibhav Gumashta)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e2a12c9a
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e2a12c9a
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e2a12c9a
Branch: refs/heads/llap
Commit: e2a12c9a630d037f3e3aaf42acc873eac86bc9f3
Parents: 652febc
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Tue May 5 10:44:16 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Tue May 5 10:44:16 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 6 ++++
.../hive/metastore/TestHiveMetaStore.java | 30 ++++++++++++++++++++
.../hive/metastore/RetryingMetaStoreClient.java | 27 +++++++++++++++---
3 files changed, 59 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 5d4dbea..69fda45 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -119,6 +119,7 @@ public class HiveConf extends Configuration {
HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES,
HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY,
HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
+ HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME,
HiveConf.ConfVars.METASTOREPWD,
HiveConf.ConfVars.METASTORECONNECTURLHOOK,
HiveConf.ConfVars.METASTORECONNECTURLKEY,
@@ -398,6 +399,11 @@ public class HiveConf extends Configuration {
METASTORE_CLIENT_SOCKET_TIMEOUT("hive.metastore.client.socket.timeout", "600s",
new TimeValidator(TimeUnit.SECONDS),
"MetaStore Client socket timeout in seconds"),
+ METASTORE_CLIENT_SOCKET_LIFETIME("hive.metastore.client.socket.lifetime", "0s",
+ new TimeValidator(TimeUnit.SECONDS),
+ "MetaStore Client socket lifetime in seconds. After this time is exceeded, client\n" +
+ "reconnects on the next MetaStore operation. A value of 0s means the connection\n" +
+ "has an infinite lifetime."),
METASTOREPWD("javax.jdo.option.ConnectionPassword", "mine",
"password to use against metastore database"),
METASTORECONNECTURLHOOK("hive.metastore.ds.connection.url.hook", "",
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
index 130fd67..dffeb34 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
@@ -2867,4 +2867,34 @@ public abstract class TestHiveMetaStore extends TestCase {
ownerName, ownerType, createTime, functionType, resources);
client.createFunction(func);
}
+
+ public void testRetriableClientWithConnLifetime() throws Exception {
+
+ HiveConf conf = new HiveConf(hiveConf);
+ conf.setLong(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME.name(), 60);
+ long timeout = 65 * 1000; // Lets use a timeout more than the socket lifetime to simulate a reconnect
+
+ // Test a normal retriable client
+ IMetaStoreClient client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+ client.getAllDatabases();
+ client.close();
+
+ // Connect after the lifetime, there should not be any failures
+ client = RetryingMetaStoreClient.getProxy(conf, getHookLoader(), HiveMetaStoreClient.class.getName());
+ Thread.sleep(timeout);
+ client.getAllDatabases();
+ client.close();
+ }
+
+ private HiveMetaHookLoader getHookLoader() {
+ HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+ @Override
+ public HiveMetaHook getHook(
+ org.apache.hadoop.hive.metastore.api.Table tbl)
+ throws MetaException {
+ return null;
+ }
+ };
+ return hookLoader;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e2a12c9a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index 77da6f7..1b6487a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -54,9 +54,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
private final int retryLimit;
private final long retryDelaySeconds;
private final Map<String, Long> metaCallTimeMap;
-
-
-
+ private final long connectionLifeTimeInMillis;
+ private long lastConnectionTime;
+ private boolean localMetaStore;
protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException {
@@ -64,6 +64,11 @@ public class RetryingMetaStoreClient implements InvocationHandler {
this.retryDelaySeconds = hiveConf.getTimeVar(
HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
this.metaCallTimeMap = metaCallTimeMap;
+ this.connectionLifeTimeInMillis =
+ hiveConf.getTimeVar(HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_LIFETIME, TimeUnit.SECONDS) * 1000;
+ this.lastConnectionTime = System.currentTimeMillis();
+ String msUri = hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS);
+ localMetaStore = (msUri == null) || msUri.trim().isEmpty();
reloginExpiringKeytabUser();
this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] {
@@ -104,8 +109,9 @@ public class RetryingMetaStoreClient implements InvocationHandler {
while (true) {
try {
reloginExpiringKeytabUser();
- if(retriesMade > 0){
+ if (retriesMade > 0 || hasConnectionLifeTimeReached(method)) {
base.reconnect();
+ lastConnectionTime = System.currentTimeMillis();
}
if (metaCallTimeMap == null) {
ret = method.invoke(base, args);
@@ -171,6 +177,19 @@ public class RetryingMetaStoreClient implements InvocationHandler {
return methodSb.toString();
}
+ private boolean hasConnectionLifeTimeReached(Method method) {
+ if (connectionLifeTimeInMillis <= 0 || localMetaStore ||
+ method.getName().equalsIgnoreCase("close")) {
+ return false;
+ }
+ boolean shouldReconnect =
+ (System.currentTimeMillis() - lastConnectionTime) >= connectionLifeTimeInMillis;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reconnection status for Method: " + method.getName() + " is " + shouldReconnect);
+ }
+ return shouldReconnect;
+ }
+
/**
* Relogin if login user is logged in using keytab
* Relogin is actually done by ugi code only if sufficient time has passed