You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kh...@apache.org on 2015/05/07 10:13:07 UTC
hive git commit: HIVE-8696 : HCatClientHMSImpl doesn't use a
Retrying-HiveMetastoreClient (Thiruvel Thirumoolan via Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master 26ec033c8 -> e0044e072
HIVE-8696 : HCatClientHMSImpl doesn't use a Retrying-HiveMetastoreClient (Thiruvel Thirumoolan via Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e0044e07
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e0044e07
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e0044e07
Branch: refs/heads/master
Commit: e0044e0723d8e831ad0f29c6615b5f70e2ee0658
Parents: 26ec033
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Thu May 7 01:12:11 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Thu May 7 01:13:05 2015 -0700
----------------------------------------------------------------------
.../hive/hcatalog/common/HiveClientCache.java | 9 +++-
.../hcatalog/mapreduce/TestPassProperties.java | 5 +-
.../hive/metastore/RetryingMetaStoreClient.java | 53 ++++++++++++++++----
3 files changed, 55 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
index 578b6ea..0966581 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/common/HiveClientCache.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.hive.shims.Utils;
@@ -81,7 +82,7 @@ class HiveClientCache {
}
public static IMetaStoreClient getNonCachedHiveMetastoreClient(HiveConf hiveConf) throws MetaException {
- return new HiveMetaStoreClient(hiveConf);
+ return RetryingMetaStoreClient.getProxy(hiveConf);
}
public HiveClientCache(HiveConf hiveConf) {
@@ -226,7 +227,11 @@ class HiveClientCache {
return hiveCache.get(cacheKey, new Callable<ICacheableMetaStoreClient>() {
@Override
public ICacheableMetaStoreClient call() throws MetaException {
- return new CacheableHiveMetaStoreClient(cacheKey.getHiveConf(), timeout);
+ return
+ (ICacheableMetaStoreClient) RetryingMetaStoreClient.getProxy(cacheKey.getHiveConf(),
+ new Class<?>[]{HiveConf.class, Integer.class},
+ new Object[]{cacheKey.getHiveConf(), timeout},
+ CacheableHiveMetaStoreClient.class.getName());
}
});
} catch (ExecutionException e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
index 735ab5f..8673b48 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestPassProperties.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
@@ -109,8 +110,10 @@ public class TestPassProperties {
new FileOutputCommitterContainer(job, null).cleanupJob(job);
} catch (Exception e) {
caughtException = true;
- assertTrue(e.getCause().getMessage().contains(
+ assertTrue(((InvocationTargetException)e.getCause().getCause().getCause()).getTargetException().getMessage().contains(
"Could not connect to meta store using any of the URIs provided"));
+ assertTrue(e.getCause().getMessage().contains(
+ "Unable to instantiate org.apache.hive.hcatalog.common.HiveClientCache$CacheableHiveMetaStoreClient"));
}
assertTrue(caughtException);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/e0044e07/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index 1b6487a..fb44484 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -60,6 +60,18 @@ public class RetryingMetaStoreClient implements InvocationHandler {
protected RetryingMetaStoreClient(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass) throws MetaException {
+
+ this(hiveConf,
+ new Class[] {HiveConf.class, HiveMetaHookLoader.class},
+ new Object[] {hiveConf, hookLoader},
+ metaCallTimeMap,
+ msClientClass);
+ }
+
+ protected RetryingMetaStoreClient(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+ Object[] constructorArgs, Map<String, Long> metaCallTimeMap, Class<? extends IMetaStoreClient> msClientClass)
+ throws MetaException {
+
this.retryLimit = hiveConf.getIntVar(HiveConf.ConfVars.METASTORETHRIFTFAILURERETRIES);
this.retryDelaySeconds = hiveConf.getTimeVar(
HiveConf.ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, TimeUnit.SECONDS);
@@ -71,12 +83,14 @@ public class RetryingMetaStoreClient implements InvocationHandler {
localMetaStore = (msUri == null) || msUri.trim().isEmpty();
reloginExpiringKeytabUser();
- this.base = MetaStoreUtils.newInstance(msClientClass, new Class[] {
- HiveConf.class, HiveMetaHookLoader.class}, new Object[] {hiveConf, hookLoader});
+ this.base = (IMetaStoreClient) MetaStoreUtils.newInstance(msClientClass, constructorArgTypes, constructorArgs);
}
- public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) throws MetaException {
- return getProxy(hiveConf, hookLoader, null, HiveMetaStoreClient.class.getName());
+ public static IMetaStoreClient getProxy(HiveConf hiveConf) throws MetaException {
+
+ return getProxy(hiveConf, new Class[]{HiveConf.class}, new Object[]{hiveConf}, null,
+ HiveMetaStoreClient.class.getName()
+ );
}
public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
@@ -84,19 +98,40 @@ public class RetryingMetaStoreClient implements InvocationHandler {
return getProxy(hiveConf, hookLoader, null, mscClassName);
}
+ public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
+ Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException {
+
+ return getProxy(hiveConf,
+ new Class[] {HiveConf.class, HiveMetaHookLoader.class},
+ new Object[] {hiveConf, hookLoader},
+ metaCallTimeMap,
+ mscClassName
+ );
+ }
+
/**
* This constructor is meant for Hive internal use only.
* Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose.
*/
- public static IMetaStoreClient getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader,
- Map<String, Long> metaCallTimeMap, String mscClassName) throws MetaException {
+ public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+ Object[] constructorArgs, String mscClassName) throws MetaException {
+ return getProxy(hiveConf, constructorArgTypes, constructorArgs, null, mscClassName);
+ }
+
+ /**
+ * This constructor is meant for Hive internal use only.
+ * Please use getProxy(HiveConf hiveConf, HiveMetaHookLoader hookLoader) for external purpose.
+ */
+ public static IMetaStoreClient getProxy(HiveConf hiveConf, Class<?>[] constructorArgTypes,
+ Object[] constructorArgs, Map<String, Long> metaCallTimeMap,
+ String mscClassName) throws MetaException {
Class<? extends IMetaStoreClient> baseClass = (Class<? extends IMetaStoreClient>) MetaStoreUtils
.getClass(mscClassName);
- RetryingMetaStoreClient handler = new RetryingMetaStoreClient(hiveConf, hookLoader,
- metaCallTimeMap, baseClass);
-
+ RetryingMetaStoreClient handler =
+ new RetryingMetaStoreClient(hiveConf, constructorArgTypes, constructorArgs,
+ metaCallTimeMap, baseClass);
return (IMetaStoreClient) Proxy.newProxyInstance(
RetryingMetaStoreClient.class.getClassLoader(), baseClass.getInterfaces(), handler);
}