You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2018/02/07 16:50:05 UTC
hive git commit: HIVE-18347 : Allow pluggable dynamic lookup of Hive
Metastores from HiveServer2 (Szehon Ho, reviewed by Vihang K)
Repository: hive
Updated Branches:
refs/heads/master fa9a38978 -> 07492e0d2
HIVE-18347 : Allow pluggable dynamic lookup of Hive Metastores from HiveServer2 (Szehon Ho, reviewed by Vihang K)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/07492e0d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/07492e0d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/07492e0d
Branch: refs/heads/master
Commit: 07492e0d2f1942c1794a3190610e10207c850cf7
Parents: fa9a389
Author: Szehon Ho <sz...@gmail.com>
Authored: Fri Dec 15 11:07:04 2017 +0100
Committer: Szehon Ho <sz...@gmail.com>
Committed: Wed Feb 7 17:49:53 2018 +0100
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 3 +-
.../hive/metastore/HiveMetaStoreClient.java | 110 +++++++++++++------
.../hive/metastore/conf/MetastoreConf.java | 2 +
.../hive/metastore/hooks/URIResolverHook.java | 37 +++++++
4 files changed, 118 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index eca3573..f3e74eb 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -649,7 +649,8 @@ public class HiveConf extends Configuration {
"Number of threads to be allocated for metastore handler for fs operations."),
METASTORE_HBASE_FILE_METADATA_THREADS("hive.metastore.hbase.file.metadata.threads", 1,
"Number of threads to use to read file metadata in background to cache it."),
-
+ METASTORE_URI_RESOLVER("hive.metastore.uri.resolver", "",
+ "If set, fully qualified class name of resolver for hive metastore uri's"),
METASTORETHRIFTCONNECTIONRETRIES("hive.metastore.connect.retries", 3,
"Number of retries while opening a connection to metastore"),
METASTORETHRIFTFAILURERETRIES("hive.metastore.failure.retries", 1,
http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index 2e76e17..23cef8d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -57,13 +57,16 @@ import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.hooks.URIResolverHook;
import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.metastore.utils.ObjectPair;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.thrift.TApplicationException;
import org.apache.thrift.TException;
@@ -112,6 +115,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
private String tokenStrForm;
private final boolean localMetaStore;
private final MetaStoreFilterHook filterHook;
+ private final URIResolverHook uriResolverHook;
private final int fileMetadataBatchSize;
private Map<String, String> currentMetaVars;
@@ -145,6 +149,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
version = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) ? TEST_VERSION : VERSION;
filterHook = loadFilterHooks();
+ uriResolverHook = loadUriResolverHook();
fileMetadataBatchSize = MetastoreConf.getIntVar(
conf, ConfVars.BATCH_RETRIEVE_OBJECTS_MAX);
@@ -172,39 +177,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
// user wants file store based configuration
if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URIS) != null) {
- String metastoreUrisString[] = MetastoreConf.getVar(conf,
- ConfVars.THRIFT_URIS).split(",");
- metastoreUris = new URI[metastoreUrisString.length];
- try {
- int i = 0;
- for (String s : metastoreUrisString) {
- URI tmpUri = new URI(s);
- if (tmpUri.getScheme() == null) {
- throw new IllegalArgumentException("URI: " + s
- + " does not have a scheme");
- }
- metastoreUris[i++] = new URI(
- tmpUri.getScheme(),
- tmpUri.getUserInfo(),
- HadoopThriftAuthBridge.getBridge().getCanonicalHostName(tmpUri.getHost()),
- tmpUri.getPort(),
- tmpUri.getPath(),
- tmpUri.getQuery(),
- tmpUri.getFragment()
- );
-
- }
- // make metastore URIS random
- if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) {
- List uriList = Arrays.asList(metastoreUris);
- Collections.shuffle(uriList);
- metastoreUris = (URI[]) uriList.toArray();
- }
- } catch (IllegalArgumentException e) {
- throw (e);
- } catch (Exception e) {
- MetaStoreUtils.logAndThrowMetaException(e);
- }
+ resolveUris();
} else {
LOG.error("NOT getting uris from conf");
throw new MetaException("MetaStoreURIs not found in conf file");
@@ -249,6 +222,51 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
open();
}
+ private void resolveUris() throws MetaException {
+ String metastoreUrisString[] = MetastoreConf.getVar(conf,
+ ConfVars.THRIFT_URIS).split(",");
+
+ List<URI> metastoreURIArray = new ArrayList<URI>();
+ try {
+ int i = 0;
+ for (String s : metastoreUrisString) {
+ URI tmpUri = new URI(s);
+ if (tmpUri.getScheme() == null) {
+ throw new IllegalArgumentException("URI: " + s
+ + " does not have a scheme");
+ }
+ if (uriResolverHook != null) {
+ metastoreURIArray.addAll(uriResolverHook.resolveURI(tmpUri));
+ } else {
+ metastoreURIArray.add(new URI(
+ tmpUri.getScheme(),
+ tmpUri.getUserInfo(),
+ HadoopThriftAuthBridge.getBridge().getCanonicalHostName(tmpUri.getHost()),
+ tmpUri.getPort(),
+ tmpUri.getPath(),
+ tmpUri.getQuery(),
+ tmpUri.getFragment()
+ ));
+ }
+ }
+ metastoreUris = new URI[metastoreURIArray.size()];
+ for (int j = 0; j < metastoreURIArray.size(); j++) {
+ metastoreUris[j] = metastoreURIArray.get(j);
+ }
+
+ if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) {
+ List uriList = Arrays.asList(metastoreUris);
+ Collections.shuffle(uriList);
+ metastoreUris = (URI[]) uriList.toArray();
+ }
+ } catch (IllegalArgumentException e) {
+ throw (e);
+ } catch (Exception e) {
+ MetaStoreUtils.logAndThrowMetaException(e);
+ }
+ }
+
+
private MetaStoreFilterHook loadFilterHooks() throws IllegalStateException {
Class<? extends MetaStoreFilterHook> authProviderClass = MetastoreConf.
getClass(conf, ConfVars.FILTER_HOOK, DefaultMetaStoreFilterHookImpl.class,
@@ -263,6 +281,26 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
}
}
+ //multiple clients may initialize the hook at the same time
+ synchronized private URIResolverHook loadUriResolverHook() throws IllegalStateException {
+
+ String uriResolverClassName =
+ MetastoreConf.getAsString(conf, ConfVars.URI_RESOLVER);
+ if (uriResolverClassName.equals("")) {
+ return null;
+ } else {
+ LOG.info("Loading uri resolver" + uriResolverClassName);
+ try {
+ Class<?> uriResolverClass = Class.forName(uriResolverClassName, true,
+ JavaUtils.getClassLoader());
+ return (URIResolverHook) ReflectionUtils.newInstance(uriResolverClass, null);
+ } catch (Exception e) {
+ LOG.error("Exception loading uri resolver hook" + e);
+ return null;
+ }
+ }
+ }
+
/**
* Swaps the first element of the metastoreUris array with a random element from the
* remainder of the array.
@@ -324,6 +362,12 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable {
" at the client level.");
} else {
close();
+
+ if (uriResolverHook != null) {
+ //for dynamic uris, re-lookup if there are new metastore locations
+ resolveUris();
+ }
+
if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) {
// Swap the first element of the metastoreUris[] with a random element from the rest
// of the array. Rationale being that this method will generally be called when the default
http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 5ef8d7a..5b8e5ca 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -777,6 +777,8 @@ public class MetastoreConf {
"class is used to store and retrieve transactions and locks"),
TXN_TIMEOUT("metastore.txn.timeout", "hive.txn.timeout", 300, TimeUnit.SECONDS,
"time after which transactions are declared aborted if the client has not sent a heartbeat."),
+ URI_RESOLVER("metastore.uri.resolver", "hive.metastore.uri.resolver", "",
+ "If set, fully qualified class name of resolver for hive metastore uri's"),
USERS_IN_ADMIN_ROLE("metastore.users.in.admin.role", "hive.users.in.admin.role", "", false,
"Comma separated list of users who are in admin role for bootstrapping.\n" +
"More users can be added in ADMIN role later."),
http://git-wip-us.apache.org/repos/asf/hive/blob/07492e0d/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java
new file mode 100644
index 0000000..d3be5dd
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/hooks/URIResolverHook.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.hooks;
+
+import org.apache.hadoop.hive.metastore.HiveMetaException;
+
+import java.net.URI;
+import java.util.List;
+
+/**
+ * Allows different metastore uris to be resolved.
+ */
+public interface URIResolverHook {
+
+ /**
+ * Resolve to a proper thrift uri, or a list of uris, given uri of another scheme.
+ * @param uri
+ * @return
+ */
+ public List<URI> resolveURI(URI uri) throws HiveMetaException;
+}