You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by xu...@apache.org on 2015/09/09 09:09:10 UTC
[40/50] [abbrv] hive git commit: HIVE-11657 : HIVE-2573 introduces
some issues during metastore init (and CLI init) (Sergey Shelukhin,
reviewed by Sushanth Sowmyan)
HIVE-11657 : HIVE-2573 introduces some issues during metastore init (and CLI init) (Sergey Shelukhin, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/730a4040
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/730a4040
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/730a4040
Branch: refs/heads/beeline-cli
Commit: 730a40401c81e18fe773ac41012125184f776a04
Parents: bb4f5e7
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Sep 3 19:57:40 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Sep 3 19:57:40 2015 -0700
----------------------------------------------------------------------
.../hive/metastore/RetryingMetaStoreClient.java | 32 +++++++----
.../hadoop/hive/ql/exec/FunctionTask.java | 2 +-
.../apache/hadoop/hive/ql/metadata/Hive.java | 56 ++++++++++++++++----
3 files changed, 70 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/730a4040/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
index e282981..5087098 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingMetaStoreClient.java
@@ -161,16 +161,27 @@ public class RetryingMetaStoreClient implements InvocationHandler {
} catch (UndeclaredThrowableException e) {
throw e.getCause();
} catch (InvocationTargetException e) {
- if ((e.getCause() instanceof TApplicationException) ||
- (e.getCause() instanceof TProtocolException) ||
- (e.getCause() instanceof TTransportException)) {
- caughtException = (TException) e.getCause();
- } else if ((e.getCause() instanceof MetaException) &&
- e.getCause().getMessage().matches
- ("(?s).*(JDO[a-zA-Z]*|TApplication|TProtocol|TTransport)Exception.*")) {
- caughtException = (MetaException) e.getCause();
+ Throwable t = e.getCause();
+ if (t instanceof TApplicationException) {
+ TApplicationException tae = (TApplicationException)t;
+ switch (tae.getType()) {
+ case TApplicationException.UNSUPPORTED_CLIENT_TYPE:
+ case TApplicationException.UNKNOWN_METHOD:
+ case TApplicationException.WRONG_METHOD_NAME:
+ case TApplicationException.INVALID_PROTOCOL:
+ throw t;
+ default:
+ // TODO: most other options are probably unrecoverable... throw?
+ caughtException = tae;
+ }
+ } else if ((t instanceof TProtocolException) || (t instanceof TTransportException)) {
+ // TODO: most protocol exceptions are probably unrecoverable... throw?
+ caughtException = (TException)t;
+ } else if ((t instanceof MetaException) && t.getMessage().matches(
+ "(?s).*(JDO[a-zA-Z]*|TProtocol|TTransport)Exception.*")) {
+ caughtException = (MetaException)t;
} else {
- throw e.getCause();
+ throw t;
}
} catch (MetaException e) {
if (e.getMessage().matches("(?s).*(IO|TTransport)Exception.*")) {
@@ -180,7 +191,8 @@ public class RetryingMetaStoreClient implements InvocationHandler {
}
}
- if (retriesMade >= retryLimit) {
+
+ if (retriesMade >= retryLimit) {
throw caughtException;
}
retriesMade++;
http://git-wip-us.apache.org/repos/asf/hive/blob/730a4040/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
index 6390740..7671d29 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionTask.java
@@ -98,7 +98,7 @@ public class FunctionTask extends Task<FunctionWork> {
if (work.getReloadFunctionDesc() != null) {
try {
- Hive.reloadFunctions();
+ Hive.get().reloadFunctions();
} catch (Exception e) {
setException(e);
LOG.error(stringifyException(e));
http://git-wip-us.apache.org/repos/asf/hive/blob/730a4040/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 396c070..c449aee 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -113,6 +113,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
import static org.apache.hadoop.hive.serde.serdeConstants.COLLECTION_DELIM;
@@ -160,24 +161,61 @@ public class Hive {
}
};
+ // Note that while this is an improvement over static initialization, it is still not,
+ // technically, valid, cause nothing prevents us from connecting to several metastores in
+ // the same process. This will still only get the functions from the first metastore.
+ private final static AtomicInteger didRegisterAllFuncs = new AtomicInteger(0);
+ private final static int REG_FUNCS_NO = 0, REG_FUNCS_DONE = 2, REG_FUNCS_PENDING = 1;
+
// register all permanent functions. need improvement
- static {
+ private void registerAllFunctionsOnce() {
+ boolean breakLoop = false;
+ while (!breakLoop) {
+ int val = didRegisterAllFuncs.get();
+ switch (val) {
+ case REG_FUNCS_NO: {
+ if (didRegisterAllFuncs.compareAndSet(val, REG_FUNCS_PENDING)) {
+ breakLoop = true;
+ break;
+ }
+ continue;
+ }
+ case REG_FUNCS_PENDING: {
+ synchronized (didRegisterAllFuncs) {
+ try {
+ didRegisterAllFuncs.wait(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return;
+ }
+ }
+ continue;
+ }
+ case REG_FUNCS_DONE: return;
+ default: throw new AssertionError(val);
+ }
+ }
try {
reloadFunctions();
} catch (Exception e) {
- LOG.warn("Failed to access metastore. This class should not accessed in runtime.",e);
+ LOG.warn("Failed to register all functions.", e);
+ } finally {
+ boolean result = didRegisterAllFuncs.compareAndSet(REG_FUNCS_PENDING, REG_FUNCS_DONE);
+ assert result;
+ synchronized (didRegisterAllFuncs) {
+ didRegisterAllFuncs.notifyAll();
+ }
}
}
- public static void reloadFunctions() throws HiveException {
- Hive db = Hive.get();
- for (Function function : db.getAllFunctions()) {
+ public void reloadFunctions() throws HiveException {
+ for (Function function : getAllFunctions()) {
String functionName = function.getFunctionName();
try {
LOG.info("Registering function " + functionName + " " + function.getClassName());
- FunctionRegistry.registerPermanentFunction(
- FunctionUtils.qualifyFunctionName(functionName, function.getDbName()), function.getClassName(),
- false, FunctionTask.toFunctionResource(function.getResourceUris()));
+ FunctionRegistry.registerPermanentFunction(FunctionUtils.qualifyFunctionName(
+ functionName, function.getDbName()), function.getClassName(), false,
+ FunctionTask.toFunctionResource(function.getResourceUris()));
} catch (Exception e) {
LOG.warn("Failed to register persistent function " +
functionName + ":" + function.getClassName() + ". Ignore and continue.");
@@ -268,6 +306,7 @@ public class Hive {
*/
private Hive(HiveConf c) throws HiveException {
conf = c;
+ registerAllFunctionsOnce();
}
@@ -2675,7 +2714,6 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
//Check if different encryption zones
- HadoopShims.HdfsFileStatus destStatus = null;
HadoopShims.HdfsEncryptionShim hdfsEncryptionShim = SessionState.get().getHdfsEncryptionShim();
return hdfsEncryptionShim != null && (hdfsEncryptionShim.isPathEncrypted(srcf) || hdfsEncryptionShim.isPathEncrypted(destf))
&& !hdfsEncryptionShim.arePathsOnSameEncryptionZone(srcf, destf);