You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/07/23 18:51:48 UTC
[08/13] hive git commit: HIVE-20192: HS2 with embedded metastore is
leaking JDOPersistenceManager objects (Sankar Hariappan,
reviewed by Vihang Karajgaonkar)
HIVE-20192: HS2 with embedded metastore is leaking JDOPersistenceManager objects (Sankar Hariappan, reviewed by Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6b158162
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6b158162
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6b158162
Branch: refs/heads/master-txnstats
Commit: 6b158162376916433e3431f7e227771efe68b204
Parents: 170a012
Author: Sankar Hariappan <sa...@apache.org>
Authored: Mon Jul 23 14:18:50 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Mon Jul 23 14:18:50 2018 +0530
----------------------------------------------------------------------
.../apache/hive/service/server/ThreadWithGarbageCleanup.java | 6 +++++-
.../java/org/apache/hadoop/hive/metastore/HiveMetaStore.java | 8 +++++++-
.../java/org/apache/hadoop/hive/metastore/ObjectStore.java | 7 ++++---
3 files changed, 16 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java b/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
index 1ec8097..a8ed93e 100644
--- a/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
+++ b/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java
@@ -68,7 +68,11 @@ public class ThreadWithGarbageCleanup extends Thread {
public void cacheThreadLocalRawStore() {
Long threadId = this.getId();
RawStore threadLocalRawStore = HiveMetaStore.HMSHandler.getRawStore();
- if (threadLocalRawStore != null && !threadRawStoreMap.containsKey(threadId)) {
+ if (threadLocalRawStore == null) {
+ LOG.debug("Thread Local RawStore is null, for the thread: " +
+ this.getName() + " and so removing entry from threadRawStoreMap.");
+ threadRawStoreMap.remove(threadId);
+ } else {
LOG.debug("Adding RawStore: " + threadLocalRawStore + ", for the thread: " +
this.getName() + " to threadRawStoreMap for future cleanup.");
threadRawStoreMap.put(threadId, threadLocalRawStore);
http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 47f819b..7fe5777 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -698,9 +698,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
RawStore ms = threadLocalMS.get();
if (ms == null) {
ms = newRawStoreForConf(conf);
- ms.verifySchema();
+ try {
+ ms.verifySchema();
+ } catch (MetaException e) {
+ ms.shutdown();
+ throw e;
+ }
threadLocalMS.set(ms);
ms = threadLocalMS.get();
+ LOG.info("Created RawStore: " + ms + " from thread id: " + Thread.currentThread().getId());
}
return ms;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6b158162/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index bdcbf41..6f8276f 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -425,6 +425,7 @@ public class ObjectStore implements RawStore, Configurable {
initializeHelper(dsProps);
return; // If we reach here, we succeed.
} catch (Exception e){
+ shutdown();
numTries--;
boolean retriable = isRetriableException(e);
if ((numTries > 0) && retriable){
@@ -486,6 +487,8 @@ public class ObjectStore implements RawStore, Configurable {
LOG.debug("ObjectStore, initialize called");
prop = dsProps;
pm = getPersistenceManager();
+ LOG.info("RawStore: {}, with PersistenceManager: {}" +
+ " created in the thread with id: {}", this, pm, Thread.currentThread().getId());
try {
String productName = MetaStoreDirectSql.getProductName(pm);
sqlGenerator = new SQLGenerator(DatabaseProduct.determineDatabaseProduct(productName), conf);
@@ -503,8 +506,6 @@ public class ObjectStore implements RawStore, Configurable {
directSql = new MetaStoreDirectSql(pm, conf, schema);
}
}
- LOG.debug("RawStore: {}, with PersistenceManager: {}" +
- " created in the thread with id: {}", this, pm, Thread.currentThread().getId());
}
private DatabaseProduct determineDatabaseProduct() {
@@ -703,7 +704,7 @@ public class ObjectStore implements RawStore, Configurable {
@Override
public void shutdown() {
- LOG.debug("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm);
+ LOG.info("RawStore: {}, with PersistenceManager: {} will be shutdown", this, pm);
if (pm != null) {
pm.close();
pm = null;