You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2022/05/23 06:56:44 UTC
[hive] branch master updated: HIVE-26237: Check if replication cause metastore connection leakage (Haymant Mangla, reviewed by Denys Kuzmenko, Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new ad92d0d4cb6 HIVE-26237: Check if replication cause metastore connection leakage (Haymant Mangla, reviewed by Denys Kuzmenko, Peter Vary)
ad92d0d4cb6 is described below
commit ad92d0d4cb6ebf582322e2f457e813ed4b407fee
Author: Haymant Mangla <79...@users.noreply.github.com>
AuthorDate: Mon May 23 12:26:37 2022 +0530
HIVE-26237: Check if replication cause metastore connection leakage (Haymant Mangla, reviewed by Denys Kuzmenko, Peter Vary)
Closes #3298
---
.../hive/ql/parse/BaseReplicationAcrossInstances.java | 2 ++
.../hadoop/hive/ql/parse/TestReplicationScenarios.java | 14 +++++++++-----
.../org/apache/hadoop/hive/ql/parse/WarehouseInstance.java | 3 +++
.../org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java | 8 +++-----
.../hive/ql/ddl/table/info/desc/DescTableOperation.java | 2 +-
.../concatenate/AlterTableConcatenateOperation.java | 2 +-
.../org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java | 7 +++----
7 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java
index e5807ee0d3d..f5f8dbdd898 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/BaseReplicationAcrossInstances.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.shims.Utils;
import org.junit.After;
import org.junit.AfterClass;
@@ -121,6 +122,7 @@ public class BaseReplicationAcrossInstances {
public static void classLevelTearDown() throws IOException {
primary.close();
replica.close();
+ Hive.closeCurrent();
}
private static void setFullyQualifiedReplicaExternalTableBase(FileSystem fs) throws IOException {
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
index 024c242b3a7..afb8648ec34 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hive.common.repl.ReplConst;
import org.apache.hadoop.hive.common.repl.ReplScope;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.repl.ReplAck;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.StringAppender;
import org.apache.hadoop.hive.ql.parse.repl.metric.MetricCollector;
import org.apache.hadoop.hive.ql.parse.repl.metric.event.Metadata;
@@ -262,17 +263,20 @@ public class TestReplicationScenarios {
// FIXME : should clean up TEST_PATH, but not doing it now, for debugging's sake
//Clean up the warehouse after test run as we are restoring the warehouse path for other metastore creation
Path warehousePath = new Path(MetastoreConf.getVar(hconf, MetastoreConf.ConfVars.WAREHOUSE));
- try {
- warehousePath.getFileSystem(hconf).delete(warehousePath, true);
- } catch (IOException e) {
-
- }
Path warehousePathReplica = new Path(MetastoreConf.getVar(hconfMirror, MetastoreConf.ConfVars.WAREHOUSE));
try {
+ warehousePath.getFileSystem(hconf).delete(warehousePath, true);
warehousePathReplica.getFileSystem(hconfMirror).delete(warehousePathReplica, true);
} catch (IOException e) {
}
+ Hive.closeCurrent();
+ if (metaStoreClient != null) {
+ metaStoreClient.close();
+ }
+ if (metaStoreClientMirror != null) {
+ metaStoreClientMirror.close();
+ }
}
@Before
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index 948bd50b558..3695a7e03b3 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -612,6 +612,9 @@ public class WarehouseInstance implements Closeable {
if (miniDFSCluster != null && miniDFSCluster.isClusterUp()) {
miniDFSCluster.shutdown();
}
+ if (client != null) {
+ client.close();
+ }
}
CurrentNotificationEventId getCurrentNotificationEventId() throws Exception {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
index 6db3a6a1ba6..6a38854a652 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/DDLOperationContext.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
* Context for DDL operations.
*/
public class DDLOperationContext {
- private final Hive db;
private final HiveConf conf;
private final Context context;
private final DDLTask task;
@@ -40,8 +39,7 @@ public class DDLOperationContext {
private final LogHelper console;
public DDLOperationContext(HiveConf conf, Context context, DDLTask task, DDLWork work, QueryState queryState,
- QueryPlan queryPlan, LogHelper console) throws HiveException {
- this.db = Hive.get(conf);
+ QueryPlan queryPlan, LogHelper console){
this.conf = conf;
this.context = context;
this.task = task;
@@ -51,8 +49,8 @@ public class DDLOperationContext {
this.console = console;
}
- public Hive getDb() {
- return db;
+ public Hive getDb() throws HiveException {
+ return Hive.get(conf);
}
public HiveConf getConf() {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
index d21673c43b4..6aee8dedb28 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableOperation.java
@@ -293,7 +293,7 @@ public class DescTableOperation extends DDLOperation<DescTableDesc> {
}
}
- private void handleMaterializedView(Table table) throws LockException {
+ private void handleMaterializedView(Table table) throws HiveException {
if (table.isMaterializedView()) {
table.setOutdatedForRewriting(context.getDb().isOutdatedMaterializedView(
table,
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
index 261f14705dd..88097df79a7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/storage/concatenate/AlterTableConcatenateOperation.java
@@ -63,7 +63,7 @@ public class AlterTableConcatenateOperation extends DDLOperation<AlterTableConca
return executeTask(generalContext, task);
}
- private MergeFileWork getMergeFileWork(CompilationOpContext opContext) {
+ private MergeFileWork getMergeFileWork(CompilationOpContext opContext) throws HiveException {
List<Path> inputDirList = Lists.newArrayList(desc.getInputDir());
// merge work only needs input and output.
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
index a8a333f640e..096fe24face 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.TableName;
import org.apache.hadoop.hive.common.repl.ReplScope;
import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.utils.SecurityUtils;
@@ -96,7 +96,6 @@ import static org.apache.hadoop.hive.ql.exec.repl.OptimisedBootstrapUtils.checkF
import static org.apache.hadoop.hive.ql.exec.repl.OptimisedBootstrapUtils.getEventIdFromFile;
import static org.apache.hadoop.hive.ql.exec.repl.OptimisedBootstrapUtils.prepareTableDiffFile;
import static org.apache.hadoop.hive.ql.exec.repl.ReplAck.LOAD_METADATA;
-import static org.apache.hadoop.hive.ql.exec.repl.ReplExternalTables.getExternalTableBaseDir;
import static org.apache.hadoop.hive.ql.exec.repl.bootstrap.load.LoadDatabase.AlterDatabase;
import static org.apache.hadoop.hive.ql.exec.repl.ReplAck.LOAD_ACKNOWLEDGEMENT;
import static org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils.RANGER_AUTHORIZER;
@@ -549,8 +548,8 @@ public class ReplLoadTask extends Task<ReplLoadWork> implements Serializable {
@Override
public void run() throws SemanticException {
try {
- HiveMetaStoreClient metaStoreClient = new HiveMetaStoreClient(conf);
- long currentNotificationID = metaStoreClient.getCurrentNotificationEventId().getEventId();
+ IMetaStoreClient client = getHive().getMSC();
+ long currentNotificationID = client.getCurrentNotificationEventId().getEventId();
Path loadMetadataFilePath = new Path(work.dumpDirectory, LOAD_METADATA.toString());
Utils.writeOutput(String.valueOf(currentNotificationID), loadMetadataFilePath, conf);
LOG.info("Created LOAD Metadata file : {} with NotificationID : {}",