You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2020/01/13 14:33:09 UTC

[hive] branch master updated: HIVE-22596: RawStore used by Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)

This is an automated email from the ASF dual-hosted git repository.

pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 31c19a0  HIVE-22596: RawStore used by Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)
31c19a0 is described below

commit 31c19a08b97babd7dda0f9f7fef1b1e2b57bf5d7
Author: denys kuzmenko <dk...@cloudera.com>
AuthorDate: Mon Jan 13 15:32:36 2020 +0100

    HIVE-22596: RawStore used by Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)
---
 .../org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java  |  8 ++++----
 .../hadoop/hive/ql/txn/compactor/CompactorThread.java     |  2 +-
 .../hive/ql/txn/compactor/MetaStoreCompactorThread.java   | 15 ++++-----------
 3 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 5dfa7ca..6f64290 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.ValidWriteIdList;
@@ -55,6 +54,7 @@ import java.util.List;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler.getMSForConf;
 import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
 
 /**
@@ -224,7 +224,7 @@ public class Cleaner extends MetaStoreCompactorThread {
     return " id=" + ci.id;
   }
   private void removeFiles(String location, ValidWriteIdList writeIdList, CompactionInfo ci)
-          throws IOException, NoSuchObjectException {
+      throws IOException, NoSuchObjectException, MetaException {
     Path locPath = new Path(location);
     AcidUtils.Directory dir = AcidUtils.getAcidState(locPath.getFileSystem(conf), locPath, conf, writeIdList, Ref.from(
         false), false, null, false);
@@ -258,8 +258,8 @@ public class Cleaner extends MetaStoreCompactorThread {
     }
 
     FileSystem fs = filesToDelete.get(0).getFileSystem(conf);
-    Database db = rs.getDatabase(getDefaultCatalog(conf), ci.dbname);
-    Boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db);
+    Database db = getMSForConf(conf).getDatabase(getDefaultCatalog(conf), ci.dbname);
+    boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db);
 
     for (Path dead : filesToDelete) {
       LOG.debug("Going to delete path " + dead.toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 99da86f..b378d40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -53,7 +53,7 @@ public abstract class CompactorThread extends Thread implements Configurable {
   protected static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
 
   protected HiveConf conf;
-  protected RawStore rs;
+
   protected AtomicBoolean stop;
   protected AtomicBoolean looped;
 
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
index 8bfb524..aa258b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
@@ -18,14 +18,11 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import org.apache.hadoop.hive.metastore.MetaStoreThread;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.RawStoreProxy;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
 import org.apache.hadoop.hive.metastore.api.Partition;
 import org.apache.hadoop.hive.metastore.api.Table;
 import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
@@ -36,6 +33,7 @@ import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler.getMSForConf;
 import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
 
 /**
@@ -45,7 +43,6 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCa
 public class MetaStoreCompactorThread extends CompactorThread implements MetaStoreThread {
 
   protected TxnStore txnHandler;
-  protected RawStore rs;
   protected int threadId;
 
   @Override
@@ -59,15 +56,11 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
 
     // Get our own instance of the transaction handler
     txnHandler = TxnUtils.getTxnStore(conf);
-
-    // Get our own connection to the database so we can get table and partition information.
-    rs = RawStoreProxy.getProxy(conf, conf,
-        MetastoreConf.getVar(conf, MetastoreConf.ConfVars.RAW_STORE_IMPL), threadId);
   }
 
   @Override Table resolveTable(CompactionInfo ci) throws MetaException {
     try {
-      return rs.getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
+      return getMSForConf(conf).getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
     } catch (MetaException e) {
       LOG.error("Unable to find table " + ci.getFullTableName() + ", " + e.getMessage());
       throw e;
@@ -76,7 +69,7 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
 
   @Override boolean replIsCompactionDisabledForDatabase(String dbName) throws TException {
     try {
-      Database database = rs.getDatabase(getDefaultCatalog(conf), dbName);
+      Database database = getMSForConf(conf).getDatabase(getDefaultCatalog(conf), dbName);
       // Compaction is disabled until after first successful incremental load. Check HIVE-21197 for more detail.
       boolean isReplCompactDisabled = ReplUtils.isFirstIncPending(database.getParameters());
       if (isReplCompactDisabled) {
@@ -91,7 +84,7 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
 
   @Override List<Partition> getPartitionsByNames(CompactionInfo ci) throws MetaException {
     try {
-      return rs.getPartitionsByNames(getDefaultCatalog(conf), ci.dbname, ci.tableName,
+      return getMSForConf(conf).getPartitionsByNames(getDefaultCatalog(conf), ci.dbname, ci.tableName,
           Collections.singletonList(ci.partName));
     } catch (MetaException e) {
       LOG.error("Unable to get partitions by name for CompactionInfo=" + ci);