You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2020/01/13 14:33:09 UTC
[hive] branch master updated: HIVE-22596: RawStore used by
Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 31c19a0 HIVE-22596: RawStore used by Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)
31c19a0 is described below
commit 31c19a08b97babd7dda0f9f7fef1b1e2b57bf5d7
Author: denys kuzmenko <dk...@cloudera.com>
AuthorDate: Mon Jan 13 15:32:36 2020 +0100
HIVE-22596: RawStore used by Initiator is not thread-safe (Denys Kuzmenko via Peter Vary)
---
.../org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java | 8 ++++----
.../hadoop/hive/ql/txn/compactor/CompactorThread.java | 2 +-
.../hive/ql/txn/compactor/MetaStoreCompactorThread.java | 15 ++++-----------
3 files changed, 9 insertions(+), 16 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
index 5dfa7ca..6f64290 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Cleaner.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.ValidWriteIdList;
@@ -55,6 +54,7 @@ import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import static org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler.getMSForConf;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
/**
@@ -224,7 +224,7 @@ public class Cleaner extends MetaStoreCompactorThread {
return " id=" + ci.id;
}
private void removeFiles(String location, ValidWriteIdList writeIdList, CompactionInfo ci)
- throws IOException, NoSuchObjectException {
+ throws IOException, NoSuchObjectException, MetaException {
Path locPath = new Path(location);
AcidUtils.Directory dir = AcidUtils.getAcidState(locPath.getFileSystem(conf), locPath, conf, writeIdList, Ref.from(
false), false, null, false);
@@ -258,8 +258,8 @@ public class Cleaner extends MetaStoreCompactorThread {
}
FileSystem fs = filesToDelete.get(0).getFileSystem(conf);
- Database db = rs.getDatabase(getDefaultCatalog(conf), ci.dbname);
- Boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db);
+ Database db = getMSForConf(conf).getDatabase(getDefaultCatalog(conf), ci.dbname);
+ boolean isSourceOfRepl = ReplChangeManager.isSourceOfReplication(db);
for (Path dead : filesToDelete) {
LOG.debug("Going to delete path " + dead.toString());
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
index 99da86f..b378d40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorThread.java
@@ -53,7 +53,7 @@ public abstract class CompactorThread extends Thread implements Configurable {
protected static final Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
protected HiveConf conf;
- protected RawStore rs;
+
protected AtomicBoolean stop;
protected AtomicBoolean looped;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
index 8bfb524..aa258b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/MetaStoreCompactorThread.java
@@ -18,14 +18,11 @@
package org.apache.hadoop.hive.ql.txn.compactor;
import org.apache.hadoop.hive.metastore.MetaStoreThread;
-import org.apache.hadoop.hive.metastore.RawStore;
-import org.apache.hadoop.hive.metastore.RawStoreProxy;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.Database;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.txn.CompactionInfo;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
@@ -36,6 +33,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import static org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler.getMSForConf;
import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog;
/**
@@ -45,7 +43,6 @@ import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCa
public class MetaStoreCompactorThread extends CompactorThread implements MetaStoreThread {
protected TxnStore txnHandler;
- protected RawStore rs;
protected int threadId;
@Override
@@ -59,15 +56,11 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
// Get our own instance of the transaction handler
txnHandler = TxnUtils.getTxnStore(conf);
-
- // Get our own connection to the database so we can get table and partition information.
- rs = RawStoreProxy.getProxy(conf, conf,
- MetastoreConf.getVar(conf, MetastoreConf.ConfVars.RAW_STORE_IMPL), threadId);
}
@Override Table resolveTable(CompactionInfo ci) throws MetaException {
try {
- return rs.getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
+ return getMSForConf(conf).getTable(getDefaultCatalog(conf), ci.dbname, ci.tableName);
} catch (MetaException e) {
LOG.error("Unable to find table " + ci.getFullTableName() + ", " + e.getMessage());
throw e;
@@ -76,7 +69,7 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
@Override boolean replIsCompactionDisabledForDatabase(String dbName) throws TException {
try {
- Database database = rs.getDatabase(getDefaultCatalog(conf), dbName);
+ Database database = getMSForConf(conf).getDatabase(getDefaultCatalog(conf), dbName);
// Compaction is disabled until after first successful incremental load. Check HIVE-21197 for more detail.
boolean isReplCompactDisabled = ReplUtils.isFirstIncPending(database.getParameters());
if (isReplCompactDisabled) {
@@ -91,7 +84,7 @@ public class MetaStoreCompactorThread extends CompactorThread implements MetaSto
@Override List<Partition> getPartitionsByNames(CompactionInfo ci) throws MetaException {
try {
- return rs.getPartitionsByNames(getDefaultCatalog(conf), ci.dbname, ci.tableName,
+ return getMSForConf(conf).getPartitionsByNames(getDefaultCatalog(conf), ci.dbname, ci.tableName,
Collections.singletonList(ci.partName));
} catch (MetaException e) {
LOG.error("Unable to get partitions by name for CompactionInfo=" + ci);