You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2020/07/08 17:38:36 UTC
[hive] branch master updated: HIVE-23780: Fail dropTable if acid
cleanup fails (Mustafa Iman reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 51221c8 HIVE-23780: Fail dropTable if acid cleanup fails (Mustafa Iman reviewed by Peter Vary)
51221c8 is described below
commit 51221c8100d4d86219c4eecd6d69dd4a1e9cd6f6
Author: Mustafa İman <mu...@gmail.com>
AuthorDate: Wed Jul 8 10:38:24 2020 -0700
HIVE-23780: Fail dropTable if acid cleanup fails (Mustafa Iman reviewed by Peter Vary)
Closes (#1192)
---
.../java/org/apache/hive/minikdc/MiniHiveKdc.java | 2 +-
.../hadoop/hive/metastore/TestAcidTableSetup.java | 2 +
.../cache/TestCachedStoreUpdateUsingEvents.java | 3 +
.../plugin/TestHiveAuthorizerCheckInvocation.java | 3 +
.../java/org/apache/hive/jdbc/TestJdbcDriver2.java | 6 +-
.../java/org/apache/hive/jdbc/miniHS2/MiniHS2.java | 18 ++-
.../metastore/txn/TestCompactionTxnHandler.java | 1 +
.../hadoop/hive/metastore/txn/TestTxnHandler.java | 1 +
.../txn/TestTxnHandlerNoConnectionPool.java | 2 +
.../ql/parse/TestUpdateDeleteSemanticAnalyzer.java | 4 +-
.../hive/ql/txn/compactor/CompactorTest.java | 1 +
.../hadoop/hive/metastore/AcidEventListener.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 2 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 17 ---
.../hadoop/hive/metastore/MetaStoreTestUtils.java | 20 ++-
.../metastore/TestCatalogNonDefaultClient.java | 2 +-
.../metastore/TestHiveMetaStoreSchemaMethods.java | 2 +-
.../hive/metastore/TestMetaStoreAcidCleanup.java | 138 +++++++++++++++++++++
.../metastore/minihms/RemoteMetaStoreForTests.java | 2 +-
.../hive/metastore/txn/ThrowingTxnHandler.java | 41 ++++++
20 files changed, 239 insertions(+), 30 deletions(-)
diff --git a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
index 5d528a5..0011e75 100644
--- a/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
+++ b/itests/hive-minikdc/src/test/java/org/apache/hive/minikdc/MiniHiveKdc.java
@@ -238,7 +238,7 @@ public class MiniHiveKdc {
String hiveMetastoreKeytab = miniHiveKdc.getKeyTabFile(
miniHiveKdc.getServicePrincipalForUser(MiniHiveKdc.HIVE_METASTORE_SERVICE_PRINCIPAL));
- return new MiniHS2.Builder().withConf(hiveConf)
+ return new MiniHS2.Builder().withTransactionalTables(false).withConf(hiveConf)
.withSecureRemoteMetastore(hiveMetastorePrincipal, hiveMetastoreKeytab).
withMiniKdc(hivePrincipal, hiveKeytab).withAuthenticationType(authenticationType)
.build();
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java
index 505b3c0..dfc3de3 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestAcidTableSetup.java
@@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +58,7 @@ public class TestAcidTableSetup {
MetastoreConf.setClass(conf, ConfVars.EXPRESSION_PROXY_CLASS,
DefaultPartitionExpressionProxy.class, PartitionExpressionProxy.class);
client = new HiveMetaStoreClient(conf);
+ TxnDbUtil.prepDb(conf);
}
@Test
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
index 2e9a4f7..8be7685 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStoreUpdateUsingEvents.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.metastore.utils.FileUtils;
import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat;
import org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat;
@@ -69,6 +70,8 @@ public class TestCachedStoreUpdateUsingEvents {
MetastoreConf.setVar(conf, ConfVars.REPLCMDIR, "cmroot");
MetaStoreTestUtils.setConfForStandloneMode(conf);
+ TxnDbUtil.prepDb(conf);
+
hmsHandler = new HiveMetaStore.HMSHandler("testCachedStore", conf, true);
rawStore = new ObjectStore();
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index 0db6eb7..2c2d96c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.Driver;
import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
import org.apache.hadoop.hive.ql.security.HiveAuthenticationProvider;
@@ -101,6 +102,8 @@ public class TestHiveAuthorizerCheckInvocation {
conf.setBoolVar(ConfVars.HIVE_QUERY_RESULTS_CACHE_ENABLED, true);
conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
+ TxnDbUtil.prepDb(conf);
+
SessionState.start(conf);
driver = new Driver(conf);
runCmd("create table " + tableName
diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
index ba1f39c..b4a8bad 100644
--- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
+++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork;
import org.apache.hadoop.hive.ql.processors.DfsProcessor;
@@ -188,8 +189,11 @@ public class TestJdbcDriver2 {
@SuppressWarnings("deprecation")
@BeforeClass
- public static void setUpBeforeClass() throws SQLException, ClassNotFoundException {
+ public static void setUpBeforeClass() throws Exception {
conf = new HiveConf(TestJdbcDriver2.class);
+ HiveConf initConf = new HiveConf(conf);
+ TxnDbUtil.setConfValues(initConf);
+ TxnDbUtil.prepDb(initConf);
dataFileDir = conf.get("test.data.files").replace('\\', '/')
.replace("c:", "");
dataFilePath = new Path(dataFileDir, "kv1.txt");
diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
index 7f25c74..d37afed 100644
--- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
+++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hive.llap.LlapItUtils;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.shims.HadoopShims.MiniDFSShim;
import org.apache.hadoop.hive.shims.HadoopShims.MiniMrShim;
@@ -83,6 +84,7 @@ public class MiniHS2 extends AbstractHiveService {
private MiniClusterType miniClusterType = MiniClusterType.LOCALFS_ONLY;
private boolean usePortsFromConf = false;
private PamAuthenticator pamAuthenticator;
+ private boolean createTransactionalTables;
public enum MiniClusterType {
MR,
@@ -103,6 +105,7 @@ public class MiniHS2 extends AbstractHiveService {
private String authType = "KERBEROS";
private boolean isHA = false;
private boolean cleanupLocalDirOnStartup = true;
+ private boolean createTransactionalTables = true;
private boolean isMetastoreSecure;
private String metastoreServerPrincipal;
private String metastoreServerKeyTab;
@@ -132,6 +135,11 @@ public class MiniHS2 extends AbstractHiveService {
return this;
}
+ public Builder withTransactionalTables(boolean createTransactionalTables) {
+ this.createTransactionalTables = createTransactionalTables;
+ return this;
+ }
+
public Builder withRemoteMetastore() {
this.isMetastoreRemote = true;
return this;
@@ -194,7 +202,7 @@ public class MiniHS2 extends AbstractHiveService {
hiveConf.setVar(ConfVars.HIVE_SERVER2_TRANSPORT_MODE, HS2_BINARY_MODE);
}
return new MiniHS2(hiveConf, miniClusterType, useMiniKdc, serverPrincipal, serverKeytab,
- isMetastoreRemote, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup,
+ isMetastoreRemote, createTransactionalTables, usePortsFromConf, authType, isHA, cleanupLocalDirOnStartup,
isMetastoreSecure, metastoreServerPrincipal, metastoreServerKeyTab, dataNodes);
}
}
@@ -232,7 +240,7 @@ public class MiniHS2 extends AbstractHiveService {
}
private MiniHS2(HiveConf hiveConf, MiniClusterType miniClusterType, boolean useMiniKdc,
- String serverPrincipal, String serverKeytab, boolean isMetastoreRemote,
+ String serverPrincipal, String serverKeytab, boolean isMetastoreRemote, boolean createTransactionalTables,
boolean usePortsFromConf, String authType, boolean isHA, boolean cleanupLocalDirOnStartup,
boolean isMetastoreSecure, String metastoreServerPrincipal, String metastoreKeyTab,
int dataNodes) throws Exception {
@@ -258,6 +266,7 @@ public class MiniHS2 extends AbstractHiveService {
this.isMetastoreSecure = isMetastoreSecure;
this.cleanupLocalDirOnStartup = cleanupLocalDirOnStartup;
this.usePortsFromConf = usePortsFromConf;
+ this.createTransactionalTables = createTransactionalTables;
baseDir = getBaseDir();
localFS = FileSystem.getLocal(hiveConf);
FileSystem fs;
@@ -355,13 +364,14 @@ public class MiniHS2 extends AbstractHiveService {
public MiniHS2(HiveConf hiveConf, MiniClusterType clusterType, boolean usePortsFromConf)
throws Exception {
this(hiveConf, clusterType, false, null, null,
- false, usePortsFromConf, "KERBEROS", false, true,
+ false, true, usePortsFromConf, "KERBEROS", false, true,
false, null, null, DEFAULT_DATANODE_COUNT);
}
public void start(Map<String, String> confOverlay) throws Exception {
if (isMetastoreRemote) {
- MetaStoreTestUtils.startMetaStoreWithRetry(getHiveConf());
+ MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(), getHiveConf(),
+ false, false, false, false, createTransactionalTables);
setWareHouseDir(MetastoreConf.getVar(getHiveConf(), MetastoreConf.ConfVars.WAREHOUSE));
}
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index ec7c6ae..8d7c7d2 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -68,6 +68,7 @@ public class TestCompactionTxnHandler {
public TestCompactionTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
+ TxnDbUtil.prepDb(conf);
// Set config so that TxnUtils.buildQueryWithINClauseStrings() will
// produce multiple queries
conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 3a38b43..85cc26e 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -104,6 +104,7 @@ public class TestTxnHandler {
public TestTxnHandler() throws Exception {
TxnDbUtil.setConfValues(conf);
+ TxnDbUtil.prepDb(conf);
LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
Configuration conf = ctx.getConfiguration();
conf.getLoggerConfig(CLASS_NAME).setLevel(Level.DEBUG);
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
index ed2d485..6b19339 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
@@ -47,6 +47,8 @@ public class TestTxnHandlerNoConnectionPool {
@Before
public void setUp() throws Exception {
conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, "None");
+ TxnDbUtil.setConfValues(conf);
+ TxnDbUtil.prepDb(conf);
txnHandler = TxnUtils.getTxnStore(conf);
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
index 461cfdc..3204669 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestUpdateDeleteSemanticAnalyzer.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
import org.apache.hadoop.hive.ql.QueryState;
@@ -222,7 +223,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
}
@Before
- public void setup() {
+ public void setup() throws Exception {
queryState = new QueryState.Builder().build();
conf = queryState.getConf();
conf
@@ -231,6 +232,7 @@ public class TestUpdateDeleteSemanticAnalyzer {
conf.setVar(HiveConf.ConfVars.HIVEMAPREDMODE, "nonstrict");
conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager");
conf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, true);
+ TxnDbUtil.prepDb(conf);
}
public void cleanupTables() throws HiveException {
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index 84a23b2..54e9529 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -103,6 +103,7 @@ public abstract class CompactorTest {
conf = new HiveConf();
TxnDbUtil.setConfValues(conf);
TxnDbUtil.cleanDb(conf);
+ TxnDbUtil.prepDb(conf);
ms = new HiveMetaStoreClient(conf);
txnHandler = TxnUtils.getTxnStore(conf);
tmpdir = new File(Files.createTempDirectory("compactor_test_table_").toString());
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
index 5279247..2ecdb96 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/AcidEventListener.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.hive.metastore.txn.TxnUtils;
/**
* It handles cleanup of dropped partition/table/database in ACID related metastore tables
*/
-public class AcidEventListener extends MetaStoreEventListener {
+public class AcidEventListener extends TransactionalMetaStoreEventListener {
private TxnStore txnHandler;
private Configuration conf;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 9cd3af0..137b4ad 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -562,9 +562,9 @@ public class HiveMetaStore extends ThriftHiveMetastore {
listeners = MetaStoreServerUtils.getMetaStoreListeners(MetaStoreEventListener.class, conf,
MetastoreConf.getVar(conf, ConfVars.EVENT_LISTENERS));
listeners.add(new SessionPropertiesListener(conf));
- listeners.add(new AcidEventListener(conf));
transactionalListeners = MetaStoreServerUtils.getMetaStoreListeners(TransactionalMetaStoreEventListener.class,
conf, MetastoreConf.getVar(conf, ConfVars.TRANSACTIONAL_EVENT_LISTENERS));
+ transactionalListeners.add(new AcidEventListener(conf));
if (Metrics.getRegistry() != null) {
listeners.add(new HMSMetricsListener(conf));
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 922edfb..c980318 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -327,8 +327,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
public void setConf(Configuration conf){
this.conf = conf;
- checkQFileTestHack();
-
synchronized (TxnHandler.class) {
if (connPool == null) {
Connection dbConn = null;
@@ -4254,21 +4252,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
// we are checking to the desired action.
private static Map<LockType, Map<LockType, Map<LockState, LockAction>>> jumpTable;
- private void checkQFileTestHack(){
- boolean hackOn = MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) ||
- MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST);
- if (hackOn) {
- LOG.info("Hacking in canned values for transaction manager");
- // Set up the transaction/locking db in the derby metastore
- TxnDbUtil.setConfValues(conf);
- try {
- TxnDbUtil.prepDb(conf);
- } catch (Exception e) {
- throw new RuntimeException("Unable to set up transaction database for" + " testing: " + e.getMessage(), e);
- }
- }
- }
-
private int abortTxns(Connection dbConn, List<Long> txnids, boolean skipCount) throws SQLException, MetaException {
return abortTxns(dbConn, txnids, false, skipCount);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
index 915586e..99aae4d 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
import org.apache.hadoop.hive.metastore.events.EventCleanerTask;
import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.metastore.txn.TxnDbUtil;
import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils;
import org.apache.thrift.TException;
import org.slf4j.Logger;
@@ -132,6 +133,13 @@ public class MetaStoreTestUtils {
return MetaStoreTestUtils.startMetaStoreWithRetry(bridge, conf, false, false, withHouseKeepingThreads, false);
}
+ public static int startMetaStoreWithRetry(HadoopThriftAuthBridge bridge, Configuration conf, boolean keepJdbcUri,
+ boolean keepWarehousePath, boolean withHouseKeepingThreads,
+ boolean waitForHouseKeepers) throws Exception {
+ return startMetaStoreWithRetry(bridge, conf, keepJdbcUri, keepWarehousePath, withHouseKeepingThreads,
+ waitForHouseKeepers, true);
+ }
+
/**
* Starts a MetaStore instance with the given configuration and given bridge.
* Tries to find a free port, and use it. If failed tries another port so the tests will not
@@ -147,7 +155,8 @@ public class MetaStoreTestUtils {
* @throws Exception Timeout after 60 seconds
*/
public static int startMetaStoreWithRetry(HadoopThriftAuthBridge bridge, Configuration conf, boolean keepJdbcUri,
- boolean keepWarehousePath, boolean withHouseKeepingThreads, boolean waitForHouseKeepers) throws Exception {
+ boolean keepWarehousePath, boolean withHouseKeepingThreads, boolean waitForHouseKeepers,
+ boolean createTransactionalTables) throws Exception {
Exception metaStoreException = null;
String warehouseDir = MetastoreConf.getVar(conf, ConfVars.WAREHOUSE);
@@ -175,6 +184,14 @@ public class MetaStoreTestUtils {
MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + metaStorePort);
}
+ if (createTransactionalTables) {
+ // Some tests may have dummy txn manager. There is no harm in creating the transactional tables but
+ // we should not change the test config in case they purposefully do not use transactions
+ Configuration txnInitConf = new Configuration(conf);
+ TxnDbUtil.setConfValues(txnInitConf);
+ TxnDbUtil.prepDb(txnInitConf);
+ }
+
MetaStoreTestUtils.startMetaStore(metaStorePort, bridge, conf, withHouseKeepingThreads, waitForHouseKeepers);
// Creating warehouse dir, if not exists
@@ -189,6 +206,7 @@ public class MetaStoreTestUtils {
LOG.info("MetaStore Thrift Server started on port: {} with warehouse dir: {} with " +
"jdbcUrl: {}", metaStorePort, warehouseDir, jdbcUrl);
+
return metaStorePort;
} catch (ConnectException ce) {
metaStoreException = ce;
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestCatalogNonDefaultClient.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestCatalogNonDefaultClient.java
index 550b107..3994101 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestCatalogNonDefaultClient.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestCatalogNonDefaultClient.java
@@ -45,7 +45,7 @@ public class TestCatalogNonDefaultClient extends NonCatCallsWithCatalog {
Configuration svrConf = new Configuration(conf);
int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(),
- svrConf);
+ svrConf, false, false, false, false, false);
// Only set the default catalog on the client.
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS, "thrift://localhost:" + port);
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT, catName);
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java
index 3d48c5f..bce1c85 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreSchemaMethods.java
@@ -91,7 +91,7 @@ public class TestHiveMetaStoreSchemaMethods {
MetastoreConf.setClass(conf, ConfVars.PRE_EVENT_LISTENERS, SchemaPreEventListener.class,
MetaStorePreEventListener.class);
int port = MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(),
- conf);
+ conf, false, false, false, false, false);
MetastoreConf.setVar(conf, ConfVars.THRIFT_URIS, "thrift://localhost:" + port);
client = new HiveMetaStoreClient(conf);
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAcidCleanup.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAcidCleanup.java
new file mode 100644
index 0000000..9813d3b
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAcidCleanup.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreCheckinTest;
+import org.apache.hadoop.hive.metastore.annotation.MetastoreUnitTest;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.client.builder.TableBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.txn.ThrowingTxnHandler;
+import org.apache.hadoop.hive.metastore.txn.TxnStore;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.thrift.TException;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.fail;
+import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.TXN_STORE_IMPL;
+
+@Category(MetastoreUnitTest.class)
+public class TestMetaStoreAcidCleanup {
+ protected static HiveMetaStoreClient client;
+ protected static Configuration conf;
+ protected static Warehouse warehouse;
+ private static final String dbName = "db";
+ private String tableName;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setClass(conf, TXN_STORE_IMPL, ThrowingTxnHandler.class, TxnStore.class);
+ conf.set("hive.metastore.client.capabilities", "HIVEMANAGEDINSERTWRITE,HIVEMANAGESTATS,HIVECACHEINVALIDATE,CONNECTORWRITE");
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ warehouse = new Warehouse(conf);
+ client = new HiveMetaStoreClient(conf);
+ Database db = new DatabaseBuilder()
+ .setName(dbName)
+ .build(conf);
+ client.createDatabase(db);
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ try {
+ ThrowingTxnHandler.doThrow = false;
+ client.dropDatabase(dbName);
+ client.close();
+ } catch (Throwable e) {
+ System.err.println("Unable to close metastore");
+ System.err.println(StringUtils.stringifyException(e));
+ throw e;
+ }
+ }
+
+ @After
+ public void afterTest() throws TException {
+ client.dropTable(dbName, tableName);
+ }
+
+ @Test
+ public void testDropTableShouldRollback_whenAcidCleanupFails() throws Exception {
+ tableName = "tableDropTable";
+
+ Table table;
+ createTable(dbName, tableName);
+
+ ThrowingTxnHandler.doThrow = true;
+ try {
+ client.dropTable(dbName, tableName);
+ } catch (Exception ex) {
+ if (!ex.getMessage().contains("during transactional cleanup")) {
+ fail("dropTable failed with unexpected exception: " + ex);
+ }
+ } finally {
+ ThrowingTxnHandler.doThrow = false;
+ }
+
+ table = client.getTable(dbName, tableName);
+ assertNotNull(table);
+ }
+
+ @Test
+ public void testDropDatabaseShouldRollback_whenAcidCleanupFails() throws Exception {
+ tableName = "tableDropDatabase";
+
+ Table table;
+ createTable(dbName, tableName);
+
+ ThrowingTxnHandler.doThrow = true;
+ try {
+ client.dropDatabase(dbName, true, false, true);
+ } catch (Exception ex) {
+ if (!ex.getMessage().contains("during transactional cleanup")) {
+ fail("dropTable failed with unexpected exception: " + ex);
+ }
+ } finally {
+ ThrowingTxnHandler.doThrow = false;
+ }
+
+ table = client.getTable(dbName, tableName);
+ assertNotNull(table);
+ }
+
+ private void createTable(String dbName, String tableName) throws TException {
+ new TableBuilder()
+ .setDbName(dbName)
+ .setTableName(tableName)
+ .addCol("foo", "string")
+ .addCol("bar", "string")
+ .addTableParam(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "true")
+ .addTableParam(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES, "insert_only")
+ .create(client, conf);
+ }
+}
\ No newline at end of file
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java
index b32d761..c725532 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/minihms/RemoteMetaStoreForTests.java
@@ -37,7 +37,7 @@ public class RemoteMetaStoreForTests extends AbstractMetaStoreService {
public void start() throws Exception {
MetastoreConf.setBoolVar(getConfiguration(), MetastoreConf.ConfVars.EXECUTE_SET_UGI, false);
MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(),
- getConfiguration());
+ getConfiguration(), false, false, false, false, false);
super.start();
}
}
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/txn/ThrowingTxnHandler.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/txn/ThrowingTxnHandler.java
new file mode 100644
index 0000000..d514a86
--- /dev/null
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/txn/ThrowingTxnHandler.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.HiveObjectType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.Table;
+
+import java.util.Iterator;
+
+public class ThrowingTxnHandler extends CompactionTxnHandler {
+
+ public static volatile boolean doThrow;
+
+ @Override
+ public void cleanupRecords(HiveObjectType type, Database db, Table table,
+ Iterator<Partition> partitionIterator) throws MetaException {
+ if (doThrow) {
+ throw new RuntimeException("during transactional cleanup");
+ }
+ super.cleanupRecords(type, db, table, partitionIterator);
+ }
+}