You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2017/10/10 00:52:03 UTC
[29/61] [abbrv] hive git commit: HIVE-17561 Move TxnStore and
implementations to standalone metastore (Alan Gates,
reviewed by Eugene Koifman)
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
index 14ff58e..e46e65b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
@@ -87,7 +87,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -102,7 +102,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -121,7 +121,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(3,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -140,7 +140,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(4,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -155,7 +155,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -171,7 +171,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -307,7 +307,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(4,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -322,7 +322,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -337,7 +337,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -352,7 +352,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.rollbackTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -367,7 +367,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.rollbackTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -382,7 +382,7 @@ public class TestDbTxnManager {
List<HiveLock> locks = ctx.getHiveLocks();
Assert.assertEquals(1, locks.size());
Assert.assertEquals(1,
- TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+ TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
txnMgr.commitTxn();
locks = txnMgr.getLockManager().getLocks(false, false);
Assert.assertEquals(0, locks.size());
@@ -481,7 +481,7 @@ public class TestDbTxnManager {
@Before
public void setUp() throws Exception {
- TxnDbUtil.prepDb();
+ TxnDbUtil.prepDb(conf);
txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
txnMgr.getLockManager();//init lock manager
Assert.assertTrue(txnMgr instanceof DbTxnManager);
@@ -497,7 +497,7 @@ public class TestDbTxnManager {
public void tearDown() throws Exception {
if(houseKeeperService != null) houseKeeperService.stop();
if (txnMgr != null) txnMgr.closeTxnManager();
- TxnDbUtil.cleanDb();
+ TxnDbUtil.cleanDb(conf);
}
private static class MockQueryPlan extends QueryPlan {
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 6d1cdcb..e9833cb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -104,8 +104,8 @@ public class TestDbTxnManager2 {
ctx = new Context(conf);
driver = new Driver(conf);
driver.init();
- TxnDbUtil.cleanDb();
- TxnDbUtil.prepDb();
+ TxnDbUtil.cleanDb(conf);
+ TxnDbUtil.prepDb(conf);
SessionState ss = SessionState.get();
ss.initTxnMgr(conf);
txnMgr = ss.getTxnMgr();
@@ -445,9 +445,9 @@ public class TestDbTxnManager2 {
checkCmdOnDriver(cpr);
cpr = driver.run("insert into temp.T13p partition (ds='tomorrow', hour='2') values (8, 8)");
checkCmdOnDriver(cpr);
- int count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')");
+ int count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')");
Assert.assertEquals(4, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')");
Assert.assertEquals(4, count);
// Fail some inserts, so that we have records in TXN_COMPONENTS
@@ -460,132 +460,132 @@ public class TestDbTxnManager2 {
checkCmdOnDriver(cpr);
cpr = driver.run("insert into temp.T13p partition (ds='today', hour='1') values (12, 12)");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(4, count);
conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
// Drop a table/partition; corresponding records in TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS should disappear
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
Assert.assertEquals(1, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
Assert.assertEquals(2, count);
cpr = driver.run("drop table temp.T10");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
Assert.assertEquals(1, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
Assert.assertEquals(1, count);
cpr = driver.run("alter table temp.T12p drop partition (ds='today', hour='1')");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
Assert.assertEquals(0, count);
// Successfully perform compaction on a table/partition, so that we have successful records in COMPLETED_COMPACTIONS
cpr = driver.run("alter table temp.T11 compact 'minor'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'");
Assert.assertEquals(1, count);
cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'minor'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'");
Assert.assertEquals(1, count);
// Fail compaction, so that we have failed records in COMPLETED_COMPACTIONS
conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, true);
cpr = driver.run("alter table temp.T11 compact 'major'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'");
Assert.assertEquals(1, count);
cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(1, count);
org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'");
Assert.assertEquals(1, count);
conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false);
// Put 2 records into COMPACTION_QUEUE and do nothing
cpr = driver.run("alter table temp.T11 compact 'major'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(1, count);
cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(1, count);
// Drop a table/partition, corresponding records in COMPACTION_QUEUE and COMPLETED_COMPACTIONS should disappear
cpr = driver.run("drop table temp.T11");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'");
Assert.assertEquals(0, count);
cpr = driver.run("alter table temp.T12p drop partition (ds='tomorrow', hour='2')");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'");
Assert.assertEquals(0, count);
// Put 1 record into COMPACTION_QUEUE and do nothing
cpr = driver.run("alter table temp.T13p partition (ds='today', hour='1') compact 'major'");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'");
Assert.assertEquals(1, count);
// Drop database, everything in all 4 meta tables should disappear
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(1, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(2, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(1, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(0, count);
cpr = driver.run("drop database if exists temp cascade");
checkCmdOnDriver(cpr);
- count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(0, count);
- count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
+ count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
Assert.assertEquals(0, count);
}
@@ -959,7 +959,7 @@ public class TestDbTxnManager2 {
@Test
public void testWriteSetTracking4() throws Exception {
dropTable(new String[] {"TAB_PART", "TAB2"});
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " +
"partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
@@ -986,14 +986,14 @@ public class TestDbTxnManager2 {
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
//update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(),
"default", "tab2", Collections.EMPTY_LIST);
adp.setOperationType(DataOperationType.UPDATE);
txnHandler.addDynamicPartitions(adp);
txnMgr2.commitTxn();
//Short Running updated nothing, so we expect 0 rows in WRITE_SET
- Assert.assertEquals( 0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals( 0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
txnMgr2.openTxn(ctx, "T3");
checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'"));//pretend this partition exists
@@ -1003,19 +1003,19 @@ public class TestDbTxnManager2 {
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);//since TAB2 is empty
//update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(),
"default", "tab2", Collections.singletonList("p=two"));
adp.setOperationType(DataOperationType.UPDATE);
txnHandler.addDynamicPartitions(adp);//simulate partition update
txnMgr2.commitTxn();
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
AcidWriteSetService houseKeeper = new AcidWriteSetService();
TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
//since T3 overlaps with Long Running (still open) GC does nothing
- Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));//no rows match
txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
//so generate empty Dyn Part call
@@ -1028,7 +1028,7 @@ public class TestDbTxnManager2 {
locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 0, locks.size());
TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
}
/**
* overlapping txns updating the same resource but 1st one rolls back; 2nd commits
@@ -1037,7 +1037,7 @@ public class TestDbTxnManager2 {
@Test
public void testWriteSetTracking5() throws Exception {
dropTable(new String[] {"TAB_PART"});
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " +
"partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
@@ -1063,9 +1063,9 @@ public class TestDbTxnManager2 {
Arrays.asList("p=blah"));
adp.setOperationType(DataOperationType.UPDATE);
txnHandler.addDynamicPartitions(adp);
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
txnMgr2.commitTxn();//since conflicting txn rolled back, commit succeeds
- Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
}
/**
* check that read query concurrent with txn works ok
@@ -1073,7 +1073,7 @@ public class TestDbTxnManager2 {
@Test
public void testWriteSetTracking6() throws Exception {
dropTable(new String[] {"TAB2"});
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
CommandProcessorResponse cpr = driver.run("create table if not exists TAB2(a int, b int) clustered " +
"by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
@@ -1086,19 +1086,19 @@ public class TestDbTxnManager2 {
swapTxnManager(txnMgr2);
checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 101"));
txnMgr2.acquireLocks(driver.getPlan(), ctx, "Horton");
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 2, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks);
checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
txnMgr2.commitTxn();//no conflict
- Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 1, locks.size());
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks);
txnMgr.commitTxn();
TestTxnCommands2.runHouseKeeperService(new AcidWriteSetService(), conf);
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
}
/**
@@ -1108,7 +1108,7 @@ public class TestDbTxnManager2 {
@Test
public void testWriteSetTracking7() throws Exception {
dropTable(new String[] {"tab2", "TAB2"});
- Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+ Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
CommandProcessorResponse cpr = driver.run("create table if not exists tab2 (a int, b int) " +
"partitioned by (p string) clustered by (a) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='true')");
checkCmdOnDriver(cpr);
@@ -1150,13 +1150,13 @@ public class TestDbTxnManager2 {
txnMgr.commitTxn();//txnid:idTxnUpdate2
//now both txns concurrently updated TAB2 but different partitions.
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'"));
//2 from txnid:1, 1 from txnid:2, 1 from txnid:3
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null"));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null"));
//================
//test with predicates such that partition pruning doesn't kick in
@@ -1204,13 +1204,13 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
txnMgr.commitTxn();//txnid:idTxnUpdate4
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
//2 from insert + 1 for each update stmt
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}
/**
* Concurrent updates with partition pruning predicate and w/o one
@@ -1261,12 +1261,12 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
txnMgr.commitTxn();//txnid:idTxnUpdate2
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}
/**
* Concurrent update/delete of different partitions - should pass
@@ -1317,18 +1317,18 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
txnMgr.commitTxn();//txnid:idTxnUpdate2
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + " and ctc_table='tab1'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + " and ctc_table='tab1' and ctc_partition='p=one'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'"));
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + " and ctc_table='tab1'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + " and ctc_table='tab1' and ctc_partition='p=one'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'"));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}
/**
* Concurrent update/delete of same partition - should fail to commit
@@ -1387,10 +1387,10 @@ public class TestDbTxnManager2 {
"Aborting [txnid:5,5] due to a write conflict on default/tab1/p=two committed by [txnid:4,5] d/u",
exception.getCause().getMessage());
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 3, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 3, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}
/**
* Concurrent delte/detele of same partition - should pass
@@ -1448,16 +1448,16 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
txnMgr.commitTxn();//"select * from tab1" txn
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
- Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
- Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
+ Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
+ Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
}
@Test
public void testCompletedTxnComponents() throws Exception {
@@ -1470,11 +1470,11 @@ public class TestDbTxnManager2 {
checkCmdOnDriver(driver.run("insert into tab_not_acid2 values(1,1),(2,2)"));
//writing both acid and non-acid resources in the same txn
checkCmdOnDriver(driver.run("from tab_not_acid2 insert into tab1 partition(p='two')(a,b) select a,b insert into tab_not_acid2(a,b) select a,b "));//txnid:1
- Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS"));
+ Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS"));
//only expect transactional components to be in COMPLETED_TXN_COMPONENTS
- Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'"));
+ Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'"));
}
/**
@@ -1495,13 +1495,13 @@ public class TestDbTxnManager2 {
//writing both acid and non-acid resources in the same txn
//tab1 write is a dynamic partition insert
checkCmdOnDriver(driver.run("from tab_not_acid insert into tab1 partition(p)(a,b,p) select a,b,p insert into tab_not_acid(a,b) select a,b where p='two'"));//txnid:9
- Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS"));
+ Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS"));
//only expect transactional components to be in COMPLETED_TXN_COMPONENTS
- Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9"));
- Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
- 2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'"));
+ Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9"));
+ Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+ 2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'"));
}
//todo: Concurrent insert/update of same partition - should pass
@@ -1611,9 +1611,9 @@ public class TestDbTxnManager2 {
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
0,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
//complete 1st txn
AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target",
Collections.singletonList("p=1/q=3"));//update clause
@@ -1629,39 +1629,39 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
" and tc_operation_type='u'"));
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
" and tc_operation_type='d'"));
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
3,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
" and tc_operation_type='i'"));
txnMgr.commitTxn();//commit T1
Assert.assertEquals(
"COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
6,
- TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1));
Assert.assertEquals(
"WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
" and ws_operation_type='u'"));
Assert.assertEquals(
"WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
" and ws_operation_type='d'"));
//re-check locks which were in Waiting state - should now be Acquired
@@ -1676,9 +1676,9 @@ public class TestDbTxnManager2 {
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
0,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2));
//complete 2nd txn
adp = new AddDynamicPartitions(txnId2, "default", "target",
Collections.singletonList(cc ? "p=1/q=3" : "p=1/p=2"));//update clause
@@ -1694,21 +1694,21 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
" and tc_operation_type='u'"));
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
" and tc_operation_type='d'"));
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
3,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
" and tc_operation_type='i'"));
LockException expectedException = null;
@@ -1725,33 +1725,33 @@ public class TestDbTxnManager2 {
"committed by [txnid:10,11] u/u", expectedException.getMessage());
Assert.assertEquals(
"COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
0,
- TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
Assert.assertEquals(
"WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
0,
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2));
}
else {
Assert.assertNull("Unexpected exception " + expectedException, expectedException);
Assert.assertEquals(
"COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
6,
- TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
Assert.assertEquals(
"WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
" and ws_operation_type='u'"));
Assert.assertEquals(
"WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
" and ws_operation_type='d'"));
}
@@ -1790,9 +1790,9 @@ public class TestDbTxnManager2 {
txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,//no DP, so it's populated from lock info
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
List<ShowLocksResponseElement> locks = getLocks(txnMgr);
Assert.assertEquals("Unexpected lock count", 1, locks.size());
@@ -1820,9 +1820,9 @@ public class TestDbTxnManager2 {
txnMgr.commitTxn();//commit T1
Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
causeConflict ? 1 : 0,//Inserts are not tracked by WRITE_SET
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid1 +
" and ws_operation_type=" + (causeConflict ? "'u'" : "'i'")));
@@ -1835,14 +1835,14 @@ public class TestDbTxnManager2 {
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,//
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,//
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 +
"and tc_operation_type='d'"));
//complete T2 txn
@@ -1862,9 +1862,9 @@ public class TestDbTxnManager2 {
}
else {
Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
1,//Unpartitioned table: 1 row for Delete; Inserts are not tracked in WRITE_SET
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
" and ws_operation_type='d'"));
}
}
@@ -1887,15 +1887,15 @@ public class TestDbTxnManager2 {
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks);
Assert.assertEquals(
"HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
- TxnDbUtil.queryToString("select * from HIVE_LOCKS"),
+ TxnDbUtil.queryToString(conf, "select * from HIVE_LOCKS"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1));
txnMgr.rollbackTxn();
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
0,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
//now actually write to table to generate some partitions
checkCmdOnDriver(driver.run("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)"));
driver.run("select count(*) from target");
@@ -1904,10 +1904,10 @@ public class TestDbTxnManager2 {
Assert.assertEquals("", "4", r.get(0));
Assert.assertEquals(//look in COMPLETED_TXN_COMPONENTS because driver.run() committed!!!!
"COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1 + 1) + "): " +
- TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
2,//2 distinct partitions created
//txnid+1 because we want txn used by previous driver.run("insert....)
- TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1)));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1)));
long txnid2 = txnMgr.openTxn(ctx, "T1");
@@ -1922,9 +1922,9 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
2,//2 distinct partitions modified
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
txnMgr.commitTxn();
}
@Test
@@ -1986,9 +1986,9 @@ public class TestDbTxnManager2 {
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
0,//because it's using a DP write
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
//complete T1 transaction (simulate writing to 2 partitions)
AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target",
Arrays.asList("p=1/q=2","p=1/q=3"));
@@ -1996,15 +1996,15 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
" and tc_operation_type='u'"));
txnMgr.commitTxn();//commit T1
Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
2,//2 partitions updated
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
" and ws_operation_type='u'"));
@@ -2021,9 +2021,9 @@ public class TestDbTxnManager2 {
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
0,//because it's using a DP write
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
//complete T2 txn
//simulate Insert into 2 partitions
adp = new AddDynamicPartitions(txnid2, "default", "target",
@@ -2032,9 +2032,9 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
2,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'"));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'"));
//simulate Update of 1 partitions; depending on causeConflict, choose one of the partitions
//which was modified by the T1 update stmt or choose a non-conflicting one
adp = new AddDynamicPartitions(txnid2, "default", "target",
@@ -2043,9 +2043,9 @@ public class TestDbTxnManager2 {
txnHandler.addDynamicPartitions(adp);
Assert.assertEquals(
"TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+ TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
1,
- TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'"));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'"));
LockException expectedException = null;
@@ -2064,14 +2064,14 @@ public class TestDbTxnManager2 {
}
else {
Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
1,//1 partitions updated
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
" and ws_operation_type='u'"));
Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
- TxnDbUtil.queryToString("select * from WRITE_SET"),
+ TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
1,//1 partitions updated (and no other entries)
- TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2));
+ TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index af8a63d..2784009 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -95,7 +95,7 @@ public abstract class CompactorTest {
protected CompactorTest() throws Exception {
conf = new HiveConf();
TxnDbUtil.setConfValues(conf);
- TxnDbUtil.cleanDb();
+ TxnDbUtil.cleanDb(conf);
ms = new HiveMetaStoreClient(conf);
txnHandler = TxnUtils.getTxnStore(conf);
tmpdir = new File (Files.createTempDirectory("compactor_test_table_").toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index d91b22d..acc50ca 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -33,6 +33,11 @@
<dependencies>
<dependency>
+ <groupId>com.jolbox</groupId>
+ <artifactId>bonecp</artifactId>
+ <version>${bonecp.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.github.joshelser</groupId>
<artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
<version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version>
@@ -48,6 +53,16 @@
<version>${protobuf.version}</version>
</dependency>
<dependency>
+ <groupId>com.zaxxer</groupId>
+ <artifactId>HikariCP</artifactId>
+ <version>${hikaricp.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-dbcp</groupId>
+ <artifactId>commons-dbcp</artifactId>
+ <version>${commons-dbcp.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${dropwizard.version}</version>
@@ -68,6 +83,11 @@
<version>${commons-lang3.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.derby</groupId>
+ <artifactId>derby</artifactId>
+ <version>${derby.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
new file mode 100644
index 0000000..9fa5cab
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * Combination of Runnable and Configurable
+ */
+public interface RunnableConfigurable extends Configurable, Runnable {
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index d3c714b..5a6ef99 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -54,11 +55,8 @@ public class MetastoreConf {
private static URL hiveSiteURL = null;
private static URL hiveMetastoreSiteURL = null;
private static URL metastoreSiteURL = null;
+ private static AtomicBoolean beenDumped = new AtomicBoolean();
- /*
- private static Map<String, String> metaToHiveKeys;
- private static Map<String, String> hiveToMetaKeys;
- */
private static Map<String, ConfVars> keyToVars;
@VisibleForTesting
@@ -406,6 +404,9 @@ public class MetastoreConf {
"not blocked.\n" +
"\n" +
"See HIVE-4409 for more details."),
+ DUMP_CONFIG_ON_CREATION("metastore.dump.config.on.creation", NO_SUCH_KEY, true,
+ "If true, a printout of the config file (minus sensitive values) will be dumped to the " +
+ "log whenever newMetastoreConf() is called. Can produce a lot of logs"),
END_FUNCTION_LISTENERS("metastore.end.function.listeners",
"hive.metastore.end.function.listeners", "",
"List of comma separated listeners for the end of metastore functions."),
@@ -932,6 +933,11 @@ public class MetastoreConf {
if (getBoolVar(conf, ConfVars.SCHEMA_VERIFICATION)) {
setBoolVar(conf, ConfVars.AUTO_CREATE_ALL, false);
}
+
+ if (!beenDumped.getAndSet(true) && getBoolVar(conf, ConfVars.DUMP_CONFIG_ON_CREATION) &&
+ LOG.isInfoEnabled()) {
+ LOG.info(dumpConfig(conf));
+ }
return conf;
}
@@ -1304,8 +1310,8 @@ public class MetastoreConf {
}
/**
- * Return the configuration value as a String. This does not work on time based values as it
- * doesn't have a time unit.
+ * Return the configuration value as a String. For time based values it will be returned in
+ * the default time unit appended with an appropriate abbreviation (eg s for seconds, ...)
* @param conf configuration to read
* @param var variable to read
* @return value as an object
@@ -1319,8 +1325,12 @@ public class MetastoreConf {
return Long.toString(getLongVar(conf, var));
} else if (var.defaultVal.getClass() == Double.class) {
return Double.toString(getDoubleVar(conf, var));
+ } else if (var.defaultVal.getClass() == TimeValue.class) {
+ TimeUnit timeUnit = (var.defaultVal.getClass() == TimeValue.class) ?
+ ((TimeValue)var.defaultVal).unit : null;
+ return Long.toString(getTimeVar(conf, var, timeUnit)) + timeAbbreviationFor(timeUnit);
} else {
- throw new RuntimeException("Invalid type for getObject " + var.defaultVal.getClass().getName());
+ throw new RuntimeException("Unknown type for getObject " + var.defaultVal.getClass().getName());
}
}
@@ -1356,6 +1366,45 @@ public class MetastoreConf {
}
/**
+ * Dump the configuration file to the log. It will be dumped at an INFO level. This can
+ * potentially produce a lot of logs, so you might want to be careful when and where you do it.
+ * It takes care not to dump hidden keys.
+ * @param conf Configuration file to dump
+ * @return String containing dumped config file.
+ */
+ public static String dumpConfig(Configuration conf) {
+ StringBuilder buf = new StringBuilder("MetastoreConf object:\n");
+ if (hiveSiteURL != null) {
+ buf.append("Used hive-site file: ")
+ .append(hiveSiteURL)
+ .append('\n');
+ }
+ if (hiveMetastoreSiteURL != null) {
+ buf.append("Used hivemetastore-site file: ")
+ .append(hiveMetastoreSiteURL)
+ .append('\n');
+ }
+ if (metastoreSiteURL != null) {
+ buf.append("Used metastore-site file: ")
+ .append(metastoreSiteURL)
+ .append('\n');
+ }
+ for (ConfVars var : ConfVars.values()) {
+ if (!unprintables.contains(var.varname)) {
+ buf.append("Key: <")
+ .append(var.varname)
+ .append("> old hive key: <")
+ .append(var.hiveName)
+ .append("> value: <")
+ .append(getAsString(conf, var))
+ .append(">\n");
+ }
+ }
+ buf.append("Finished MetastoreConf object.\n");
+ return buf.toString();
+ }
+
+ /**
* validate value for a ConfVar, return non-null string for fail message
*/
public interface Validator {
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
new file mode 100644
index 0000000..6a2f770
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.jolbox.bonecp.BoneCPConfig;
+import com.jolbox.bonecp.BoneCPDataSource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * DataSourceProvider for the BoneCP connection pool.
+ */
+public class BoneCPDataSourceProvider implements DataSourceProvider {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class);
+
+ public static final String BONECP = "bonecp";
+ private static final String CONNECTION_TIMEOUT_PROPERTY= "bonecp.connectionTimeoutInMs";
+ private static final String PARTITION_COUNT_PROPERTY= "bonecp.partitionCount";
+
+ @Override
+ public DataSource create(Configuration hdpConfig) throws SQLException {
+
+ LOG.debug("Creating BoneCP connection pool for the MetaStore");
+
+ String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
+ String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
+ String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
+ int maxPoolSize = MetastoreConf.getIntVar(hdpConfig,
+ MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
+
+ Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP);
+ long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
+ String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1");
+
+ BoneCPConfig config = null;
+ try {
+ config = new BoneCPConfig(properties);
+ } catch (Exception e) {
+ throw new SQLException("Cannot create BoneCP configuration: ", e);
+ }
+ config.setJdbcUrl(driverUrl);
+ //if we are waiting for connection for a long time, something is really wrong
+ //better raise an error than hang forever
+ //see DefaultConnectionStrategy.getConnectionInternal()
+ config.setConnectionTimeoutInMs(connectionTimeout);
+ config.setMaxConnectionsPerPartition(maxPoolSize);
+ config.setPartitionCount(Integer.parseInt(partitionCount));
+ config.setUser(user);
+ config.setPassword(passwd);
+ return new BoneCPDataSource(config);
+ }
+
+ @Override
+ public boolean mayReturnClosedConnection() {
+ // See HIVE-11915 for details
+ return true;
+ }
+
+ @Override
+ public boolean supports(Configuration configuration) {
+ String poolingType =
+ MetastoreConf.getVar(configuration,
+ MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE).toLowerCase();
+ if (BONECP.equals(poolingType)) {
+ int boneCpPropsNr = DataSourceProvider.getPrefixedProperties(configuration, BONECP).size();
+ LOG.debug("Found " + boneCpPropsNr + " nr. of bonecp specific configurations");
+ return boneCpPropsNr > 0;
+ }
+ LOG.debug("Configuration requested " + poolingType + " pooling, BoneCpDSProvider exiting");
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
new file mode 100644
index 0000000..17ff8d1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+
+import javax.sql.DataSource;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+public interface DataSourceProvider {
+
+ /**
+ * @param hdpConfig
+ * @return the new connection pool
+ */
+ DataSource create(Configuration hdpConfig) throws SQLException;
+
+ /**
+ * BoneCp has a bug which causes closed connections to be returned to the pool
+ * under certain conditions. (HIVE-11915)
+ * @return true if the factory creates BoneCp pools which need "special attention"
+ */
+ boolean mayReturnClosedConnection();
+
+ /**
+ * @param configuration Hadoop configuration object
+ * @return factory able to create a connection pool for the implementation
+ * specified in the configuration
+ */
+ boolean supports(Configuration configuration);
+
+ /**
+ * @param hdpConfig
+ * @return subset of properties prefixed by a connection pool specific substring
+ */
+ static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) {
+ Properties dataSourceProps = new Properties();
+ Iterables.filter(
+ hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix)))
+ .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue()));
+ return dataSourceProps;
+ }
+
+ static String getMetastoreJdbcUser(Configuration conf) {
+ return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME);
+ }
+
+ static String getMetastoreJdbcPasswd(Configuration conf) throws SQLException {
+ try {
+ return MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
+ } catch (IOException err) {
+ throw new SQLException("Error getting metastore password", err);
+ }
+ }
+
+ static String getMetastoreJdbcDriverUrl(Configuration conf) throws SQLException {
+ return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTURLKEY);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
new file mode 100644
index 0000000..e3c18e3
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Create a DataSourceProvider for a connectionPool configured in a hadoop
+ * Configuration object.
+ */
+public abstract class DataSourceProviderFactory {
+
+ private static final ImmutableList<DataSourceProvider> FACTORIES =
+ ImmutableList.<DataSourceProvider>builder().add(new HikariCPDataSourceProvider(), new BoneCPDataSourceProvider()).build();
+
+ /**
+ * @param hdpConfig hadoop configuration
+ * @return factory for the configured datanucleus.connectionPoolingType
+ */
+ public static DataSourceProvider getDataSourceProvider(Configuration hdpConfig) {
+
+ for (DataSourceProvider factory : FACTORIES) {
+
+ if (factory.supports(hdpConfig)) {
+ return factory;
+ }
+ }
+ return null;
+ }
+
+}