You are viewing a plain text version of this content. The canonical link for it is here.
Posted to gitbox@hive.apache.org by GitBox <gi...@apache.org> on 2022/05/03 14:10:47 UTC
[GitHub] [hive] deniskuzZ commented on a diff in pull request #3122: HIVE-26076: Non blocking ADD PARTITION if not exists
deniskuzZ commented on code in PR #3122:
URL: https://github.com/apache/hive/pull/3122#discussion_r863821986
##########
ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java:
##########
@@ -4028,4 +4005,88 @@ public void testAddDropConstraintNonBlocking() throws Exception {
checkLock(LockType.EXCL_WRITE,
LockState.ACQUIRED, "default", "tab_acid", null, locks);
}
+
+ @Test
+ public void testAddPartitionIfNotExistsNonBlocking() throws Exception {
+ testAddPartitionIfNotExists(false);
+ }
+ @Test
+ public void testAddPartitionIfNotExistsBlocking() throws Exception {
+ testAddPartitionIfNotExists(true);
+ }
+
+ private void testAddPartitionIfNotExists(boolean blocking) throws Exception {
+ dropTable(new String[] {"T", "Tstage"});
+
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_LOCKS_PARTITION_THRESHOLD, 1);
+ driver = Mockito.spy(driver);
+
+ HiveConf.setBoolVar(driver2.getConf(), HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED, !blocking);
+ driver2 = Mockito.spy(driver2);
+
+ driver.run("create table if not exists T (a int, b int) partitioned by (p string) " +
+ "stored as orc TBLPROPERTIES ('transactional'='true')");
+ //bucketed just so that we get 2 files
+ driver.run("create table if not exists Tstage (a int, b int) clustered by (a) into 2 " +
+ "buckets stored as orc TBLPROPERTIES ('transactional'='false')");
+ driver.run("insert into Tstage values(1,2),(3,4)");
+ String exportLoc = exportFolder.newFolder("1").toString();
+ driver.run("export table Tstage to '" + exportLoc + "'");
+
+ driver.compileAndRespond("select * from T");
+ List<String> res = new ArrayList<>();
+
+ driver.lockAndRespond();
+ List<ShowLocksResponseElement> locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+
+ checkLock(LockType.SHARED_READ,
+ LockState.ACQUIRED, "default", "T", null, locks);
+
+ DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ swapTxnManager(txnMgr2);
+ driver2.compileAndRespond("alter table T add if not exists partition (p='foo') location '" + exportLoc + "/data'");
+
+ if (blocking) {
+ txnMgr2.acquireLocks(driver2.getPlan(), ctx, null, false);
+ locks = getLocks();
+
+ ShowLocksResponseElement checkLock = checkLock(LockType.EXCLUSIVE,
+ LockState.WAITING, "default", "T", null, locks);
+
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+
+ driver.getFetchTask().fetch(res);
+ swapTxnManager(txnMgr2);
+
+ FieldSetter.setField(txnMgr2, txnMgr2.getClass().getDeclaredField("numStatements"), 0);
+ txnMgr2.getMS().unlock(checkLock.getLockid());
+ }
+ driver2.lockAndRespond();
+ locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", blocking ? 1 : 2, locks.size());
+
+ checkLock(blocking ? LockType.EXCLUSIVE : LockType.EXCL_WRITE,
+ LockState.ACQUIRED, "default", "T", null, locks);
+
+ Mockito.doNothing().when(driver2).lockAndRespond();
+ driver2.run();
+
+ if (!blocking) {
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+ }
+ Mockito.reset(driver, driver2);
+
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 2 rows and found " + res.size(), 0, res.size());
Review Comment:
fixed
##########
ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java:
##########
@@ -4028,4 +4005,88 @@ public void testAddDropConstraintNonBlocking() throws Exception {
checkLock(LockType.EXCL_WRITE,
LockState.ACQUIRED, "default", "tab_acid", null, locks);
}
+
+ @Test
+ public void testAddPartitionIfNotExistsNonBlocking() throws Exception {
+ testAddPartitionIfNotExists(false);
+ }
+ @Test
+ public void testAddPartitionIfNotExistsBlocking() throws Exception {
+ testAddPartitionIfNotExists(true);
+ }
+
+ private void testAddPartitionIfNotExists(boolean blocking) throws Exception {
+ dropTable(new String[] {"T", "Tstage"});
+
+ HiveConf.setIntVar(conf, HiveConf.ConfVars.HIVE_LOCKS_PARTITION_THRESHOLD, 1);
+ driver = Mockito.spy(driver);
+
+ HiveConf.setBoolVar(driver2.getConf(), HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED, !blocking);
+ driver2 = Mockito.spy(driver2);
+
+ driver.run("create table if not exists T (a int, b int) partitioned by (p string) " +
+ "stored as orc TBLPROPERTIES ('transactional'='true')");
+ //bucketed just so that we get 2 files
+ driver.run("create table if not exists Tstage (a int, b int) clustered by (a) into 2 " +
+ "buckets stored as orc TBLPROPERTIES ('transactional'='false')");
+ driver.run("insert into Tstage values(1,2),(3,4)");
+ String exportLoc = exportFolder.newFolder("1").toString();
+ driver.run("export table Tstage to '" + exportLoc + "'");
+
+ driver.compileAndRespond("select * from T");
+ List<String> res = new ArrayList<>();
+
+ driver.lockAndRespond();
+ List<ShowLocksResponseElement> locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+
+ checkLock(LockType.SHARED_READ,
+ LockState.ACQUIRED, "default", "T", null, locks);
+
+ DbTxnManager txnMgr2 = (DbTxnManager) TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ swapTxnManager(txnMgr2);
+ driver2.compileAndRespond("alter table T add if not exists partition (p='foo') location '" + exportLoc + "/data'");
+
+ if (blocking) {
+ txnMgr2.acquireLocks(driver2.getPlan(), ctx, null, false);
+ locks = getLocks();
+
+ ShowLocksResponseElement checkLock = checkLock(LockType.EXCLUSIVE,
+ LockState.WAITING, "default", "T", null, locks);
+
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+
+ driver.getFetchTask().fetch(res);
+ swapTxnManager(txnMgr2);
+
+ FieldSetter.setField(txnMgr2, txnMgr2.getClass().getDeclaredField("numStatements"), 0);
+ txnMgr2.getMS().unlock(checkLock.getLockid());
+ }
+ driver2.lockAndRespond();
+ locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", blocking ? 1 : 2, locks.size());
+
+ checkLock(blocking ? LockType.EXCLUSIVE : LockType.EXCL_WRITE,
+ LockState.ACQUIRED, "default", "T", null, locks);
+
+ Mockito.doNothing().when(driver2).lockAndRespond();
+ driver2.run();
+
+ if (!blocking) {
+ swapTxnManager(txnMgr);
+ Mockito.doNothing().when(driver).lockAndRespond();
+ driver.run();
+ }
+ Mockito.reset(driver, driver2);
+
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 2 rows and found " + res.size(), 0, res.size());
+
+ driver.run("select * from T where p='foo'");
+ res = new ArrayList<>();
+ driver.getFetchTask().fetch(res);
+ Assert.assertEquals("Expecting 0 rows and found " + res.size(), 2, res.size());
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscribe@hive.apache.org
For additional commands, e-mail: gitbox-help@hive.apache.org