You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by pv...@apache.org on 2022/05/24 11:58:23 UTC
[hive] branch master updated: HIVE-26258: Provide an option for enable locking of external tables (#3313) (Peter Vary reviewed by Denys Kuzmenko)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new dee417faa73 HIVE-26258: Provide an option for enable locking of external tables (#3313) (Peter Vary reviewed by Denys Kuzmenko)
dee417faa73 is described below
commit dee417faa7341b827f8289814f0aacdeeed82abb
Author: pvary <pv...@cloudera.com>
AuthorDate: Tue May 24 13:58:15 2022 +0200
HIVE-26258: Provide an option for enable locking of external tables (#3313) (Peter Vary reviewed by Denys Kuzmenko)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 6 ++++
.../org/apache/hadoop/hive/ql/io/AcidUtils.java | 15 +++++----
.../hadoop/hive/ql/lockmgr/TestDbTxnManager2.java | 36 ++++++++++++++++++++++
3 files changed, 51 insertions(+), 6 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a14872995b5..a0cd44583cc 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3008,6 +3008,12 @@ public class HiveConf extends Configuration {
"and hive.exec.dynamic.partition.mode (nonstrict).\n" +
"The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" +
"no transactions."),
+ HIVE_TXN_EXT_LOCKING_ENABLED("hive.txn.ext.locking.enabled", false,
+ "When enabled use standard R/W lock semantics based on hive.txn.strict.locking.mode for external resources,\n" +
+ "e.g. INSERT will acquire lock based on hive.txn.strict.locking.mode\n" +
+ "(exclusive if it is true, shared if that is false),\n" +
+ "SELECT will acquire shared lock based on hive.txn.nonacid.read.locks.\n" +
+ "When disabled no locks are acquired for external resources."),
HIVE_TXN_STRICT_LOCKING_MODE("hive.txn.strict.locking.mode", true, "In strict mode non-ACID\n" +
"resources use standard R/W lock semantics, e.g. INSERT will acquire exclusive lock.\n" +
"In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock, which\n" +
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
index 70fa42bad66..42ab15e9625 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java
@@ -2844,12 +2844,12 @@ public class AcidUtils {
tblProps.remove(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES);
}
- private static boolean needsLock(Entity entity) {
+ private static boolean needsLock(Entity entity, boolean isExternalEnabled) {
switch (entity.getType()) {
case TABLE:
- return isLockableTable(entity.getTable());
+ return isLockableTable(entity.getTable(), isExternalEnabled);
case PARTITION:
- return isLockableTable(entity.getPartition().getTable());
+ return isLockableTable(entity.getPartition().getTable(), isExternalEnabled);
default:
return true;
}
@@ -2863,7 +2863,7 @@ public class AcidUtils {
return t;
}
- private static boolean isLockableTable(Table t) {
+ private static boolean isLockableTable(Table t, boolean isExternalEnabled) {
if (t.isTemporary()) {
return false;
}
@@ -2871,6 +2871,8 @@ public class AcidUtils {
case MANAGED_TABLE:
case MATERIALIZED_VIEW:
return true;
+ case EXTERNAL_TABLE:
+ return isExternalEnabled;
default:
return false;
}
@@ -2890,6 +2892,7 @@ public class AcidUtils {
boolean skipReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_READ_LOCKS);
boolean skipNonAcidReadLock = !conf.getBoolVar(ConfVars.HIVE_TXN_NONACID_READ_LOCKS);
boolean sharedWrite = !conf.getBoolVar(HiveConf.ConfVars.TXN_WRITE_X_LOCK);
+ boolean isExternalEnabled = conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED);
boolean isMerge = operation == Context.Operation.MERGE;
// We don't want to acquire read locks during update or delete as we'll be acquiring write
@@ -2898,7 +2901,7 @@ public class AcidUtils {
.filter(input -> !input.isDummy()
&& input.needsLock()
&& !input.isUpdateOrDelete()
- && AcidUtils.needsLock(input)
+ && AcidUtils.needsLock(input, isExternalEnabled)
&& !skipReadLock)
.collect(Collectors.toList());
@@ -2959,7 +2962,7 @@ public class AcidUtils {
for (WriteEntity output : outputs) {
LOG.debug("output is null " + (output == null));
if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR || !AcidUtils
- .needsLock(output)) {
+ .needsLock(output, isExternalEnabled)) {
// We don't lock files or directories. We also skip locking temp tables.
continue;
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index e0e62263ee7..a28151ecacd 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -864,6 +864,42 @@ public class TestDbTxnManager2 extends DbTxnManagerEndToEndTestBase{
dropTable(new String[] {"tab_not_acid"});
}
+ @Test
+ public void testLockingExternalInStrictModeInsert() throws Exception {
+ dropTable(new String[] {"tab_not_acid"});
+ driver.run("create external table if not exists tab_not_acid (na int, nb int) partitioned by (np string) " +
+ "clustered by (na) into 2 buckets stored as orc TBLPROPERTIES ('transactional'='false')");
+ driver.run("insert into tab_not_acid partition(np) (na,nb,np) values(1,2,'blah'),(3,4,'doh')");
+
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
+ HiveTxnManager txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr.openTxn(ctx, "T1");
+ driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)", true);
+ ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+ checkLock(LockType.EXCLUSIVE, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, false);
+ }
+
+ @Test
+ public void testLockingExternalInStrictModeSelect() throws Exception {
+ dropTable(new String[] {"tab_not_acid"});
+ driver.run("create external table if not exists tab_not_acid (na int, nb int) " +
+ "stored as orc TBLPROPERTIES ('transactional'='false')");
+ driver.run("insert into tab_not_acid values(1,2),(3,4)");
+
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, true);
+ HiveTxnManager txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr.openTxn(ctx, "T1");
+ driver.compileAndRespond("select * from tab_not_acid", true);
+ ((DbTxnManager)txnMgr).acquireLocks(driver.getPlan(), ctx, "T1", false);
+ List<ShowLocksResponseElement> locks = getLocks(txnMgr);
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED, false);
+ }
+
/** The list is small, and the object is generated, so we don't use sets/equals/etc. */
public static ShowLocksResponseElement checkLock(LockType expectedType, LockState expectedState, String expectedDb,
String expectedTable, String expectedPartition, List<ShowLocksResponseElement> actuals) {