You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/02/02 22:07:40 UTC
hive git commit: HIVE-15774 : Ensure DbLockManager backward
compatibility for non-ACID resources (Wei Zheng, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master ed3b05185 -> aa20b777e
HIVE-15774 : Ensure DbLockManager backward compatibility for non-ACID resources (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/aa20b777
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/aa20b777
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/aa20b777
Branch: refs/heads/master
Commit: aa20b777e87dd1e5e4fadb26680aac200fb10060
Parents: ed3b051
Author: Wei Zheng <we...@apache.org>
Authored: Thu Feb 2 14:07:26 2017 -0800
Committer: Wei Zheng <we...@apache.org>
Committed: Thu Feb 2 14:07:26 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +++++
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 8 ++++++--
.../hive/ql/lockmgr/TestDbTxnManager2.java | 19 +++++++++++++++++++
3 files changed, 30 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 29de605..53b9b0c 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -1832,6 +1832,11 @@ public class HiveConf extends Configuration {
"(true), and hive.exec.dynamic.partition.mode (nonstrict).\n" +
"The default DummyTxnManager replicates pre-Hive-0.13 behavior and provides\n" +
"no transactions."),
+ HIVE_TXN_STRICT_LOCKING_MODE("hive.txn.strict.locking.mode", true, "In strict mode non-ACID\n" +
+ "resources use standard R/W lock semantics, e.g. INSERT will acquire exclusive lock.\n" +
+ "In nonstrict mode, for non-ACID resources, INSERT will only acquire shared lock, which\n" +
+ "allows two concurrent writes to the same partition but still lets lock manager prevent\n" +
+ "DROP TABLE etc. when the table is being written to"),
HIVE_TXN_TIMEOUT("hive.txn.timeout", "300s", new TimeValidator(TimeUnit.SECONDS),
"time after which transactions are declared aborted if the client has not sent a heartbeat."),
HIVE_TXN_HEARTBEAT_THREADPOOL_SIZE("hive.txn.heartbeat.threadpool.size", 5, "The number of " +
http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 28ca77c..a985eb1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -250,6 +250,7 @@ public class DbTxnManager extends HiveTxnManagerImpl {
// overwrite) than we need a shared. If it's update or delete then we
// need a SEMI-SHARED.
for (WriteEntity output : plan.getOutputs()) {
+ LOG.debug("output is null " + (output == null));
if (output.getType() == Entity.Type.DFS_DIR || output.getType() == Entity.Type.LOCAL_DIR ||
(output.getType() == Entity.Type.TABLE && output.getTable().isTemporary())) {
// We don't lock files or directories. We also skip locking temp tables.
@@ -257,7 +258,6 @@ public class DbTxnManager extends HiveTxnManagerImpl {
}
LockComponentBuilder compBuilder = new LockComponentBuilder();
Table t = null;
- LOG.debug("output is null " + (output == null));
switch (output.getWriteType()) {
case DDL_EXCLUSIVE:
case INSERT_OVERWRITE:
@@ -272,7 +272,11 @@ public class DbTxnManager extends HiveTxnManagerImpl {
compBuilder.setIsAcid(true);
}
else {
- compBuilder.setExclusive();
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE)) {
+ compBuilder.setExclusive();
+ } else { // this is backward compatible for non-ACID resources, w/o ACID semantics
+ compBuilder.setShared();
+ }
compBuilder.setIsAcid(false);
}
compBuilder.setOperationType(DataOperationType.INSERT);
http://git-wip-us.apache.org/repos/asf/hive/blob/aa20b777/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 384f6eb..180eea6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -687,6 +687,25 @@ public class TestDbTxnManager2 {
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
+
+ // Test strict locking mode, i.e. backward compatible locking mode for non-ACID resources.
+ // With non-strict mode, INSERT got SHARED_READ lock, instead of EXCLUSIVE with ACID semantics
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, false);
+ HiveTxnManager txnMgr3 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ txnMgr3.openTxn(ctx, "T3");
+ checkCmdOnDriver(driver.compileAndRespond("insert into tab_not_acid partition(np='blah') values(7,8)"));
+ ((DbTxnManager)txnMgr3).acquireLocks(driver.getPlan(), ctx, "T3", false);
+ locks = getLocks(txnMgr3);
+ Assert.assertEquals("Unexpected lock count", 8, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=bar", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_acid", "p=foo", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", null, locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=doh", locks);
+ checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "tab_not_acid", "np=doh", locks);
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "tab_not_acid", "np=blah", locks);
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, true);
}
/** The list is small, and the object is generated, so we don't use sets/equals/etc. */