You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/10/16 23:10:33 UTC
hive git commit: HIVE-16688 Make sure Alter Table to set
transaction=true acquires X lock (Eugene Koifman, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 599a74f8f -> 45b9b8d76
HIVE-16688 Make sure Alter Table to set transaction=true acquires X lock (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/45b9b8d7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/45b9b8d7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/45b9b8d7
Branch: refs/heads/master
Commit: 45b9b8d76759c437d6cf0cb1b9013f9ef2a93835
Parents: 599a74f
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Mon Oct 16 16:10:15 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Mon Oct 16 16:10:15 2017 -0700
----------------------------------------------------------------------
.../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +++++++-
.../hive/ql/lockmgr/TestDbTxnManager2.java | 39 ++++++++++++++++----
2 files changed, 45 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/45b9b8d7/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 1d5d896..9730518 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -23,6 +23,7 @@ import com.google.common.collect.Lists;
import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.Tree;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1511,6 +1512,18 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
alterTblDesc), conf));
}
+ private WriteType determineAlterTableWriteType(Table tab, AlterTableDesc desc, AlterTableTypes op) {
+ boolean convertingToAcid = false;
+ if(desc != null && desc.getProps() != null && Boolean.parseBoolean(desc.getProps().get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL))) {
+ convertingToAcid = true;
+ }
+ if(!AcidUtils.isAcidTable(tab) && convertingToAcid) {
+ //non to acid conversion (property itself) must be mutexed to prevent concurrent writes.
+ // See HIVE-16688 for use case.
+ return WriteType.DDL_EXCLUSIVE;
+ }
+ return WriteEntity.determineAlterTableWriteType(op);
+ }
private void addInputsOutputsAlterTable(String tableName, Map<String, String> partSpec,
AlterTableTypes op) throws SemanticException {
addInputsOutputsAlterTable(tableName, partSpec, null, op, false);
@@ -1545,7 +1558,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
// Determine the lock type to acquire
WriteEntity.WriteType writeType = doForceExclusive
- ? WriteType.DDL_EXCLUSIVE : WriteEntity.determineAlterTableWriteType(op);
+ ? WriteType.DDL_EXCLUSIVE : determineAlterTableWriteType(tab, desc, op);
if (!alterPartitions) {
inputs.add(new ReadEntity(tab));
http://git-wip-us.apache.org/repos/asf/hive/blob/45b9b8d7/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index e9833cb..15045d6 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.ql.lockmgr;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.metastore.api.AddDynamicPartitions;
import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
@@ -42,8 +41,6 @@ import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,11 +51,6 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.ThreadFactory;
/**
* See additional tests in {@link org.apache.hadoop.hive.ql.lockmgr.TestDbTxnManager}
@@ -118,6 +110,37 @@ public class TestDbTxnManager2 {
driver.close();
if (txnMgr != null) txnMgr.closeTxnManager();
}
+
+ /**
+ * HIVE-16688
+ */
+ @Test
+ public void testMetadataOperationLocks() throws Exception {
+ boolean isStrict = conf.getBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE);
+ //to make insert into non-acid take shared lock
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, false);
+ dropTable(new String[] {"T"});
+ checkCmdOnDriver(driver.run("create table if not exists T (a int, b int)"));
+ checkCmdOnDriver(driver.compileAndRespond("insert into T values (1,2)"));
+ txnMgr.acquireLocks(driver.getPlan(), ctx, "Fifer");
+ List<ShowLocksResponseElement> locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 1, locks.size());
+ //since LM is using non strict mode we get shared lock
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks);
+
+ //simulate concurrent session
+ HiveTxnManager txnMgr2 = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
+ swapTxnManager(txnMgr2);
+ checkCmdOnDriver(driver.compileAndRespond("alter table T SET TBLPROPERTIES ('transactional'='true')"));
+ ((DbTxnManager)txnMgr2).acquireLocks(driver.getPlan(), ctx, "Fiddler", false);
+ locks = getLocks();
+ Assert.assertEquals("Unexpected lock count", 2, locks.size());
+ checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "T", null, locks);
+ checkLock(LockType.EXCLUSIVE, LockState.WAITING, "default", "T", null, locks);
+ txnMgr2.rollbackTxn();
+ txnMgr.commitTxn();
+ conf.setBoolVar(HiveConf.ConfVars.HIVE_TXN_STRICT_LOCKING_MODE, isStrict);
+ }
@Test
public void testLocksInSubquery() throws Exception {
dropTable(new String[] {"T","S", "R"});