You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/04/04 22:36:46 UTC
[11/50] [abbrv] hive git commit: HIVE-12367 : Lock/unlock database
should add current database to inputs and outputs of authz hook (Dapeng Sun
via Ashutosh Chauhan)
HIVE-12367 : Lock/unlock database should add current database to inputs and outputs of authz hook (Dapeng Sun via Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6bfec2e9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6bfec2e9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6bfec2e9
Branch: refs/heads/llap
Commit: 6bfec2e97c4e434646aa9aeffd98c9939313fa6e
Parents: a71edcf
Author: Dapeng Sun <sd...@apache.org>
Authored: Tue Dec 15 19:39:00 2015 -0800
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu Mar 24 19:22:28 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 20 ++++++++++++++++++++
.../hive/ql/parse/DDLSemanticAnalyzer.java | 15 +++++++++++++--
.../clientnegative/dbtxnmgr_nodblock.q.out | 2 ++
.../clientnegative/dbtxnmgr_nodbunlock.q.out | 2 ++
.../lockneg_query_tbl_in_locked_db.q.out | 6 ++++++
.../lockneg_try_db_lock_conflict.q.out | 6 ++++++
.../lockneg_try_drop_locked_db.q.out | 4 ++++
.../lockneg_try_lock_db_in_use.q.out | 6 ++++++
8 files changed, 59 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index d7e4ac7..7276e31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1416,6 +1416,10 @@ public class Driver implements CommandProcessor {
if (!checkConcurrency()) {
return false;
}
+ // Lock operations themselves don't require the lock.
+ if (isExplicitLockOperation()){
+ return false;
+ }
if (!HiveConf.getBoolVar(conf, ConfVars.HIVE_LOCK_MAPRED_ONLY)) {
return true;
}
@@ -1438,6 +1442,22 @@ public class Driver implements CommandProcessor {
return false;
}
+ private boolean isExplicitLockOperation() {
+ HiveOperation currentOpt = plan.getOperation();
+ if (currentOpt != null) {
+ switch (currentOpt) {
+ case LOCKDB:
+ case UNLOCKDB:
+ case LOCKTABLE:
+ case UNLOCKTABLE:
+ return true;
+ default:
+ return false;
+ }
+ }
+ return false;
+ }
+
private CommandProcessorResponse createProcessorResponse(int ret) {
queryDisplay.setErrorMessage(errorMessage);
return new CommandProcessorResponse(ret, errorMessage, SQLState, downstreamError);
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 0c087ed..fe9b8cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -2444,8 +2444,12 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String dbName = unescapeIdentifier(ast.getChild(0).getText());
String mode = unescapeIdentifier(ast.getChild(1).getText().toUpperCase());
- //inputs.add(new ReadEntity(dbName));
- //outputs.add(new WriteEntity(dbName));
+ inputs.add(new ReadEntity(getDatabase(dbName)));
+ // Lock database operation is to acquire the lock explicitly, the operation
+ // itself doesn't need to be locked. Set the WriteEntity as WriteType:
+ // DDL_NO_LOCK here, otherwise it will conflict with Hive's transaction.
+ outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
LockDatabaseDesc lockDatabaseDesc = new LockDatabaseDesc(dbName, mode,
HiveConf.getVar(conf, ConfVars.HIVEQUERYID));
lockDatabaseDesc.setQueryStr(ctx.getCmd());
@@ -2457,6 +2461,13 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
private void analyzeUnlockDatabase(ASTNode ast) throws SemanticException {
String dbName = unescapeIdentifier(ast.getChild(0).getText());
+ inputs.add(new ReadEntity(getDatabase(dbName)));
+ // Unlock database operation is to release the lock explicitly, the
+ // operation itself don't need to be locked. Set the WriteEntity as
+ // WriteType: DDL_NO_LOCK here, otherwise it will conflict with
+ // Hive's transaction.
+ outputs.add(new WriteEntity(getDatabase(dbName), WriteType.DDL_NO_LOCK));
+
UnlockDatabaseDesc unlockDatabaseDesc = new UnlockDatabaseDesc(dbName);
DDLWork work = new DDLWork(getInputs(), getOutputs(), unlockDatabaseDesc);
rootTasks.add(TaskFactory.get(work, conf));
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
index b8bbe93..e783251 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodblock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:drop_nodblock
PREHOOK: query: lock database drop_nodblock shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:drop_nodblock
+PREHOOK: Output: database:drop_nodblock
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
index ea7a3eb..d7a39f0 100644
--- a/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
+++ b/ql/src/test/results/clientnegative/dbtxnmgr_nodbunlock.q.out
@@ -10,4 +10,6 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:drop_nodbunlock
PREHOOK: query: unlock database drop_nodbunlock
PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:drop_nodbunlock
+PREHOOK: Output: database:drop_nodbunlock
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Current transaction manager does not support explicit lock requests. Transaction manager: org.apache.hadoop.hive.ql.lockmgr.DbTxnManager
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
index 134a57b..2c46159 100644
--- a/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_query_tbl_in_locked_db.q.out
@@ -34,8 +34,12 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: lock database lockneg1 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
POSTHOOK: query: lock database lockneg1 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg1
+POSTHOOK: Output: database:lockneg1
PREHOOK: query: show locks database lockneg1
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks database lockneg1
@@ -53,4 +57,6 @@ POSTHOOK: Input: lockneg1@tstsrcpart@ds=2008-04-08/hr=11
500
PREHOOK: query: unlock database lockneg1
PREHOOK: type: UNLOCKDATABASE
+PREHOOK: Input: database:lockneg1
+PREHOOK: Output: database:lockneg1
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Database lockneg1 is not locked
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
index 0a5d98a..fe12d83 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_db_lock_conflict.q.out
@@ -6,9 +6,15 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:lockneg4
PREHOOK: query: lock database lockneg4 exclusive
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
POSTHOOK: query: lock database lockneg4 exclusive
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg4
+POSTHOOK: Output: database:lockneg4
PREHOOK: query: lock database lockneg4 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg4
+PREHOOK: Output: database:lockneg4
Unable to acquire EXPLICIT, SHARED lock lockneg4 after 1 attempts.
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
index 5ac4e3c..e66965e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_drop_locked_db.q.out
@@ -6,8 +6,12 @@ POSTHOOK: type: CREATEDATABASE
POSTHOOK: Output: database:lockneg9
PREHOOK: query: lock database lockneg9 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg9
+PREHOOK: Output: database:lockneg9
POSTHOOK: query: lock database lockneg9 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg9
+POSTHOOK: Output: database:lockneg9
PREHOOK: query: show locks database lockneg9
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks database lockneg9
http://git-wip-us.apache.org/repos/asf/hive/blob/6bfec2e9/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
index 5486151..e5c8f3e 100644
--- a/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
+++ b/ql/src/test/results/clientnegative/lockneg_try_lock_db_in_use.q.out
@@ -34,13 +34,19 @@ POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).key SIMPLE [(srcpar
POSTHOOK: Lineage: tstsrcpart PARTITION(ds=2008-04-08,hr=11).value SIMPLE [(srcpart)srcpart.FieldSchema(name:value, type:string, comment:default), ]
PREHOOK: query: lock database lockneg2 shared
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
POSTHOOK: query: lock database lockneg2 shared
POSTHOOK: type: LOCKDATABASE
+POSTHOOK: Input: database:lockneg2
+POSTHOOK: Output: database:lockneg2
PREHOOK: query: show locks
PREHOOK: type: SHOWLOCKS
POSTHOOK: query: show locks
POSTHOOK: type: SHOWLOCKS
PREHOOK: query: lock database lockneg2 exclusive
PREHOOK: type: LOCKDATABASE
+PREHOOK: Input: database:lockneg2
+PREHOOK: Output: database:lockneg2
Unable to acquire EXPLICIT, EXCLUSIVE lock lockneg2 after 1 attempts.
FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask