You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2018/06/12 23:10:31 UTC
[2/2] hive git commit: HIVE-19859: Inspect lock components for
DBHiveLock while verifying whether transaction list is valid (Jesus Camacho
Rodriguez, reviewed by Ashutosh Chauhan)
HIVE-19859: Inspect lock components for DBHiveLock while verifying whether transaction list is valid (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/59f8aae2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/59f8aae2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/59f8aae2
Branch: refs/heads/branch-3
Commit: 59f8aae2c5ce26d14874b98d3135a81640085435
Parents: 1cd543f
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Tue Jun 12 09:23:28 2018 -0700
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Jun 12 16:10:12 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Driver.java | 31 +++++++++++++++-----
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 18 ++++++++++--
.../apache/hadoop/hive/ql/lockmgr/HiveLock.java | 26 ++++++++++++++++
3 files changed, 66 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 8b5262a..1f6bb6a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreUtils;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+import org.apache.hadoop.hive.metastore.api.LockType;
import org.apache.hadoop.hive.metastore.api.Schema;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
@@ -806,13 +808,27 @@ public class Driver implements IDriver {
}
Set<String> nonSharedLocks = new HashSet<>();
for (HiveLock lock : ctx.getHiveLocks()) {
- if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
- lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
- if (lock.getHiveLockObject().getPaths().length == 2) {
- // Pos 0 of lock paths array contains dbname, pos 1 contains tblname
- nonSharedLocks.add(
- Warehouse.getQualifiedName(
- lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1]));
+ if (lock.mayContainComponents()) {
+ // The lock may have multiple components, e.g., DbHiveLock, hence we need
+ // to check for each of them
+ for (LockComponent lckCmp : lock.getHiveLockComponents()) {
+ if (lckCmp.getType() == LockType.EXCLUSIVE ||
+ lckCmp.getType() == LockType.SHARED_WRITE) {
+ nonSharedLocks.add(
+ Warehouse.getQualifiedName(
+ lckCmp.getDbname(), lckCmp.getTablename()));
+ }
+ }
+ } else {
+ // The lock has a single components, e.g., SimpleHiveLock or ZooKeeperHiveLock
+ if (lock.getHiveLockMode() == HiveLockMode.EXCLUSIVE ||
+ lock.getHiveLockMode() == HiveLockMode.SEMI_SHARED) {
+ if (lock.getHiveLockObject().getPaths().length == 2) {
+ // Pos 0 of lock paths array contains dbname, pos 1 contains tblname
+ nonSharedLocks.add(
+ Warehouse.getQualifiedName(
+ lock.getHiveLockObject().getPaths()[0], lock.getHiveLockObject().getPaths()[1]));
+ }
}
}
}
@@ -1937,6 +1953,7 @@ public class Driver implements IDriver {
try {
if (!isValidTxnListState()) {
+ LOG.info("Compiling after acquiring locks");
// Snapshot was outdated when locks were acquired, hence regenerate context,
// txn list and retry
// TODO: Lock acquisition should be moved before analyze, this is a bit hackish.
http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
index ab94475..1a04278 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbLockManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.lockmgr;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.DDLTask;
import org.slf4j.Logger;
@@ -114,7 +115,7 @@ public final class DbLockManager implements HiveLockManager{
res = txnManager.getMS().checkLock(res.getLockid());
}
long retryDuration = System.currentTimeMillis() - startRetry;
- DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, lock.getTxnid());
+ DbHiveLock hl = new DbHiveLock(res.getLockid(), queryId, lock.getTxnid(), lock.getComponent());
if(locks.size() > 0) {
boolean logMsg = false;
for(DbHiveLock l : locks) {
@@ -310,14 +311,17 @@ public final class DbLockManager implements HiveLockManager{
long lockId;
String queryId;
long txnId;
+ List<LockComponent> components;
DbHiveLock(long id) {
lockId = id;
}
- DbHiveLock(long id, String queryId, long txnId) {
+
+ DbHiveLock(long id, String queryId, long txnId, List<LockComponent> components) {
lockId = id;
this.queryId = queryId;
this.txnId = txnId;
+ this.components = ImmutableList.copyOf(components);
}
@Override
@@ -331,6 +335,16 @@ public final class DbLockManager implements HiveLockManager{
}
@Override
+ public boolean mayContainComponents() {
+ return true;
+ }
+
+ @Override
+ public List<LockComponent> getHiveLockComponents() {
+ return components;
+ }
+
+ @Override
public boolean equals(Object other) {
if (other instanceof DbHiveLock) {
return lockId == ((DbHiveLock)other).lockId;
http://git-wip-us.apache.org/repos/asf/hive/blob/59f8aae2/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
index 5373137..8239245 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLock.java
@@ -18,7 +18,33 @@
package org.apache.hadoop.hive.ql.lockmgr;
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.hive.metastore.api.LockComponent;
+
+import java.util.List;
+
public abstract class HiveLock {
+
public abstract HiveLockObject getHiveLockObject();
+
public abstract HiveLockMode getHiveLockMode();
+
+ /**
+ * Returns true if for this lock implementation, a single lock can in turn
+ * lock multiple objects, e.g., multi-statement transaction.
+ */
+ public boolean mayContainComponents() {
+ return false;
+ }
+
+ /**
+ * Returns the lock components if a single lock can in turn
+ * lock multiple objects, e.g., multi-statement transaction.
+ *
+ * Returns an empty list if the lock does not have multiple
+ * components.
+ */
+ public List<LockComponent> getHiveLockComponents() {
+ return ImmutableList.of();
+ }
}