You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by su...@apache.org on 2015/07/14 20:49:55 UTC
hive git commit: HIVE-11130 - Refactoring the code so that
HiveTxnManager interface will support lock/unlock table/database object
(Aihua Xu, reviewed by Alan Gates and Chao Sun)
Repository: hive
Updated Branches:
refs/heads/master 999e0e361 -> 46c76d6b8
HIVE-11130 - Refactoring the code so that HiveTxnManager interface will support lock/unlock table/database object (Aihua Xu, reviewed by Alan Gates and Chao Sun)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/46c76d6b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/46c76d6b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/46c76d6b
Branch: refs/heads/master
Commit: 46c76d6b8e1fb323d1f8ac75061b6fd3bb747d6d
Parents: 999e0e3
Author: Aihua Xu <ai...@gmail.com>
Authored: Tue Jul 14 11:49:09 2015 -0700
Committer: Chao Sun <ch...@cloudera.com>
Committed: Tue Jul 14 11:49:09 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 147 +------------------
.../hadoop/hive/ql/lockmgr/HiveLockObject.java | 35 +++++
.../hadoop/hive/ql/lockmgr/HiveTxnManager.java | 47 +++++-
.../hive/ql/lockmgr/HiveTxnManagerImpl.java | 140 ++++++++++++++++++
4 files changed, 227 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 049857b..a8c6aca 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2444,8 +2444,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
locks = lockMgr.getLocks(false, isExt);
}
else {
- locks = lockMgr.getLocks(getHiveObject(showLocks.getTableName(),
- showLocks.getPartSpec()),
+ locks = lockMgr.getLocks(HiveLockObject.createFrom(db,
+ showLocks.getTableName(), showLocks.getPartSpec()),
true, isExt);
}
@@ -2705,46 +2705,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int lockTable(LockTableDesc lockTbl) throws HiveException {
Context ctx = driverContext.getCtx();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Table LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
- String tabName = lockTbl.getTableName();
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- Map<String, String> partSpec = lockTbl.getPartSpec();
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockTbl.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT",
- lockTbl.getQueryStr());
-
- if (partSpec == null) {
- HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
- }
-
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockTable(db, lockTbl);
}
/**
@@ -2759,33 +2720,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int lockDatabase(LockDatabaseDesc lockDb) throws HiveException {
Context ctx = driverContext.getCtx();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("lock Database LockManager not specified");
- }
-
- HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
- String dbName = lockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
-
- HiveLockObjectData lockData =
- new HiveLockObjectData(lockDb.getQueryId(),
- String.valueOf(System.currentTimeMillis()),
- "EXPLICIT", lockDb.getQueryStr());
-
- HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
- if (lck == null) {
- return 1;
- }
- return 0;
+ return txnManager.lockDatabase(db, lockDb);
}
/**
@@ -2800,55 +2735,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int unlockDatabase(UnlockDatabaseDesc unlockDb) throws HiveException {
Context ctx = driverContext.getCtx();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Database LockManager not specified");
- }
-
- String dbName = unlockDb.getDatabaseName();
-
- Database dbObj = db.getDatabase(dbName);
- if (dbObj == null) {
- throw new HiveException("Database " + dbName + " does not exist ");
- }
- HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Database " + dbName + " is not locked ");
- }
-
- for (HiveLock lock: locks) {
- lockMgr.unlock(lock);
-
- }
- return 0;
- }
-
- private HiveLockObject getHiveObject(String tabName,
- Map<String, String> partSpec) throws HiveException {
- Table tbl = db.getTable(tabName);
- if (tbl == null) {
- throw new HiveException("Table " + tabName + " does not exist ");
- }
-
- HiveLockObject obj = null;
-
- if (partSpec == null) {
- obj = new HiveLockObject(tbl, null);
- }
- else {
- Partition par = db.getPartition(tbl, partSpec, false);
- if (par == null) {
- throw new HiveException("Partition " + partSpec + " for table " + tabName + " does not exist");
- }
- obj = new HiveLockObject(par, null);
- }
- return obj;
+ return txnManager.unlockDatabase(db, unlockDb);
}
/**
@@ -2863,29 +2750,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int unlockTable(UnlockTableDesc unlockTbl) throws HiveException {
Context ctx = driverContext.getCtx();
HiveTxnManager txnManager = ctx.getHiveTxnManager();
- if (!txnManager.supportsExplicitLock()) {
- throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
- conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
- }
- HiveLockManager lockMgr = txnManager.getLockManager();
- if (lockMgr == null) {
- throw new HiveException("unlock Table LockManager not specified");
- }
-
- String tabName = unlockTbl.getTableName();
- HiveLockObject obj = getHiveObject(tabName, unlockTbl.getPartSpec());
-
- List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
- if ((locks == null) || (locks.isEmpty())) {
- throw new HiveException("Table " + tabName + " is not locked ");
- }
- Iterator<HiveLock> locksIter = locks.iterator();
- while (locksIter.hasNext()) {
- HiveLock lock = locksIter.next();
- lockMgr.unlock(lock);
- }
-
- return 0;
+ return txnManager.unlockTable(db, unlockTbl);
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
index 7e93387..fadd074 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveLockObject.java
@@ -19,9 +19,12 @@
package org.apache.hadoop.hive.ql.lockmgr;
import java.util.Arrays;
+import java.util.Map;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -196,6 +199,38 @@ public class HiveLockObject {
this(new String[] {par.getName()}, lockData);
}
+ /**
+ * Creates a locking object for a table (when partition spec is not provided)
+ * or a table partition
+ * @param hiveDB an object to communicate with the metastore
+ * @param tableName the table to create the locking object on
+ * @param partSpec the spec of a partition to create the locking object on
+ * @return the locking object
+ * @throws HiveException
+ */
+ public static HiveLockObject createFrom(Hive hiveDB, String tableName,
+ Map<String, String> partSpec) throws HiveException {
+ Table tbl = hiveDB.getTable(tableName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tableName + " does not exist ");
+ }
+
+ HiveLockObject obj = null;
+
+ if (partSpec == null) {
+ obj = new HiveLockObject(tbl, null);
+ }
+ else {
+ Partition par = hiveDB.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tableName + " does not exist");
+ }
+ obj = new HiveLockObject(par, null);
+ }
+ return obj;
+ }
+
public String[] getPaths() {
return pathNames;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
index 6c3dc33..c900548 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManager.java
@@ -18,9 +18,14 @@
package org.apache.hadoop.hive.ql.lockmgr;
import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryPlan;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
/**
* An interface that allows Hive to manage transactions. All classes
@@ -116,6 +121,46 @@ public interface HiveTxnManager {
boolean supportsExplicitLock();
/**
+ * This function is called to lock the table when explicit lock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockTbl table locking info, such as table name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockTable(Hive hiveDB, LockTableDesc lockTbl) throws HiveException;
+
+ /**
+ * This function is called to unlock the table when explicit unlock command is
+ * issued on a table.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockTbl table unlocking info, such as table name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException;
+
+ /**
+ * This function is called to lock the database when explicit lock command is
+ * issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param lockDb database locking info, such as database name, locking mode
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException;
+
+ /**
+ * This function is called to unlock the database when explicit unlock command
+ * is issued on a database.
+ * @param hiveDB an object to communicate with the metastore
+ * @param unlockDb database unlocking info, such as database name
+ * @return 0 if the locking succeeds, 1 otherwise.
+ * @throws HiveException
+ */
+ int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException;
+
+ /**
* Indicate whether this transaction manager returns information about locks in the new format
* for show locks or the old one.
* @return true if the new format should be used.
http://git-wip-us.apache.org/repos/asf/hive/blob/46c76d6b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
index eccb8d1..ceeae68 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/HiveTxnManagerImpl.java
@@ -17,7 +17,22 @@
*/
package org.apache.hadoop.hive.ql.lockmgr;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.lockmgr.HiveLockObject.HiveLockObjectData;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.plan.LockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.LockTableDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockDatabaseDesc;
+import org.apache.hadoop.hive.ql.plan.UnlockTableDesc;
/**
* An implementation HiveTxnManager that includes internal methods that all
@@ -44,4 +59,129 @@ abstract class HiveTxnManagerImpl implements HiveTxnManager {
destruct();
}
+ @Override
+ public int lockTable(Hive db, LockTableDesc lockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockTbl.getMode());
+ String tabName = lockTbl.getTableName();
+ Table tbl = db.getTable(tabName);
+ if (tbl == null) {
+ throw new HiveException("Table " + tabName + " does not exist ");
+ }
+
+ Map<String, String> partSpec = lockTbl.getPartSpec();
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockTbl.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT",
+ lockTbl.getQueryStr());
+
+ if (partSpec == null) {
+ HiveLock lck = lockMgr.lock(new HiveLockObject(tbl, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ Partition par = db.getPartition(tbl, partSpec, false);
+ if (par == null) {
+ throw new HiveException("Partition " + partSpec + " for table " +
+ tabName + " does not exist");
+ }
+ HiveLock lck = lockMgr.lock(new HiveLockObject(par, lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockTable(Hive hiveDB, UnlockTableDesc unlockTbl) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String tabName = unlockTbl.getTableName();
+ HiveLockObject obj = HiveLockObject.createFrom(hiveDB, tabName,
+ unlockTbl.getPartSpec());
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Table " + tabName + " is not locked ");
+ }
+ Iterator<HiveLock> locksIter = locks.iterator();
+ while (locksIter.hasNext()) {
+ HiveLock lock = locksIter.next();
+ lockMgr.unlock(lock);
+ }
+
+ return 0;
+ }
+
+ @Override
+ public int lockDatabase(Hive hiveDB, LockDatabaseDesc lockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ HiveLockMode mode = HiveLockMode.valueOf(lockDb.getMode());
+ String dbName = lockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+
+ HiveLockObjectData lockData =
+ new HiveLockObjectData(lockDb.getQueryId(),
+ String.valueOf(System.currentTimeMillis()),
+ "EXPLICIT", lockDb.getQueryStr());
+
+ HiveLock lck = lockMgr.lock(new HiveLockObject(dbObj.getName(), lockData), mode, true);
+ if (lck == null) {
+ return 1;
+ }
+ return 0;
+ }
+
+ @Override
+ public int unlockDatabase(Hive hiveDB, UnlockDatabaseDesc unlockDb) throws HiveException {
+ HiveLockManager lockMgr = getAndCheckLockManager();
+
+ String dbName = unlockDb.getDatabaseName();
+
+ Database dbObj = hiveDB.getDatabase(dbName);
+ if (dbObj == null) {
+ throw new HiveException("Database " + dbName + " does not exist ");
+ }
+ HiveLockObject obj = new HiveLockObject(dbObj.getName(), null);
+
+ List<HiveLock> locks = lockMgr.getLocks(obj, false, false);
+ if ((locks == null) || (locks.isEmpty())) {
+ throw new HiveException("Database " + dbName + " is not locked ");
+ }
+
+ for (HiveLock lock: locks) {
+ lockMgr.unlock(lock);
+
+ }
+ return 0;
+ }
+
+ /**
+ * Gets the lock manager and verifies if the explicit lock is supported
+ * @return the lock manager
+ * @throws HiveException
+ */
+ protected HiveLockManager getAndCheckLockManager() throws HiveException {
+ HiveLockManager lockMgr = getLockManager();
+ if (lockMgr == null) {
+ throw new HiveException("LockManager cannot be acquired");
+ }
+
+ if (!supportsExplicitLock()) {
+ throw new HiveException(ErrorMsg.LOCK_REQUEST_UNSUPPORTED,
+ conf.getVar(HiveConf.ConfVars.HIVE_TXN_MANAGER));
+ }
+
+ return lockMgr;
+ }
}