You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2014/05/20 06:33:06 UTC
svn commit: r1596109 - in
/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn:
CompactionTxnHandler.java TxnHandler.java
Author: gates
Date: Tue May 20 04:33:05 2014
New Revision: 1596109
URL: http://svn.apache.org/r1596109
Log:
HIVE-6965 Transaction manager should use RDBMS time instead of machine time (Alan Gates, reviewed by Ashutosh Chauhan)
Modified:
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java?rev=1596109&r1=1596108&r2=1596109&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java Tue May 20 04:33:05 2014
@@ -168,7 +168,7 @@ public class CompactionTxnHandler extend
}
// Now, update this record as being worked on by this worker.
- long now = System.currentTimeMillis();
+ long now = getDbTime(dbConn);
s = "update COMPACTION_QUEUE set cq_worker_id = '" + workerId + "', " +
"cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id;
LOG.debug("Going to execute update <" + s + ">");
@@ -485,7 +485,7 @@ public class CompactionTxnHandler extend
public void revokeTimedoutWorkers(long timeout) throws MetaException {
try {
Connection dbConn = getDbConn();
- long latestValidStart = System.currentTimeMillis() - timeout;
+ long latestValidStart = getDbTime(dbConn) - timeout;
try {
Statement stmt = dbConn.createStatement();
String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java?rev=1596109&r1=1596108&r2=1596109&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java Tue May 20 04:33:05 2014
@@ -74,6 +74,7 @@ public class TxnHandler {
*/
protected int deadlockCnt;
protected HiveConf conf;
+ protected DatabaseProduct dbProduct;
// Transaction timeout, in milliseconds.
private long timeout;
@@ -253,7 +254,7 @@ public class TxnHandler {
String s = "update NEXT_TXN_ID set ntxn_next = " + (first + numTxns);
LOG.debug("Going to execute update <" + s + ">");
stmt.executeUpdate(s);
- long now = System.currentTimeMillis();
+ long now = getDbTime(dbConn);
s = "insert into TXNS (txn_id, txn_state, txn_started, " +
"txn_last_heartbeat, txn_user, txn_host) values (?, 'o', " + now + ", " +
now + ", '" + rqst.getUser() + "', '" + rqst.getHostname() + "')";
@@ -443,7 +444,7 @@ public class TxnHandler {
heartbeatLock(dbConn, extLockId);
long txnid = getTxnIdFromLockId(dbConn, extLockId);
if (txnid > 0) heartbeatTxn(dbConn, txnid);
- return checkLock(dbConn, extLockId, txnid, true);
+ return checkLock(dbConn, extLockId, true);
} catch (SQLException e) {
try {
LOG.debug("Going to rollback");
@@ -836,6 +837,90 @@ public class TxnHandler {
}
}
+ /**
+ * Determine the current time, using the RDBMS as a source of truth
+ * @param conn database connection
+ * @return current time in milliseconds
+ * @throws org.apache.hadoop.hive.metastore.api.MetaException if the time cannot be determined
+ */
+ protected long getDbTime(Connection conn) throws MetaException {
+ try {
+ Statement stmt = conn.createStatement();
+ String s;
+ ResultSet rs;
+ DatabaseProduct prod = determineDatabaseProduct(conn);
+ switch (prod) {
+ case DERBY:
+ s = "values current_timestamp";
+ break;
+
+ case MYSQL:
+ case POSTGRES:
+ case SQLSERVER:
+ s = "select current_timestamp";
+ break;
+
+ case ORACLE:
+ s = "select current_timestamp from dual";
+ break;
+
+ default:
+ String msg = "Unknown database product: " + prod.toString();
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+ LOG.debug("Going to execute query <" + s + ">");
+ rs = stmt.executeQuery(s);
+ if (!rs.next()) throw new MetaException("No results from date query");
+ return rs.getTimestamp(1).getTime();
+ } catch (SQLException e) {
+ String msg = "Unable to determine current time: " + e.getMessage();
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+ }
+
+ protected enum DatabaseProduct { DERBY, MYSQL, POSTGRES, ORACLE, SQLSERVER}
+
+ /**
+ * Determine the database product type
+ * @param conn database connection
+ * @return database product type
+ * @throws MetaException if the type cannot be determined or is unknown
+ */
+ protected DatabaseProduct determineDatabaseProduct(Connection conn) throws MetaException {
+ if (dbProduct == null) {
+ try {
+ String s = conn.getMetaData().getDatabaseProductName();
+ if (s == null) {
+ String msg = "getDatabaseProductName returns null, can't determine database product";
+ LOG.error(msg);
+ throw new MetaException(msg);
+ } else if (s.equals("Apache Derby")) {
+ dbProduct = DatabaseProduct.DERBY;
+ } else if (s.equals("Microsoft SQL Server")) {
+ dbProduct = DatabaseProduct.SQLSERVER;
+ } else if (s.equals("MySQL")) {
+ dbProduct = DatabaseProduct.MYSQL;
+ } else if (s.equals("Oracle")) {
+ dbProduct = DatabaseProduct.ORACLE;
+ } else if (s.equals("PostgreSQL")) {
+ dbProduct = DatabaseProduct.POSTGRES;
+ } else {
+ String msg = "Unrecognized database product name <" + s + ">";
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+
+ } catch (SQLException e) {
+ String msg = "Unable to get database product name: " + e.getMessage();
+ LOG.error(msg);
+ throw new MetaException(msg);
+ }
+ }
+ return dbProduct;
+ }
+
private static class LockInfo {
long extLockId;
long intLockId;
@@ -947,7 +1032,7 @@ public class TxnHandler {
* Abort a group of txns
* @param dbConn An active connection
* @param txnids list of transactions to abort
- * @return
+ * @return Number of aborted transactions
* @throws SQLException
*/
private int abortTxns(Connection dbConn, List<Long> txnids) throws SQLException {
@@ -1070,7 +1155,7 @@ public class TxnHandler {
case SHARED_READ: lockChar = LOCK_SHARED; break;
case SHARED_WRITE: lockChar = LOCK_SEMI_SHARED; break;
}
- long now = System.currentTimeMillis();
+ long now = getDbTime(dbConn);
s = "insert into HIVE_LOCKS " +
" (hl_lock_ext_id, hl_lock_int_id, hl_txnid, hl_db, hl_table, " +
"hl_partition, hl_lock_state, hl_lock_type, hl_last_heartbeat, hl_user, hl_host)" +
@@ -1083,7 +1168,7 @@ public class TxnHandler {
LOG.debug("Going to execute update <" + s + ">");
stmt.executeUpdate(s);
}
- LockResponse rsp = checkLock(dbConn, extLockId, txnid, wait);
+ LockResponse rsp = checkLock(dbConn, extLockId, wait);
if (!wait && rsp.getState() != LockState.ACQUIRED) {
LOG.debug("Lock not acquired, going to rollback");
dbConn.rollback();
@@ -1100,14 +1185,12 @@ public class TxnHandler {
private LockResponse checkLock(Connection dbConn,
long extLockId,
- long txnid,
boolean alwaysCommit)
throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException, SQLException {
List<LockInfo> locksBeingChecked = getLockInfoFromLockId(dbConn, extLockId);
LockResponse response = new LockResponse();
response.setLockid(extLockId);
- long now = System.currentTimeMillis();
LOG.debug("Setting savepoint");
Savepoint save = dbConn.setSavepoint();
Statement stmt = dbConn.createStatement();
@@ -1288,8 +1371,8 @@ public class TxnHandler {
}
private void acquire(Connection dbConn, Statement stmt, long extLockId, long intLockId)
- throws SQLException, NoSuchLockException {
- long now = System.currentTimeMillis();
+ throws SQLException, NoSuchLockException, MetaException {
+ long now = getDbTime(dbConn);
String s = "update HIVE_LOCKS set hl_lock_state = '" + LOCK_ACQUIRED + "', " +
"hl_last_heartbeat = " + now + ", hl_acquired_at = " + now + " where hl_lock_ext_id = " +
extLockId + " and hl_lock_int_id = " + intLockId;
@@ -1308,11 +1391,11 @@ public class TxnHandler {
// Heartbeats on the lock table. This commits, so do not enter it with any state
private void heartbeatLock(Connection dbConn, long extLockId)
- throws NoSuchLockException, SQLException {
+ throws NoSuchLockException, SQLException, MetaException {
// If the lock id is 0, then there are no locks in this heartbeat
if (extLockId == 0) return;
Statement stmt = dbConn.createStatement();
- long now = System.currentTimeMillis();
+ long now = getDbTime(dbConn);
String s = "update HIVE_LOCKS set hl_last_heartbeat = " +
now + " where hl_lock_ext_id = " + extLockId;
@@ -1329,11 +1412,11 @@ public class TxnHandler {
// Heartbeats on the txn table. This commits, so do not enter it with any state
private void heartbeatTxn(Connection dbConn, long txnid)
- throws NoSuchTxnException, TxnAbortedException, SQLException {
+ throws NoSuchTxnException, TxnAbortedException, SQLException, MetaException {
// If the txnid is 0, then there are no transactions in this heartbeat
if (txnid == 0) return;
Statement stmt = dbConn.createStatement();
- long now = System.currentTimeMillis();
+ long now = getDbTime(dbConn);
// We need to check whether this transaction is valid and open
String s = "select txn_state from TXNS where txn_id = " +
txnid + " for update";
@@ -1400,8 +1483,8 @@ public class TxnHandler {
// Clean time out locks from the database. This does a commit,
// and thus should be done before any calls to heartbeat that will leave
// open transactions.
- private void timeOutLocks(Connection dbConn) throws SQLException {
- long now = System.currentTimeMillis();
+ private void timeOutLocks(Connection dbConn) throws SQLException, MetaException {
+ long now = getDbTime(dbConn);
Statement stmt = dbConn.createStatement();
// Remove any timed out locks from the table.
String s = "delete from HIVE_LOCKS where hl_last_heartbeat < " +
@@ -1410,14 +1493,13 @@ public class TxnHandler {
stmt.executeUpdate(s);
LOG.debug("Going to commit");
dbConn.commit();
- return;
}
// Abort timed out transactions. This calls abortTxn(), which does a commit,
// and thus should be done before any calls to heartbeat that will leave
// open transactions on the underlying database.
- private void timeOutTxns(Connection dbConn) throws SQLException {
- long now = System.currentTimeMillis();
+ private void timeOutTxns(Connection dbConn) throws SQLException, MetaException {
+ long now = getDbTime(dbConn);
Statement stmt = dbConn.createStatement();
// Abort any timed out locks from the table.
String s = "select txn_id from TXNS where txn_state = '" + TXN_OPEN +