You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ga...@apache.org on 2017/10/06 16:57:41 UTC

[01/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Repository: hive
Updated Branches:
  refs/heads/master c1f3d9a48 -> f4a12a561


http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
new file mode 100644
index 0000000..00ee820
--- /dev/null
+++ b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.util.BitSet;
+
+/**
+ * Tests for {@link ValidReadTxnList}
+ */
+public class TestValidReadTxnList {
+
+  @Test
+  public void noExceptions() throws Exception {
+    ValidTxnList txnList = new ValidReadTxnList(new long[0], new BitSet(), 1, Long.MAX_VALUE);
+    String str = txnList.writeToString();
+    Assert.assertEquals("1:" + Long.MAX_VALUE + "::", str);
+    ValidTxnList newList = new ValidReadTxnList();
+    newList.readFromString(str);
+    Assert.assertTrue(newList.isTxnValid(1));
+    Assert.assertFalse(newList.isTxnValid(2));
+  }
+
+  @Test
+  public void exceptions() throws Exception {
+    ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, new BitSet(), 5, 4L);
+    String str = txnList.writeToString();
+    Assert.assertEquals("5:4:2,4:", str);
+    ValidTxnList newList = new ValidReadTxnList();
+    newList.readFromString(str);
+    Assert.assertTrue(newList.isTxnValid(1));
+    Assert.assertFalse(newList.isTxnValid(2));
+    Assert.assertTrue(newList.isTxnValid(3));
+    Assert.assertFalse(newList.isTxnValid(4));
+    Assert.assertTrue(newList.isTxnValid(5));
+    Assert.assertFalse(newList.isTxnValid(6));
+  }
+
+  @Test
+  public void longEnoughToCompress() throws Exception {
+    long[] exceptions = new long[1000];
+    for (int i = 0; i < 1000; i++) exceptions[i] = i + 100;
+    ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900);
+    String str = txnList.writeToString();
+    ValidTxnList newList = new ValidReadTxnList();
+    newList.readFromString(str);
+    for (int i = 0; i < 100; i++) Assert.assertTrue(newList.isTxnValid(i));
+    for (int i = 100; i < 1100; i++) Assert.assertFalse(newList.isTxnValid(i));
+    for (int i = 1100; i < 2001; i++) Assert.assertTrue(newList.isTxnValid(i));
+    Assert.assertFalse(newList.isTxnValid(2001));
+  }
+
+  @Test
+  public void readWriteConfig() throws Exception {
+    long[] exceptions = new long[1000];
+    for (int i = 0; i < 1000; i++) exceptions[i] = i + 100;
+    ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900);
+    String str = txnList.writeToString();
+    Configuration conf = new Configuration();
+    conf.set(ValidTxnList.VALID_TXNS_KEY, str);
+    File tmpFile = File.createTempFile("TestValidTxnImpl", "readWriteConfig");
+    DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile));
+    conf.write(out);
+    out.close();
+    DataInputStream in = new DataInputStream(new FileInputStream(tmpFile));
+    Configuration newConf = new Configuration();
+    newConf.readFields(in);
+    Assert.assertEquals(str, newConf.get(ValidTxnList.VALID_TXNS_KEY));
+  }
+
+  @Test
+  public void testAbortedTxn() throws Exception {
+    long[] exceptions = {2L, 4L, 6L, 8L, 10L};
+    BitSet bitSet = new BitSet(exceptions.length);
+    bitSet.set(0);  // mark txn "2L" aborted
+    bitSet.set(3);  // mark txn "8L" aborted
+    ValidTxnList txnList = new ValidReadTxnList(exceptions, bitSet, 11, 4L);
+    String str = txnList.writeToString();
+    Assert.assertEquals("11:4:4,6,10:2,8", str);
+    Assert.assertTrue(txnList.isTxnAborted(2L));
+    Assert.assertFalse(txnList.isTxnAborted(4L));
+    Assert.assertFalse(txnList.isTxnAborted(6L));
+    Assert.assertTrue(txnList.isTxnAborted(8L));
+    Assert.assertFalse(txnList.isTxnAborted(10L));
+  }
+}


[03/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
new file mode 100644
index 0000000..96a7f56
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
@@ -0,0 +1,420 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hive.common.classification.RetrySemantics;
+import org.apache.hadoop.hive.metastore.api.*;
+
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * A handler to answer transaction related calls that come into the metastore
+ * server.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface TxnStore extends Configurable {
+
+  enum MUTEX_KEY {Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock,
+    WriteSetCleaner, CompactionScheduler}
+  // Compactor states (Should really be enum)
+  String INITIATED_RESPONSE = "initiated";
+  String WORKING_RESPONSE = "working";
+  String CLEANING_RESPONSE = "ready for cleaning";
+  String FAILED_RESPONSE = "failed";
+  String SUCCEEDED_RESPONSE = "succeeded";
+  String ATTEMPTED_RESPONSE = "attempted";
+
+  int TIMED_OUT_TXN_ABORT_BATCH_SIZE = 50000;
+
+  /**
+   * Get information about open transactions.  This gives extensive information about the
+   * transactions rather than just the list of transactions.  This should be used when the need
+   * is to see information about the transactions (e.g. show transactions).
+   * @return information about open transactions
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException;
+
+  /**
+   * Get list of valid transactions.  This gives just the list of transactions that are open.
+   * @return list of open transactions, as well as a high water mark.
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  GetOpenTxnsResponse getOpenTxns() throws MetaException;
+
+  /**
+   * Get the count for open transactions.
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  void countOpenTxns() throws MetaException;
+
+  /**
+   * Open a set of transactions
+   * @param rqst request to open transactions
+   * @return information on opened transactions
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException;
+
+  /**
+   * Abort (rollback) a transaction.
+   * @param rqst info on transaction to abort
+   * @throws NoSuchTxnException
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException, TxnAbortedException;
+
+  /**
+   * Abort (rollback) a list of transactions in one request.
+   * @param rqst info on transactions to abort
+   * @throws NoSuchTxnException
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaException;
+
+  /**
+   * Commit a transaction
+   * @param rqst info on transaction to commit
+   * @throws NoSuchTxnException
+   * @throws TxnAbortedException
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  void commitTxn(CommitTxnRequest rqst)
+    throws NoSuchTxnException, TxnAbortedException,  MetaException;
+
+  /**
+   * Obtain a lock.
+   * @param rqst information on the lock to obtain.  If the requester is part of a transaction
+   *             the txn information must be included in the lock request.
+   * @return info on the lock, including whether it was obtained.
+   * @throws NoSuchTxnException
+   * @throws TxnAbortedException
+   * @throws MetaException
+   */
+  @RetrySemantics.CannotRetry
+  LockResponse lock(LockRequest rqst)
+    throws NoSuchTxnException, TxnAbortedException, MetaException;
+
+  /**
+   * Check whether a lock has been obtained.  This is used after {@link #lock} returned a wait
+   * state.
+   * @param rqst info on the lock to check
+   * @return info on the state of the lock
+   * @throws NoSuchTxnException
+   * @throws NoSuchLockException
+   * @throws TxnAbortedException
+   * @throws MetaException
+   */
+  @RetrySemantics.SafeToRetry
+  LockResponse checkLock(CheckLockRequest rqst)
+    throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException;
+
+  /**
+   * Unlock a lock.  It is not legal to call this if the caller is part of a txn.  In that case
+   * the txn should be committed or aborted instead.  (Note someday this will change since
+   * multi-statement transactions will allow unlocking in the transaction.)
+   * @param rqst lock to unlock
+   * @throws NoSuchLockException
+   * @throws TxnOpenException
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  void unlock(UnlockRequest rqst)
+    throws NoSuchLockException, TxnOpenException, MetaException;
+
+  /**
+   * Get information on current locks.
+   * @param rqst lock information to retrieve
+   * @return lock information.
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException;
+
+  /**
+   * Send a heartbeat for a lock or a transaction
+   * @param ids lock and/or txn id to heartbeat
+   * @throws NoSuchTxnException
+   * @throws NoSuchLockException
+   * @throws TxnAbortedException
+   * @throws MetaException
+   */
+  @RetrySemantics.SafeToRetry
+  void heartbeat(HeartbeatRequest ids)
+    throws NoSuchTxnException,  NoSuchLockException, TxnAbortedException, MetaException;
+
+  /**
+   * Heartbeat a group of transactions together
+   * @param rqst set of transactions to heartbat
+   * @return info on txns that were heartbeated
+   * @throws MetaException
+   */
+  @RetrySemantics.SafeToRetry
+  HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst)
+    throws MetaException;
+
+  /**
+   * Submit a compaction request into the queue.  This is called when a user manually requests a
+   * compaction.
+   * @param rqst information on what to compact
+   * @return id of the compaction that has been started or existing id if this resource is already scheduled
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  CompactionResponse compact(CompactionRequest rqst) throws MetaException;
+
+  /**
+   * Show list of current compactions
+   * @param rqst info on which compactions to show
+   * @return compaction information
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException;
+
+  /**
+   * Add information on a set of dynamic partitions that participated in a transaction.
+   * @param rqst dynamic partition info.
+   * @throws NoSuchTxnException
+   * @throws TxnAbortedException
+   * @throws MetaException
+   */
+  @RetrySemantics.SafeToRetry
+  void addDynamicPartitions(AddDynamicPartitions rqst)
+      throws NoSuchTxnException,  TxnAbortedException, MetaException;
+
+  /**
+   * Clean up corresponding records in metastore tables
+   * @param type Hive object type
+   * @param db database object
+   * @param table table object
+   * @param partitionIterator partition iterator
+   * @throws MetaException
+   */
+  @RetrySemantics.Idempotent
+  void cleanupRecords(HiveObjectType type, Database db, Table table,
+                             Iterator<Partition> partitionIterator) throws MetaException;
+
+  /**
+   * Timeout transactions and/or locks.  This should only be called by the compactor.
+   */
+  @RetrySemantics.Idempotent
+  void performTimeOuts();
+
+  /**
+   * This will look through the completed_txn_components table and look for partitions or tables
+   * that may be ready for compaction.  Also, look through txns and txn_components tables for
+   * aborted transactions that we should add to the list.
+   * @param maxAborted Maximum number of aborted queries to allow before marking this as a
+   *                   potential compaction.
+   * @return list of CompactionInfo structs.  These will not have id, type,
+   * or runAs set since these are only potential compactions not actual ones.
+   */
+  @RetrySemantics.ReadOnly
+  Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException;
+
+  /**
+   * Sets the user to run as.  This is for the case
+   * where the request was generated by the user and so the worker must set this value later.
+   * @param cq_id id of this entry in the queue
+   * @param user user to run the jobs as
+   */
+  @RetrySemantics.Idempotent
+  void setRunAs(long cq_id, String user) throws MetaException;
+
+  /**
+   * This will grab the next compaction request off of
+   * the queue, and assign it to the worker.
+   * @param workerId id of the worker calling this, will be recorded in the db
+   * @return an info element for this compaction request, or null if there is no work to do now.
+   */
+  @RetrySemantics.ReadOnly
+  CompactionInfo findNextToCompact(String workerId) throws MetaException;
+
+  /**
+   * This will mark an entry in the queue as compacted
+   * and put it in the ready to clean state.
+   * @param info info on the compaction entry to mark as compacted.
+   */
+  @RetrySemantics.SafeToRetry
+  void markCompacted(CompactionInfo info) throws MetaException;
+
+  /**
+   * Find entries in the queue that are ready to
+   * be cleaned.
+   * @return information on the entry in the queue.
+   */
+  @RetrySemantics.ReadOnly
+  List<CompactionInfo> findReadyToClean() throws MetaException;
+
+  /**
+   * This will remove an entry from the queue after
+   * it has been compacted.
+   * 
+   * @param info info on the compaction entry to remove
+   */
+  @RetrySemantics.CannotRetry
+  void markCleaned(CompactionInfo info) throws MetaException;
+
+  /**
+   * Mark a compaction entry as failed.  This will move it to the compaction history queue with a
+   * failed status.  It will NOT clean up aborted transactions in the table/partition associated
+   * with this compaction.
+   * @param info information on the compaction that failed.
+   * @throws MetaException
+   */
+  @RetrySemantics.CannotRetry
+  void markFailed(CompactionInfo info) throws MetaException;
+
+  /**
+   * Clean up aborted transactions from txns that have no components in txn_components.  The reson such
+   * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
+   * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
+   */
+  @RetrySemantics.SafeToRetry
+  void cleanEmptyAbortedTxns() throws MetaException;
+
+  /**
+   * This will take all entries assigned to workers
+   * on a host return them to INITIATED state.  The initiator should use this at start up to
+   * clean entries from any workers that were in the middle of compacting when the metastore
+   * shutdown.  It does not reset entries from worker threads on other hosts as those may still
+   * be working.
+   * @param hostname Name of this host.  It is assumed this prefixes the thread's worker id,
+   *                 so that like hostname% will match the worker id.
+   */
+  @RetrySemantics.Idempotent
+  void revokeFromLocalWorkers(String hostname) throws MetaException;
+
+  /**
+   * This call will return all compaction queue
+   * entries assigned to a worker but over the timeout back to the initiated state.
+   * This should be called by the initiator on start up and occasionally when running to clean up
+   * after dead threads.  At start up {@link #revokeFromLocalWorkers(String)} should be called
+   * first.
+   * @param timeout number of milliseconds since start time that should elapse before a worker is
+   *                declared dead.
+   */
+  @RetrySemantics.Idempotent
+  void revokeTimedoutWorkers(long timeout) throws MetaException;
+
+  /**
+   * Queries metastore DB directly to find columns in the table which have statistics information.
+   * If {@code ci} includes partition info then per partition stats info is examined, otherwise
+   * table level stats are examined.
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  List<String> findColumnsWithStats(CompactionInfo ci) throws MetaException;
+
+  /**
+   * Record the highest txn id that the {@code ci} compaction job will pay attention to.
+   */
+  @RetrySemantics.Idempotent
+  void setCompactionHighestTxnId(CompactionInfo ci, long highestTxnId) throws MetaException;
+
+  /**
+   * For any given compactable entity (partition, table if not partitioned) the history of compactions
+   * may look like "sssfffaaasffss", for example.  The idea is to retain the tail (most recent) of the
+   * history such that a configurable number of each type of state is present.  Any other entries
+   * can be purged.  This scheme has advantage of always retaining the last failure/success even if
+   * it's not recent.
+   * @throws MetaException
+   */
+  @RetrySemantics.SafeToRetry
+  void purgeCompactionHistory() throws MetaException;
+
+  /**
+   * WriteSet tracking is used to ensure proper transaction isolation.  This method deletes the 
+   * transaction metadata once it becomes unnecessary.  
+   */
+  @RetrySemantics.SafeToRetry
+  void performWriteSetGC();
+
+  /**
+   * Determine if there are enough consecutive failures compacting a table or partition that no
+   * new automatic compactions should be scheduled.  User initiated compactions do not do this
+   * check.
+   * @param ci  Table or partition to check.
+   * @return true if it is ok to compact, false if there have been too many failures.
+   * @throws MetaException
+   */
+  @RetrySemantics.ReadOnly
+  boolean checkFailedCompactions(CompactionInfo ci) throws MetaException;
+
+  @VisibleForTesting
+  int numLocksInLockTable() throws SQLException, MetaException;
+
+  @VisibleForTesting
+  long setTimeout(long milliseconds);
+
+  @RetrySemantics.Idempotent
+  MutexAPI getMutexAPI();
+
+  /**
+   * This is primarily designed to provide coarse grained mutex support to operations running
+   * inside the Metastore (of which there could be several instances).  The initial goal is to 
+   * ensure that various sub-processes of the Compactor don't step on each other.
+   * 
+   * In RDMBS world each {@code LockHandle} uses a java.sql.Connection so use it sparingly.
+   */
+  interface MutexAPI {
+    /**
+     * The {@code key} is name of the lock. Will acquire and exclusive lock or block.  It retuns
+     * a handle which must be used to release the lock.  Each invocation returns a new handle.
+     */
+    LockHandle acquireLock(String key) throws MetaException;
+
+    /**
+     * Same as {@link #acquireLock(String)} but takes an already existing handle as input.  This 
+     * will associate the lock on {@code key} with the same handle.  All locks associated with
+     * the same handle will be released together.
+     * @param handle not NULL
+     */
+    void acquireLock(String key, LockHandle handle) throws MetaException;
+    interface LockHandle {
+      /**
+       * Releases all locks associated with this handle.
+       */
+      void releaseLocks();
+    }
+  }
+
+  /**
+   * Once a {@link java.util.concurrent.ThreadPoolExecutor} Worker submits a job to the cluster,
+   * it calls this to update the metadata.
+   * @param id {@link CompactionInfo#id}
+   */
+  @RetrySemantics.Idempotent
+  void setHadoopJobId(String hadoopJobId, long id);
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
new file mode 100644
index 0000000..7579ae8
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.ValidCompactorTxnList;
+import org.apache.hadoop.hive.common.ValidReadTxnList;
+import org.apache.hadoop.hive.common.ValidTxnList;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.hadoop.hive.metastore.api.TxnInfo;
+import org.apache.hadoop.hive.metastore.api.TxnState;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.List;
+import java.util.Map;
+
+public class TxnUtils {
+  private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class);
+
+  /**
+   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
+   * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
+   * read the files, and thus treats both open and aborted transactions as invalid.
+   * @param txns txn list from the metastore
+   * @param currentTxn Current transaction that the user has open.  If this is greater than 0 it
+   *                   will be removed from the exceptions list so that the user sees his own
+   *                   transaction as valid.
+   * @return a valid txn list.
+   */
+  public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
+    /*todo: should highWater be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
+     * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
+     * doesn't make sense for Snapshot Isolation.  Of course for Read Committed, the list should
+     * inlude the latest committed set.*/
+    long highWater = txns.getTxn_high_water_mark();
+    List<Long> open = txns.getOpen_txns();
+    BitSet abortedBits = BitSet.valueOf(txns.getAbortedBits());
+    long[] exceptions = new long[open.size() - (currentTxn > 0 ? 1 : 0)];
+    int i = 0;
+    for(long txn: open) {
+      if (currentTxn > 0 && currentTxn == txn) continue;
+      exceptions[i++] = txn;
+    }
+    if(txns.isSetMin_open_txn()) {
+      return new ValidReadTxnList(exceptions, abortedBits, highWater, txns.getMin_open_txn());
+    }
+    else {
+      return new ValidReadTxnList(exceptions, abortedBits, highWater);
+    }
+  }
+
+  /**
+   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse} to a
+   * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
+   * compact the files, and thus treats only open transactions as invalid.  Additionally any
+   * txnId &gt; highestOpenTxnId is also invalid.  This is to avoid creating something like
+   * delta_17_120 where txnId 80, for example, is still open.
+   * @param txns txn list from the metastore
+   * @return a valid txn list.
+   */
+  public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
+    //highWater is the last txn id that has been allocated
+    long highWater = txns.getTxn_high_water_mark();
+    long minOpenTxn = Long.MAX_VALUE;
+    long[] exceptions = new long[txns.getOpen_txnsSize()];
+    int i = 0;
+    for (TxnInfo txn : txns.getOpen_txns()) {
+      if (txn.getState() == TxnState.OPEN) {
+        minOpenTxn = Math.min(minOpenTxn, txn.getId());
+      }
+      else {
+        //only need aborted since we don't consider anything above minOpenTxn
+        exceptions[i++] = txn.getId();
+      }
+    }
+    if(i < exceptions.length) {
+      exceptions = Arrays.copyOf(exceptions, i);
+    }
+    highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
+    BitSet bitSet = new BitSet(exceptions.length);
+    bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted
+    if(minOpenTxn == Long.MAX_VALUE) {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+    }
+    else {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
+    }
+  }
+
+  /**
+   * Get an instance of the TxnStore that is appropriate for this store
+   * @param conf configuration
+   * @return txn store
+   */
+  public static TxnStore getTxnStore(Configuration conf) {
+    String className = MetastoreConf.getVar(conf, ConfVars.TXN_STORE_IMPL);
+    try {
+      TxnStore handler = JavaUtils.getClass(className, TxnStore.class).newInstance();
+      handler.setConf(conf);
+      return handler;
+    } catch (Exception e) {
+      LOG.error("Unable to instantiate raw store directly in fastpath mode", e);
+      throw new RuntimeException(e);
+    }
+  }
+
+  /** Checks if a table is a valid ACID table.
+   * Note, users are responsible for using the correct TxnManager. We do not look at
+   * SessionState.get().getTxnMgr().supportsAcid() here
+   * @param table table
+   * @return true if table is a legit ACID table, false otherwise
+   */
+  public static boolean isAcidTable(Table table) {
+    if (table == null) {
+      return false;
+    }
+    Map<String, String> parameters = table.getParameters();
+    String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
+    return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
+  }
+
+  /**
+   * Build a query (or queries if one query is too big) with specified "prefix" and "suffix",
+   * while populating the IN list into multiple OR clauses, e.g. id in (1,2,3) OR id in (4,5,6)
+   * For NOT IN case, NOT IN list is broken into multiple AND clauses.
+   * @param queries array of complete query strings
+   * @param prefix part of the query that comes before IN list
+   * @param suffix part of the query that comes after IN list
+   * @param inList the list containing IN list values
+   * @param inColumn column name of IN list operator
+   * @param addParens add a pair of parenthesis outside the IN lists
+   *                  e.g. ( id in (1,2,3) OR id in (4,5,6) )
+   * @param notIn clause to be broken up is NOT IN
+   */
+  public static void buildQueryWithINClause(Configuration conf, List<String> queries, StringBuilder prefix,
+                                            StringBuilder suffix, List<Long> inList,
+                                            String inColumn, boolean addParens, boolean notIn) {
+    if (inList == null || inList.size() == 0) {
+      throw new IllegalArgumentException("The IN list is empty!");
+    }
+    int batchSize = MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE);
+    int numWholeBatches = inList.size() / batchSize;
+    StringBuilder buf = new StringBuilder();
+    buf.append(prefix);
+    if (addParens) {
+      buf.append("(");
+    }
+    buf.append(inColumn);
+    if (notIn) {
+      buf.append(" not in (");
+    } else {
+      buf.append(" in (");
+    }
+
+    for (int i = 0; i <= numWholeBatches; i++) {
+      if (i * batchSize == inList.size()) {
+        // At this point we just realized we don't need another query
+        break;
+      }
+
+      if (needNewQuery(conf, buf)) {
+        // Wrap up current query string
+        if (addParens) {
+          buf.append(")");
+        }
+        buf.append(suffix);
+        queries.add(buf.toString());
+
+        // Prepare a new query string
+        buf.setLength(0);
+      }
+
+      if (i > 0) {
+        if (notIn) {
+          if (buf.length() == 0) {
+            buf.append(prefix);
+            if (addParens) {
+              buf.append("(");
+            }
+          } else {
+            buf.append(" and ");
+          }
+          buf.append(inColumn);
+          buf.append(" not in (");
+        } else {
+          if (buf.length() == 0) {
+            buf.append(prefix);
+            if (addParens) {
+              buf.append("(");
+            }
+          } else {
+            buf.append(" or ");
+          }
+          buf.append(inColumn);
+          buf.append(" in (");
+        }
+      }
+
+      for (int j = i * batchSize; j < (i + 1) * batchSize && j < inList.size(); j++) {
+        buf.append(inList.get(j)).append(",");
+      }
+      buf.setCharAt(buf.length() - 1, ')');
+    }
+
+    if (addParens) {
+      buf.append(")");
+    }
+    buf.append(suffix);
+    queries.add(buf.toString());
+  }
+
+  /** Estimate if the size of a string will exceed certain limit */
+  private static boolean needNewQuery(Configuration conf, StringBuilder sb) {
+    int queryMemoryLimit = MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH);
+    // http://www.javamex.com/tutorials/memory/string_memory_usage.shtml
+    long sizeInBytes = 8 * (((sb.length() * 2) + 45) / 8);
+    return sizeInBytes / 1024 > queryMemoryLimit;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
index 81f8a85..40f7393 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/JavaUtils.java
@@ -17,7 +17,16 @@
  */
 package org.apache.hadoop.hive.metastore.utils;
 
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
 public class JavaUtils {
+  public static final Logger LOG = LoggerFactory.getLogger(JavaUtils.class);
+
   /**
    * Standard way of getting classloader in Hive code (outside of Hadoop).
    *
@@ -34,4 +43,41 @@ public class JavaUtils {
     }
     return classLoader;
   }
+
+  @SuppressWarnings(value = "unchecked")
+  public static <T> Class<? extends T> getClass(String className, Class<T> clazz)
+      throws MetaException {
+    try {
+      return (Class<? extends T>) Class.forName(className, true, getClassLoader());
+    } catch (ClassNotFoundException e) {
+      throw new MetaException(className + " class not found");
+    }
+  }
+
+  /**
+   * @return name of current host
+   */
+  public static String hostname() {
+    try {
+      return InetAddress.getLocalHost().getHostName();
+    } catch (UnknownHostException e) {
+      LOG.error("Unable to resolve my host name " + e.getMessage());
+      throw new RuntimeException(e);
+    }
+  }
+
+  /**
+   * Utility method for ACID to normalize logging info.  Matches
+   * org.apache.hadoop.hive.metastore.api.LockRequest#toString
+   */
+  public static String lockIdToString(long extLockId) {
+    return "lockid:" + extLockId;
+  }
+  /**
+   * Utility method for ACID to normalize logging info.  Matches
+   * org.apache.hadoop.hive.metastore.api.LockResponse#toString
+   */
+  public static String txnIdToString(long txnId) {
+    return "txnid:" + txnId;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StringableMap.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StringableMap.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StringableMap.java
new file mode 100644
index 0000000..b3f1749
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/utils/StringableMap.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore.utils;
+
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * A utility class that can convert a HashMap of Properties into a colon separated string,
+ * and can take the same format of string and convert it to a HashMap of Properties.
+ */
+public class StringableMap extends HashMap<String, String> {
+
+  public StringableMap(String s) {
+    String[] parts = s.split(":", 2);
+    // read that many chars
+    int numElements = Integer.parseInt(parts[0]);
+    s = parts[1];
+    for (int i = 0; i < numElements; i++) {
+      parts = s.split(":", 2);
+      int len = Integer.parseInt(parts[0]);
+      String key = null;
+      if (len > 0) key = parts[1].substring(0, len);
+      parts = parts[1].substring(len).split(":", 2);
+      len = Integer.parseInt(parts[0]);
+      String value = null;
+      if (len > 0) value = parts[1].substring(0, len);
+      s = parts[1].substring(len);
+      put(key, value);
+    }
+  }
+
+  public StringableMap(Map<String, String> m) {
+    super(m);
+  }
+
+  @Override
+  public String toString() {
+    StringBuilder buf = new StringBuilder();
+    buf.append(size());
+    buf.append(':');
+    if (size() > 0) {
+      for (Map.Entry<String, String> entry : entrySet()) {
+        int length = (entry.getKey() == null) ? 0 : entry.getKey().length();
+        buf.append(entry.getKey() == null ? 0 : length);
+        buf.append(':');
+        if (length > 0) buf.append(entry.getKey());
+        length = (entry.getValue() == null) ? 0 : entry.getValue().length();
+        buf.append(length);
+        buf.append(':');
+        if (length > 0) buf.append(entry.getValue());
+      }
+    }
+    return buf.toString();
+  }
+
+  public Properties toProperties() {
+    Properties props = new Properties();
+    props.putAll(this);
+    return props;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
index 082b1b0..9aff091 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
@@ -19,6 +19,9 @@ package org.apache.hadoop.hive.metastore.conf;
 
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
 import org.apache.hadoop.conf.Configuration;
+import org.hamcrest.CoreMatchers;
+import org.hamcrest.core.StringContains;
+import org.hamcrest.core.StringEndsWith;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
@@ -411,4 +414,18 @@ public class TestMetastoreConf {
     Assert.assertEquals("abc", MetastoreConf.get(conf, "a.random.key"));
   }
 
+  @Test
+  public void dumpConfig() throws IOException {
+    createConfFile("metastore-site.xml", true, "METASTORE_HOME", instaMap(
+        "test.long", "23"
+    ));
+    conf = MetastoreConf.newMetastoreConf();
+    String dump = MetastoreConf.dumpConfig(conf);
+    Assert.assertThat(dump, new StringContains("Used metastore-site file: file:/"));
+    Assert.assertThat(dump, new StringContains("Key: <test.long> old hive key: <hive.test.long>  value: <23>"));
+    Assert.assertThat(dump, new StringContains("Key: <test.str> old hive key: <hive.test.str>  value: <defaultval>"));
+    Assert.assertThat(dump, new StringEndsWith("Finished MetastoreConf object.\n"));
+    // Make sure the hidden keys didn't get published
+    Assert.assertThat(dump, CoreMatchers.not(new StringContains(ConfVars.PWD.varname)));
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
new file mode 100644
index 0000000..3f21ed1
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.jolbox.bonecp.BoneCPDataSource;
+import com.zaxxer.hikari.HikariDataSource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+
+public class TestDataSourceProviderFactory {
+
+  private Configuration conf;
+
+  @Before
+  public void init() {
+    conf = MetastoreConf.newMetastoreConf();
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_USER_NAME, "dummyUser");
+    MetastoreConf.setVar(conf, ConfVars.PWD, "dummyPass");
+  }
+
+  @Test
+  public void testNoDataSourceCreatedWithoutProps() throws SQLException {
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNull(dsp);
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
+
+    dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNull(dsp);
+  }
+
+  @Test
+  public void testCreateBoneCpDataSource() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
+    conf.set(BoneCPDataSourceProvider.BONECP + ".firstProp", "value");
+    conf.set(BoneCPDataSourceProvider.BONECP + ".secondProp", "value");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof BoneCPDataSource);
+  }
+
+  @Test
+  public void testSetBoneCpStringProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
+    conf.set(BoneCPDataSourceProvider.BONECP + ".initSQL", "select 1 from dual");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof BoneCPDataSource);
+    Assert.assertEquals("select 1 from dual", ((BoneCPDataSource)ds).getInitSQL());
+  }
+
+  @Test
+  public void testSetBoneCpNumberProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
+    conf.set(BoneCPDataSourceProvider.BONECP + ".acquireRetryDelayInMs", "599");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof BoneCPDataSource);
+    Assert.assertEquals(599L, ((BoneCPDataSource)ds).getAcquireRetryDelayInMs());
+  }
+
+  @Test
+  public void testSetBoneCpBooleanProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
+    conf.set(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof BoneCPDataSource);
+    Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX());
+  }
+
+  @Test
+  public void testCreateHikariCpDataSource() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+    // This is needed to prevent the HikariDataSource from trying to connect to the DB
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof HikariDataSource);
+  }
+
+  @Test
+  public void testSetHikariCpStringProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".connectionInitSql", "select 1 from dual");
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof HikariDataSource);
+    Assert.assertEquals("select 1 from dual", ((HikariDataSource)ds).getConnectionInitSql());
+  }
+
+  @Test
+  public void testSetHikariCpNumberProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".idleTimeout", "59999");
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof HikariDataSource);
+    Assert.assertEquals(59999L, ((HikariDataSource)ds).getIdleTimeout());
+  }
+
+  @Test
+  public void testSetHikariCpBooleanProperty() throws SQLException {
+
+    MetastoreConf.setVar(conf, ConfVars.CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".allowPoolSuspension", "false");
+    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
+
+    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
+    Assert.assertNotNull(dsp);
+
+    DataSource ds = dsp.create(conf);
+    Assert.assertTrue(ds instanceof HikariDataSource);
+    Assert.assertEquals(false, ((HikariDataSource)ds).isAllowPoolSuspension());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
new file mode 100644
index 0000000..d8640b5
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.junit.Test;
+
+import static junit.framework.Assert.assertNotNull;
+import static junit.framework.Assert.assertTrue;
+
+public class TestTxnHandlerNegative {
+  static final private Logger LOG = LoggerFactory.getLogger(TestTxnHandlerNegative.class);
+
+  /**
+   * this intentionally sets a bad URL for connection to test error handling logic
+   * in TxnHandler
+   * @throws Exception
+   */
+  @Test
+  public void testBadConnection() throws Exception {
+    Configuration conf = MetastoreConf.newMetastoreConf();
+    MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CONNECTURLKEY, "blah");
+    RuntimeException e = null;
+    try {
+      TxnUtils.getTxnStore(conf);
+    }
+    catch(RuntimeException ex) {
+      LOG.info("Expected error: " + ex.getMessage(), ex);
+      e = ex;
+    }
+    assertNotNull(e);
+    assertTrue(
+        e.getMessage().contains("No suitable driver found for blah")
+        || e.getMessage().contains("Failed to get driver instance for jdbcUrl=blah")
+    );
+  }
+}


[06/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
index 14ff58e..e46e65b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager.java
@@ -87,7 +87,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -102,7 +102,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -121,7 +121,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(3,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -140,7 +140,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(4,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -155,7 +155,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -171,7 +171,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -307,7 +307,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(4,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -322,7 +322,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -337,7 +337,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -352,7 +352,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.rollbackTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -367,7 +367,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.rollbackTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -382,7 +382,7 @@ public class TestDbTxnManager {
     List<HiveLock> locks = ctx.getHiveLocks();
     Assert.assertEquals(1, locks.size());
     Assert.assertEquals(1,
-        TxnDbUtil.countLockComponents(((DbLockManager.DbHiveLock) locks.get(0)).lockId));
+        TxnDbUtil.countLockComponents(conf, ((DbLockManager.DbHiveLock) locks.get(0)).lockId));
     txnMgr.commitTxn();
     locks = txnMgr.getLockManager().getLocks(false, false);
     Assert.assertEquals(0, locks.size());
@@ -481,7 +481,7 @@ public class TestDbTxnManager {
 
   @Before
   public void setUp() throws Exception {
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(conf);
     txnMgr = TxnManagerFactory.getTxnManagerFactory().getTxnManager(conf);
     txnMgr.getLockManager();//init lock manager
     Assert.assertTrue(txnMgr instanceof DbTxnManager);
@@ -497,7 +497,7 @@ public class TestDbTxnManager {
   public void tearDown() throws Exception {
     if(houseKeeperService != null) houseKeeperService.stop();
     if (txnMgr != null) txnMgr.closeTxnManager();
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
   }
 
   private static class MockQueryPlan extends QueryPlan {

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
index 6d1cdcb..e9833cb 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/lockmgr/TestDbTxnManager2.java
@@ -104,8 +104,8 @@ public class TestDbTxnManager2 {
     ctx = new Context(conf);
     driver = new Driver(conf);
     driver.init();
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(conf);
+    TxnDbUtil.prepDb(conf);
     SessionState ss = SessionState.get();
     ss.initTxnMgr(conf);
     txnMgr = ss.getTxnMgr();
@@ -445,9 +445,9 @@ public class TestDbTxnManager2 {
     checkCmdOnDriver(cpr);
     cpr = driver.run("insert into temp.T13p partition (ds='tomorrow', hour='2') values (8, 8)");
     checkCmdOnDriver(cpr);
-    int count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')");
+    int count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11')");
     Assert.assertEquals(4, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t12p', 't13p')");
     Assert.assertEquals(4, count);
 
     // Fail some inserts, so that we have records in TXN_COMPONENTS
@@ -460,132 +460,132 @@ public class TestDbTxnManager2 {
     checkCmdOnDriver(cpr);
     cpr = driver.run("insert into temp.T13p partition (ds='today', hour='1') values (12, 12)");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(4, count);
     conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEROLLBACKTXN, false);
 
     // Drop a table/partition; corresponding records in TXN_COMPONENTS and COMPLETED_TXN_COMPONENTS should disappear
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
     Assert.assertEquals(1, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
     Assert.assertEquals(2, count);
     cpr = driver.run("drop table temp.T10");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t10'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t10'");
     Assert.assertEquals(0, count);
 
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
     Assert.assertEquals(1, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
     Assert.assertEquals(1, count);
     cpr = driver.run("alter table temp.T12p drop partition (ds='today', hour='1')");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE='t12p' and TC_PARTITION='ds=today/hour=1'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE='t12p' and CTC_PARTITION='ds=today/hour=1'");
     Assert.assertEquals(0, count);
 
     // Successfully perform compaction on a table/partition, so that we have successful records in COMPLETED_COMPACTIONS
     cpr = driver.run("alter table temp.T11 compact 'minor'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='i'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='r' and CQ_TYPE='i'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='s' and CC_TYPE='i'");
     Assert.assertEquals(1, count);
 
     cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'minor'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='i'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='r' and CQ_TYPE='i'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runCleaner(conf);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='s' and CC_TYPE='i'");
     Assert.assertEquals(1, count);
 
     // Fail compaction, so that we have failed records in COMPLETED_COMPACTIONS
     conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, true);
     cpr = driver.run("alter table temp.T11 compact 'major'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11' and CC_STATE='f' and CC_TYPE='a'");
     Assert.assertEquals(1, count);
 
     cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(1, count);
     org.apache.hadoop.hive.ql.TestTxnCommands2.runWorker(conf); // will fail
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p' and CC_STATE='f' and CC_TYPE='a'");
     Assert.assertEquals(1, count);
     conf.setBoolVar(HiveConf.ConfVars.HIVETESTMODEFAILCOMPACTION, false);
 
     // Put 2 records into COMPACTION_QUEUE and do nothing
     cpr = driver.run("alter table temp.T11 compact 'major'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(1, count);
     cpr = driver.run("alter table temp.T12p partition (ds='tomorrow', hour='2') compact 'major'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p' and CQ_PARTITION='ds=tomorrow/hour=2' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(1, count);
 
     // Drop a table/partition, corresponding records in COMPACTION_QUEUE and COMPLETED_COMPACTIONS should disappear
     cpr = driver.run("drop table temp.T11");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t11'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t11'");
     Assert.assertEquals(0, count);
 
     cpr = driver.run("alter table temp.T12p drop partition (ds='tomorrow', hour='2')");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t12p'");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE='t12p'");
     Assert.assertEquals(0, count);
 
     // Put 1 record into COMPACTION_QUEUE and do nothing
     cpr = driver.run("alter table temp.T13p partition (ds='today', hour='1') compact 'major'");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE='t13p' and CQ_STATE='i' and CQ_TYPE='a'");
     Assert.assertEquals(1, count);
 
     // Drop database, everything in all 4 meta tables should disappear
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(1, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(2, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(1, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(0, count);
     cpr = driver.run("drop database if exists temp cascade");
     checkCmdOnDriver(cpr);
-    count = TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where TC_DATABASE='temp' and TC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where CTC_DATABASE='temp' and CTC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPACTION_QUEUE where CQ_DATABASE='temp' and CQ_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(0, count);
-    count = TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
+    count = TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_COMPACTIONS where CC_DATABASE='temp' and CC_TABLE in ('t10', 't11', 't12p', 't13p')");
     Assert.assertEquals(0, count);
   }
 
@@ -959,7 +959,7 @@ public class TestDbTxnManager2 {
   @Test
   public void testWriteSetTracking4() throws Exception {
     dropTable(new String[] {"TAB_PART", "TAB2"});
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " +
       "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
     checkCmdOnDriver(cpr);
@@ -986,14 +986,14 @@ public class TestDbTxnManager2 {
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
     checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
     //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     AddDynamicPartitions adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(),
       "default", "tab2", Collections.EMPTY_LIST);
     adp.setOperationType(DataOperationType.UPDATE);
     txnHandler.addDynamicPartitions(adp);
     txnMgr2.commitTxn();
     //Short Running updated nothing, so we expect 0 rows in WRITE_SET
-    Assert.assertEquals( 0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals( 0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
 
     txnMgr2.openTxn(ctx, "T3");
     checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 7 where p = 'two'"));//pretend this partition exists
@@ -1003,19 +1003,19 @@ public class TestDbTxnManager2 {
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB_PART", null, locks);
     checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);//since TAB2 is empty
     //update stmt has p=blah, thus nothing is actually update and we generate empty dyn part list
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     adp = new AddDynamicPartitions(txnMgr2.getCurrentTxnId(),
       "default", "tab2", Collections.singletonList("p=two"));
     adp.setOperationType(DataOperationType.UPDATE);
     txnHandler.addDynamicPartitions(adp);//simulate partition update
     txnMgr2.commitTxn();
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
 
     AcidWriteSetService houseKeeper = new AcidWriteSetService();
     TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
     //since T3 overlaps with Long Running (still open) GC does nothing
-    Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 1"));//no rows match
     txnMgr.acquireLocks(driver.getPlan(), ctx, "Long Running");
     //so generate empty Dyn Part call
@@ -1028,7 +1028,7 @@ public class TestDbTxnManager2 {
     locks = getLocks(txnMgr);
     Assert.assertEquals("Unexpected lock count", 0, locks.size());
     TestTxnCommands2.runHouseKeeperService(houseKeeper, conf);
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
   }
   /**
    * overlapping txns updating the same resource but 1st one rolls back; 2nd commits
@@ -1037,7 +1037,7 @@ public class TestDbTxnManager2 {
   @Test
   public void testWriteSetTracking5() throws Exception {
     dropTable(new String[] {"TAB_PART"});
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     CommandProcessorResponse cpr = driver.run("create table if not exists TAB_PART (a int, b int) " +
       "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
     checkCmdOnDriver(cpr);
@@ -1063,9 +1063,9 @@ public class TestDbTxnManager2 {
       Arrays.asList("p=blah"));
     adp.setOperationType(DataOperationType.UPDATE);
     txnHandler.addDynamicPartitions(adp);
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     txnMgr2.commitTxn();//since conflicting txn rolled back, commit succeeds
-    Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
   }
   /**
    * check that read query concurrent with txn works ok
@@ -1073,7 +1073,7 @@ public class TestDbTxnManager2 {
   @Test
   public void testWriteSetTracking6() throws Exception {
     dropTable(new String[] {"TAB2"});
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     CommandProcessorResponse cpr = driver.run("create table if not exists TAB2(a int, b int) clustered " +
       "by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
     checkCmdOnDriver(cpr);
@@ -1086,19 +1086,19 @@ public class TestDbTxnManager2 {
     swapTxnManager(txnMgr2);
     checkCmdOnDriver(driver.compileAndRespond("update TAB2 set b = 17 where a = 101"));
     txnMgr2.acquireLocks(driver.getPlan(), ctx, "Horton");
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     locks = getLocks(txnMgr);
     Assert.assertEquals("Unexpected lock count", 2, locks.size());
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks);
     checkLock(LockType.SHARED_WRITE, LockState.ACQUIRED, "default", "TAB2", null, locks);
     txnMgr2.commitTxn();//no conflict
-    Assert.assertEquals(1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     locks = getLocks(txnMgr);
     Assert.assertEquals("Unexpected lock count", 1, locks.size());
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "TAB2", null, locks);
     txnMgr.commitTxn();
     TestTxnCommands2.runHouseKeeperService(new AcidWriteSetService(), conf);
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
   }
 
   /**
@@ -1108,7 +1108,7 @@ public class TestDbTxnManager2 {
   @Test
   public void testWriteSetTracking7() throws Exception {
     dropTable(new String[] {"tab2", "TAB2"});
-    Assert.assertEquals(0, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET"));
+    Assert.assertEquals(0, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET"));
     CommandProcessorResponse cpr = driver.run("create table if not exists tab2 (a int, b int) " +
       "partitioned by (p string) clustered by (a) into 2  buckets stored as orc TBLPROPERTIES ('transactional'='true')");
     checkCmdOnDriver(cpr);
@@ -1150,13 +1150,13 @@ public class TestDbTxnManager2 {
     txnMgr.commitTxn();//txnid:idTxnUpdate2
     //now both txns concurrently updated TAB2 but different partitions.
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u'"));
     //2 from txnid:1, 1 from txnid:2, 1 from txnid:3
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null"));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab2' and ctc_partition is not null"));
 
     //================
     //test with predicates such that partition pruning doesn't kick in
@@ -1204,13 +1204,13 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     txnMgr.commitTxn();//txnid:idTxnUpdate4
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
     //2 from insert + 1 for each update stmt
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
   }
   /**
    * Concurrent updates with partition pruning predicate and w/o one
@@ -1261,12 +1261,12 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     txnMgr.commitTxn();//txnid:idTxnUpdate2
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
   }
   /**
    * Concurrent update/delete of different partitions - should pass
@@ -1317,18 +1317,18 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     txnMgr.commitTxn();//txnid:idTxnUpdate2
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + "  and ctc_table='tab1'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + "  and ctc_table='tab1' and ctc_partition='p=one'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'"));
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (idTxnUpdate1 - 1) + "  and ctc_table='tab1'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnUpdate1 + "  and ctc_table='tab1' and ctc_partition='p=one'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + idTxnDelete1 + " and ctc_table='tab1' and ctc_partition='p=two'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=one' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1'"));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
   }
   /**
    * Concurrent update/delete of same partition - should fail to commit
@@ -1387,10 +1387,10 @@ public class TestDbTxnManager2 {
       "Aborting [txnid:5,5] due to a write conflict on default/tab1/p=two committed by [txnid:4,5] d/u",
       exception.getCause().getMessage());
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      3, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='u' and ws_table='tab1'"));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      3, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
   }
   /**
    * Concurrent delte/detele of same partition - should pass
@@ -1448,16 +1448,16 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     txnMgr.commitTxn();//"select * from tab1" txn
 
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
-    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString("select * from WRITE_SET"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
-    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdDelete));
+    Assert.assertEquals("WRITE_SET mismatch: " + TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_partition='p=two' and ws_operation_type='d' and ws_table='tab1' and ws_txnid=" + txnIdSelect));
+    Assert.assertEquals("COMPLETED_TXN_COMPONENTS mismatch: " + TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_table='tab1' and ctc_partition is not null"));
   }
   @Test
   public void testCompletedTxnComponents() throws Exception {
@@ -1470,11 +1470,11 @@ public class TestDbTxnManager2 {
     checkCmdOnDriver(driver.run("insert into tab_not_acid2 values(1,1),(2,2)"));
     //writing both acid and non-acid resources in the same txn
     checkCmdOnDriver(driver.run("from tab_not_acid2 insert into tab1 partition(p='two')(a,b) select a,b insert into tab_not_acid2(a,b) select a,b "));//txnid:1
-    Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS"));
+    Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS"));
     //only expect transactional components to be in COMPLETED_TXN_COMPONENTS
-    Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      1, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'"));
+    Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      1, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=6 and ctc_table='tab1'"));
   }
 
   /**
@@ -1495,13 +1495,13 @@ public class TestDbTxnManager2 {
     //writing both acid and non-acid resources in the same txn
     //tab1 write is a dynamic partition insert
     checkCmdOnDriver(driver.run("from tab_not_acid insert into tab1 partition(p)(a,b,p) select a,b,p insert into tab_not_acid(a,b) select a,b where p='two'"));//txnid:9
-    Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      4, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS"));
+    Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      4, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS"));
     //only expect transactional components to be in COMPLETED_TXN_COMPONENTS
-    Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9"));
-    Assert.assertEquals(TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
-      2, TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'"));
+    Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9"));
+    Assert.assertEquals(TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
+      2, TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=9 and ctc_table='tab1'"));
   }
   //todo: Concurrent insert/update of same partition - should pass
 
@@ -1611,9 +1611,9 @@ public class TestDbTxnManager2 {
 
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       0,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
     //complete 1st txn
     AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target",
       Collections.singletonList("p=1/q=3"));//update clause
@@ -1629,39 +1629,39 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
         " and tc_operation_type='u'"));
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       2,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
         " and tc_operation_type='d'"));
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       3,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
         " and tc_operation_type='i'"));
     txnMgr.commitTxn();//commit T1
     Assert.assertEquals(
       "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
       6,
-      TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId1));
     Assert.assertEquals(
       "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from WRITE_SET"),
+        TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
       1,
-      TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
         " and ws_operation_type='u'"));
     Assert.assertEquals(
       "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from WRITE_SET"),
+        TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
       2,
-      TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
         " and ws_operation_type='d'"));
 
     //re-check locks which were in Waiting state - should now be Acquired
@@ -1676,9 +1676,9 @@ public class TestDbTxnManager2 {
 
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       0,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2));
     //complete 2nd txn
     adp = new AddDynamicPartitions(txnId2, "default", "target",
       Collections.singletonList(cc ? "p=1/q=3" : "p=1/p=2"));//update clause
@@ -1694,21 +1694,21 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
         " and tc_operation_type='u'"));
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       2,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
         " and tc_operation_type='d'"));
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       3,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId2 +
         " and tc_operation_type='i'"));
 
     LockException expectedException = null;
@@ -1725,33 +1725,33 @@ public class TestDbTxnManager2 {
         "committed by [txnid:10,11] u/u", expectedException.getMessage());
       Assert.assertEquals(
         "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-          TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+          TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
         0,
-        TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
       Assert.assertEquals(
         "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         0,
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2));
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2));
     }
     else {
       Assert.assertNull("Unexpected exception " + expectedException, expectedException);
       Assert.assertEquals(
         "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-          TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+          TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
         6,
-        TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + txnId2));
       Assert.assertEquals(
         "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         1,
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
           " and ws_operation_type='u'"));
       Assert.assertEquals(
         "WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId2) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         2,
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId2 +
           " and ws_operation_type='d'"));
     }
 
@@ -1790,9 +1790,9 @@ public class TestDbTxnManager2 {
     txnMgr.acquireLocks(driver.getPlan(), ctx, "T1");
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,//no DP, so it's populated from lock info
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
 
     List<ShowLocksResponseElement> locks = getLocks(txnMgr);
     Assert.assertEquals("Unexpected lock count", 1, locks.size());
@@ -1820,9 +1820,9 @@ public class TestDbTxnManager2 {
     txnMgr.commitTxn();//commit T1
 
     Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
-        TxnDbUtil.queryToString("select * from WRITE_SET"),
+        TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
       causeConflict ? 1 : 0,//Inserts are not tracked by WRITE_SET
-      TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid1 +
         " and ws_operation_type=" + (causeConflict ? "'u'" : "'i'")));
 
 
@@ -1835,14 +1835,14 @@ public class TestDbTxnManager2 {
 
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,//
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,//
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 +
       "and tc_operation_type='d'"));
 
     //complete T2 txn
@@ -1862,9 +1862,9 @@ public class TestDbTxnManager2 {
     }
     else {
       Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         1,//Unpartitioned table: 1 row for Delete; Inserts are not tracked in WRITE_SET
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
           " and ws_operation_type='d'"));
     }
   }
@@ -1887,15 +1887,15 @@ public class TestDbTxnManager2 {
     checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", "target", null, locks);
     Assert.assertEquals(
       "HIVE_LOCKS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
-        TxnDbUtil.queryToString("select * from HIVE_LOCKS"),
+        TxnDbUtil.queryToString(conf, "select * from HIVE_LOCKS"),
       1,
-      TxnDbUtil.countQueryAgent("select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from HIVE_LOCKS where hl_txnid=" + txnid1));
     txnMgr.rollbackTxn();
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       0,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid1));
     //now actually write to table to generate some partitions
     checkCmdOnDriver(driver.run("insert into target partition(p=1,q) values (1,2,2), (3,4,2), (5,6,3), (7,8,2)"));
     driver.run("select count(*) from target");
@@ -1904,10 +1904,10 @@ public class TestDbTxnManager2 {
     Assert.assertEquals("", "4", r.get(0));
     Assert.assertEquals(//look in COMPLETED_TXN_COMPONENTS because driver.run() committed!!!!
       "COMPLETED_TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid1 + 1) + "): " +
-        TxnDbUtil.queryToString("select * from COMPLETED_TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from COMPLETED_TXN_COMPONENTS"),
       2,//2 distinct partitions created
       //txnid+1 because we want txn used by previous driver.run("insert....)
-      TxnDbUtil.countQueryAgent("select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1)));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from COMPLETED_TXN_COMPONENTS where ctc_txnid=" + (txnid1 + 1)));
 
 
     long txnid2 = txnMgr.openTxn(ctx, "T1");
@@ -1922,9 +1922,9 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       2,//2 distinct partitions modified
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
     txnMgr.commitTxn();
   }
   @Test
@@ -1986,9 +1986,9 @@ public class TestDbTxnManager2 {
 
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       0,//because it's using a DP write
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1));
     //complete T1 transaction (simulate writing to 2 partitions)
     AddDynamicPartitions adp = new AddDynamicPartitions(txnId1, "default", "target",
       Arrays.asList("p=1/q=2","p=1/q=3"));
@@ -1996,15 +1996,15 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       2,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnId1 +
         " and tc_operation_type='u'"));
     txnMgr.commitTxn();//commit T1
     Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnId1) + "): " +
-      TxnDbUtil.queryToString("select * from WRITE_SET"),
+      TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
       2,//2 partitions updated
-      TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnId1 +
       " and ws_operation_type='u'"));
 
 
@@ -2021,9 +2021,9 @@ public class TestDbTxnManager2 {
 
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       0,//because it's using a DP write
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2));
     //complete T2 txn
     //simulate Insert into 2 partitions
     adp = new AddDynamicPartitions(txnid2, "default", "target",
@@ -2032,9 +2032,9 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       2,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'"));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='i'"));
     //simulate Update of 1 partitions; depending on causeConflict, choose one of the partitions
     //which was modified by the T1 update stmt or choose a non-conflicting one
     adp = new AddDynamicPartitions(txnid2, "default", "target",
@@ -2043,9 +2043,9 @@ public class TestDbTxnManager2 {
     txnHandler.addDynamicPartitions(adp);
     Assert.assertEquals(
       "TXN_COMPONENTS mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-        TxnDbUtil.queryToString("select * from TXN_COMPONENTS"),
+        TxnDbUtil.queryToString(conf, "select * from TXN_COMPONENTS"),
       1,
-      TxnDbUtil.countQueryAgent("select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'"));
+      TxnDbUtil.countQueryAgent(conf, "select count(*) from TXN_COMPONENTS where tc_txnid=" + txnid2 + " and tc_operation_type='u'"));
 
 
     LockException expectedException = null;
@@ -2064,14 +2064,14 @@ public class TestDbTxnManager2 {
     }
     else {
       Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         1,//1 partitions updated
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2 +
           " and ws_operation_type='u'"));
       Assert.assertEquals("WRITE_SET mismatch(" + JavaUtils.txnIdToString(txnid2) + "): " +
-          TxnDbUtil.queryToString("select * from WRITE_SET"),
+          TxnDbUtil.queryToString(conf, "select * from WRITE_SET"),
         1,//1 partitions updated (and no other entries)
-        TxnDbUtil.countQueryAgent("select count(*) from WRITE_SET where ws_txnid=" + txnid2));
+        TxnDbUtil.countQueryAgent(conf, "select count(*) from WRITE_SET where ws_txnid=" + txnid2));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
index af8a63d..2784009 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java
@@ -95,7 +95,7 @@ public abstract class CompactorTest {
   protected CompactorTest() throws Exception {
     conf = new HiveConf();
     TxnDbUtil.setConfValues(conf);
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
     ms = new HiveMetaStoreClient(conf);
     txnHandler = TxnUtils.getTxnStore(conf);
     tmpdir = new File (Files.createTempDirectory("compactor_test_table_").toString());

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/pom.xml
----------------------------------------------------------------------
diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml
index d91b22d..acc50ca 100644
--- a/standalone-metastore/pom.xml
+++ b/standalone-metastore/pom.xml
@@ -33,6 +33,11 @@
 
   <dependencies>
     <dependency>
+      <groupId>com.jolbox</groupId>
+      <artifactId>bonecp</artifactId>
+      <version>${bonecp.version}</version>
+    </dependency>
+    <dependency>
       <groupId>com.github.joshelser</groupId>
       <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId>
       <version>${dropwizard-metrics-hadoop-metrics2-reporter.version}</version>
@@ -48,6 +53,16 @@
       <version>${protobuf.version}</version>
     </dependency>
     <dependency>
+      <groupId>com.zaxxer</groupId>
+      <artifactId>HikariCP</artifactId>
+      <version>${hikaricp.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>commons-dbcp</groupId>
+      <artifactId>commons-dbcp</artifactId>
+      <version>${commons-dbcp.version}</version>
+    </dependency>
+    <dependency>
       <groupId>io.dropwizard.metrics</groupId>
       <artifactId>metrics-core</artifactId>
       <version>${dropwizard.version}</version>
@@ -68,6 +83,11 @@
       <version>${commons-lang3.version}</version>
     </dependency>
     <dependency>
+      <groupId>org.apache.derby</groupId>
+      <artifactId>derby</artifactId>
+      <version>${derby.version}</version>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
       <version>${hadoop.version}</version>

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
new file mode 100644
index 0000000..9fa5cab
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/RunnableConfigurable.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+
+/**
+ * Combination of Runnable and Configurable
+ */
+public interface RunnableConfigurable extends Configurable, Runnable {
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index d3c714b..5a6ef99 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -38,6 +38,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -54,11 +55,8 @@ public class MetastoreConf {
   private static URL hiveSiteURL = null;
   private static URL hiveMetastoreSiteURL = null;
   private static URL metastoreSiteURL = null;
+  private static AtomicBoolean beenDumped = new AtomicBoolean();
 
-  /*
-  private static Map<String, String> metaToHiveKeys;
-  private static Map<String, String> hiveToMetaKeys;
-  */
   private static Map<String, ConfVars> keyToVars;
 
   @VisibleForTesting
@@ -406,6 +404,9 @@ public class MetastoreConf {
             "not blocked.\n" +
             "\n" +
             "See HIVE-4409 for more details."),
+    DUMP_CONFIG_ON_CREATION("metastore.dump.config.on.creation", NO_SUCH_KEY, true,
+        "If true, a printout of the config file (minus sensitive values) will be dumped to the " +
+            "log whenever newMetastoreConf() is called.  Can produce a lot of logs"),
     END_FUNCTION_LISTENERS("metastore.end.function.listeners",
         "hive.metastore.end.function.listeners", "",
         "List of comma separated listeners for the end of metastore functions."),
@@ -932,6 +933,11 @@ public class MetastoreConf {
     if (getBoolVar(conf, ConfVars.SCHEMA_VERIFICATION)) {
       setBoolVar(conf, ConfVars.AUTO_CREATE_ALL, false);
     }
+
+    if (!beenDumped.getAndSet(true) && getBoolVar(conf, ConfVars.DUMP_CONFIG_ON_CREATION) &&
+        LOG.isInfoEnabled()) {
+      LOG.info(dumpConfig(conf));
+    }
     return conf;
   }
 
@@ -1304,8 +1310,8 @@ public class MetastoreConf {
   }
 
   /**
-   * Return the configuration value as a String.  This does not work on time based values as it
-   * doesn't have a time unit.
+   * Return the configuration value as a String.  For time based values it will be returned in
+   * the default time unit appended with an appropriate abbreviation (eg s for seconds, ...)
    * @param conf configuration to read
    * @param var variable to read
    * @return value as an object
@@ -1319,8 +1325,12 @@ public class MetastoreConf {
       return Long.toString(getLongVar(conf, var));
     } else if (var.defaultVal.getClass() == Double.class) {
       return Double.toString(getDoubleVar(conf, var));
+    } else if (var.defaultVal.getClass() == TimeValue.class) {
+      TimeUnit timeUnit = (var.defaultVal.getClass() == TimeValue.class) ?
+          ((TimeValue)var.defaultVal).unit : null;
+      return Long.toString(getTimeVar(conf, var, timeUnit)) + timeAbbreviationFor(timeUnit);
     } else {
-      throw new RuntimeException("Invalid type for getObject " + var.defaultVal.getClass().getName());
+      throw new RuntimeException("Unknown type for getObject " + var.defaultVal.getClass().getName());
     }
   }
 
@@ -1356,6 +1366,45 @@ public class MetastoreConf {
   }
 
   /**
+   * Dump the configuration file to the log.  It will be dumped at an INFO level.  This can
+   * potentially produce a lot of logs, so you might want to be careful when and where you do it.
+   * It takes care not to dump hidden keys.
+   * @param conf Configuration file to dump
+   * @return String containing dumped config file.
+   */
+  public static String dumpConfig(Configuration conf) {
+    StringBuilder buf = new StringBuilder("MetastoreConf object:\n");
+    if (hiveSiteURL != null) {
+      buf.append("Used hive-site file: ")
+          .append(hiveSiteURL)
+          .append('\n');
+    }
+    if (hiveMetastoreSiteURL != null) {
+      buf.append("Used hivemetastore-site file: ")
+          .append(hiveMetastoreSiteURL)
+          .append('\n');
+    }
+    if (metastoreSiteURL != null) {
+      buf.append("Used metastore-site file: ")
+          .append(metastoreSiteURL)
+          .append('\n');
+    }
+    for (ConfVars var : ConfVars.values()) {
+      if (!unprintables.contains(var.varname)) {
+        buf.append("Key: <")
+            .append(var.varname)
+            .append("> old hive key: <")
+            .append(var.hiveName)
+            .append(">  value: <")
+            .append(getAsString(conf, var))
+            .append(">\n");
+      }
+    }
+    buf.append("Finished MetastoreConf object.\n");
+    return buf.toString();
+  }
+
+  /**
    * validate value for a ConfVar, return non-null string for fail message
    */
   public interface Validator {

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
new file mode 100644
index 0000000..6a2f770
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.jolbox.bonecp.BoneCPConfig;
+import com.jolbox.bonecp.BoneCPDataSource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * DataSourceProvider for the BoneCP connection pool.
+ */
+public class BoneCPDataSourceProvider implements DataSourceProvider {
+
+  private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class);
+
+  public static final String BONECP = "bonecp";
+  private static final String CONNECTION_TIMEOUT_PROPERTY= "bonecp.connectionTimeoutInMs";
+  private static final String PARTITION_COUNT_PROPERTY= "bonecp.partitionCount";
+
+  @Override
+  public DataSource create(Configuration hdpConfig) throws SQLException {
+
+    LOG.debug("Creating BoneCP connection pool for the MetaStore");
+
+    String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
+    String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
+    String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
+    int maxPoolSize = MetastoreConf.getIntVar(hdpConfig,
+        MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
+
+    Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP);
+    long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
+    String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1");
+
+    BoneCPConfig config = null;
+    try {
+      config = new BoneCPConfig(properties);
+    } catch (Exception e) {
+      throw new SQLException("Cannot create BoneCP configuration: ", e);
+    }
+    config.setJdbcUrl(driverUrl);
+    //if we are waiting for connection for a long time, something is really wrong
+    //better raise an error than hang forever
+    //see DefaultConnectionStrategy.getConnectionInternal()
+    config.setConnectionTimeoutInMs(connectionTimeout);
+    config.setMaxConnectionsPerPartition(maxPoolSize);
+    config.setPartitionCount(Integer.parseInt(partitionCount));
+    config.setUser(user);
+    config.setPassword(passwd);
+    return new BoneCPDataSource(config);
+  }
+
+  @Override
+  public boolean mayReturnClosedConnection() {
+    // See HIVE-11915 for details
+    return true;
+  }
+
+  @Override
+  public boolean supports(Configuration configuration) {
+    String poolingType =
+        MetastoreConf.getVar(configuration,
+            MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE).toLowerCase();
+    if (BONECP.equals(poolingType)) {
+      int boneCpPropsNr = DataSourceProvider.getPrefixedProperties(configuration, BONECP).size();
+      LOG.debug("Found " + boneCpPropsNr + " nr. of bonecp specific configurations");
+      return boneCpPropsNr > 0;
+    }
+    LOG.debug("Configuration requested " + poolingType + " pooling, BoneCpDSProvider exiting");
+    return false;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
new file mode 100644
index 0000000..17ff8d1
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+
+import javax.sql.DataSource;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+public interface DataSourceProvider {
+
+  /**
+   * @param hdpConfig
+   * @return the new connection pool
+   */
+  DataSource create(Configuration hdpConfig) throws SQLException;
+
+  /**
+   * BoneCp has a bug which causes closed connections to be returned to the pool
+   * under certain conditions. (HIVE-11915)
+   * @return true if the factory creates BoneCp pools which need "special attention"
+   */
+  boolean mayReturnClosedConnection();
+
+  /**
+   * @param configuration Hadoop configuration object
+   * @return factory able to create a connection pool for the implementation
+   * specified in the configuration
+   */
+  boolean supports(Configuration configuration);
+
+  /**
+   * @param hdpConfig
+   * @return subset of properties prefixed by a connection pool specific substring
+   */
+  static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) {
+    Properties dataSourceProps = new Properties();
+    Iterables.filter(
+        hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix)))
+        .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue()));
+    return dataSourceProps;
+  }
+
+  static String getMetastoreJdbcUser(Configuration conf) {
+    return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTION_USER_NAME);
+  }
+
+  static String getMetastoreJdbcPasswd(Configuration conf) throws SQLException {
+    try {
+      return MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
+    } catch (IOException err) {
+      throw new SQLException("Error getting metastore password", err);
+    }
+  }
+
+  static String getMetastoreJdbcDriverUrl(Configuration conf) throws SQLException {
+    return MetastoreConf.getVar(conf, MetastoreConf.ConfVars.CONNECTURLKEY);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
new file mode 100644
index 0000000..e3c18e3
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Create a DataSourceProvider for a connectionPool configured in a hadoop
+ * Configuration object.
+ */
+public abstract  class DataSourceProviderFactory {
+
+  private static final ImmutableList<DataSourceProvider> FACTORIES =
+      ImmutableList.<DataSourceProvider>builder().add(new HikariCPDataSourceProvider(), new BoneCPDataSourceProvider()).build();
+
+  /**
+   * @param hdpConfig hadoop configuration
+   * @return factory for the configured datanucleus.connectionPoolingType
+   */
+  public static DataSourceProvider getDataSourceProvider(Configuration hdpConfig) {
+
+    for (DataSourceProvider factory : FACTORIES) {
+
+      if (factory.supports(hdpConfig)) {
+        return factory;
+      }
+    }
+    return null;
+  }
+
+}


[05/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
new file mode 100644
index 0000000..9fc3697
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.datasource;
+
+import com.zaxxer.hikari.HikariConfig;
+import com.zaxxer.hikari.HikariDataSource;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.sql.DataSource;
+import java.sql.SQLException;
+import java.util.Properties;
+
+/**
+ * DataSourceProvider for the HikariCP connection pool.
+ */
+public class HikariCPDataSourceProvider implements DataSourceProvider {
+
+  private static final Logger LOG = LoggerFactory.getLogger(HikariCPDataSourceProvider.class);
+
+  public static final String HIKARI = "hikari";
+  private static final String CONNECTION_TIMEOUT_PROPERTY= "hikari.connectionTimeout";
+
+  @Override
+  public DataSource create(Configuration hdpConfig) throws SQLException {
+
+    LOG.debug("Creating Hikari connection pool for the MetaStore");
+
+    String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
+    String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
+    String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
+    int maxPoolSize = MetastoreConf.getIntVar(hdpConfig,
+        MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
+
+    Properties properties = replacePrefix(
+        DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI));
+    long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
+    HikariConfig config = null;
+    try {
+      config = new HikariConfig(properties);
+    } catch (Exception e) {
+      throw new SQLException("Cannot create HikariCP configuration: ", e);
+    }
+    config.setMaximumPoolSize(maxPoolSize);
+    config.setJdbcUrl(driverUrl);
+    config.setUsername(user);
+    config.setPassword(passwd);
+    //https://github.com/brettwooldridge/HikariCP
+    config.setConnectionTimeout(connectionTimeout);
+    return new HikariDataSource(config);
+  }
+
+  @Override
+  public boolean mayReturnClosedConnection() {
+    // Only BoneCP should return true
+    return false;
+  }
+
+  @Override
+  public boolean supports(Configuration configuration) {
+    String poolingType =
+        MetastoreConf.getVar(configuration,
+            MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE).toLowerCase();
+    if (HIKARI.equals(poolingType)) {
+      int hikariPropsNr = DataSourceProvider.getPrefixedProperties(configuration, HIKARI).size();
+      LOG.debug("Found " + hikariPropsNr + " nr. of hikari specific configurations");
+      return hikariPropsNr > 0;
+    }
+    LOG.debug("Configuration requested " + poolingType + " pooling, HikariCpDSProvider exiting");
+    return false;
+  }
+
+  private Properties replacePrefix(Properties props) {
+    Properties newProps = new Properties();
+    props.forEach((key,value) ->
+        newProps.put(key.toString().replaceFirst(HIKARI + ".", ""), value));
+    return newProps;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/package-info.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
new file mode 100644
index 0000000..9a4f22a
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * DataSource providers that can construct a connection pool from configuration
+ * properties in a hadoop configuration object.
+ */
+package org.apache.hadoop.hive.metastore.datasource;

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
new file mode 100644
index 0000000..8268af9
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
@@ -0,0 +1,172 @@
+/*
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.tools;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * Helper class that generates SQL queries with syntax specific to target DB
+ * todo: why throw MetaException?
+ */
+@VisibleForTesting
+public final class SQLGenerator {
+  static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName());
+  private final DatabaseProduct dbProduct;
+  private final Configuration conf;
+
+  public SQLGenerator(DatabaseProduct dbProduct, Configuration conf) {
+    this.dbProduct = dbProduct;
+    this.conf = conf;
+  }
+
+  /**
+   * Genereates "Insert into T(a,b,c) values(1,2,'f'),(3,4,'c')" for appropriate DB
+   *
+   * @param tblColumns e.g. "T(a,b,c)"
+   * @param rows       e.g. list of Strings like 3,4,'d'
+   * @return fully formed INSERT INTO ... statements
+   */
+  public List<String> createInsertValuesStmt(String tblColumns, List<String> rows) {
+    if (rows == null || rows.size() == 0) {
+      return Collections.emptyList();
+    }
+    List<String> insertStmts = new ArrayList<>();
+    StringBuilder sb = new StringBuilder();
+    switch (dbProduct) {
+    case ORACLE:
+      if (rows.size() > 1) {
+        //http://www.oratable.com/oracle-insert-all/
+        //https://livesql.oracle.com/apex/livesql/file/content_BM1LJQ87M5CNIOKPOWPV6ZGR3.html
+        for (int numRows = 0; numRows < rows.size(); numRows++) {
+          if (numRows % MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
+            if (numRows > 0) {
+              sb.append(" select * from dual");
+              insertStmts.add(sb.toString());
+            }
+            sb.setLength(0);
+            sb.append("insert all ");
+          }
+          sb.append("into ").append(tblColumns).append(" values(").append(rows.get(numRows))
+              .append(") ");
+        }
+        sb.append("select * from dual");
+        insertStmts.add(sb.toString());
+        return insertStmts;
+      }
+      //fall through
+    case DERBY:
+    case MYSQL:
+    case POSTGRES:
+    case SQLSERVER:
+      for (int numRows = 0; numRows < rows.size(); numRows++) {
+        if (numRows % MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
+          if (numRows > 0) {
+            insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
+          }
+          sb.setLength(0);
+          sb.append("insert into ").append(tblColumns).append(" values");
+        }
+        sb.append('(').append(rows.get(numRows)).append("),");
+      }
+      insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
+      return insertStmts;
+    default:
+      String msg = "Unrecognized database product name <" + dbProduct + ">";
+      LOG.error(msg);
+      throw new IllegalStateException(msg);
+    }
+  }
+
+  /**
+   * Given a {@code selectStatement}, decorated it with FOR UPDATE or semantically equivalent
+   * construct.  If the DB doesn't support, return original select.
+   */
+  public String addForUpdateClause(String selectStatement) throws MetaException {
+    switch (dbProduct) {
+    case DERBY:
+      //https://db.apache.org/derby/docs/10.1/ref/rrefsqlj31783.html
+      //sadly in Derby, FOR UPDATE doesn't meant what it should
+      return selectStatement;
+    case MYSQL:
+      //http://dev.mysql.com/doc/refman/5.7/en/select.html
+    case ORACLE:
+      //https://docs.oracle.com/cd/E17952_01/refman-5.6-en/select.html
+    case POSTGRES:
+      //http://www.postgresql.org/docs/9.0/static/sql-select.html
+      return selectStatement + " for update";
+    case SQLSERVER:
+      //https://msdn.microsoft.com/en-us/library/ms189499.aspx
+      //https://msdn.microsoft.com/en-us/library/ms187373.aspx
+      String modifier = " with (updlock)";
+      int wherePos = selectStatement.toUpperCase().indexOf(" WHERE ");
+      if (wherePos < 0) {
+        return selectStatement + modifier;
+      }
+      return selectStatement.substring(0, wherePos) + modifier +
+          selectStatement.substring(wherePos, selectStatement.length());
+    default:
+      String msg = "Unrecognized database product name <" + dbProduct + ">";
+      LOG.error(msg);
+      throw new MetaException(msg);
+    }
+  }
+
+  /**
+   * Suppose you have a query "select a,b from T" and you want to limit the result set
+   * to the first 5 rows.  The mechanism to do that differs in different DBs.
+   * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the
+   * appropriately modified row limiting query.
+   * <p>
+   * Note that if {@code noSelectsqlQuery} contains a join, you must make sure that
+   * all columns are unique for Oracle.
+   */
+  public String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaException {
+    switch (dbProduct) {
+    case DERBY:
+      //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html
+      return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only";
+    case MYSQL:
+      //http://www.postgresql.org/docs/7.3/static/queries-limit.html
+    case POSTGRES:
+      //https://dev.mysql.com/doc/refman/5.0/en/select.html
+      return "select " + noSelectsqlQuery + " limit " + numRows;
+    case ORACLE:
+      //newer versions (12c and later) support OFFSET/FETCH
+      return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows;
+    case SQLSERVER:
+      //newer versions (2012 and later) support OFFSET/FETCH
+      //https://msdn.microsoft.com/en-us/library/ms189463.aspx
+      return "select TOP(" + numRows + ") " + noSelectsqlQuery;
+    default:
+      String msg = "Unrecognized database product name <" + dbProduct + ">";
+      LOG.error(msg);
+      throw new MetaException(msg);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java
new file mode 100644
index 0000000..1223b52
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/AcidOpenTxnsCounterService.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.RunnableConfigurable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Background running thread, periodically updating number of open transactions.
+ * Runs inside Hive Metastore Service.
+ */
+public class AcidOpenTxnsCounterService implements RunnableConfigurable {
+  private static final Logger LOG = LoggerFactory.getLogger(AcidOpenTxnsCounterService.class);
+
+  private Configuration conf;
+  private int isAliveCounter = 0;
+  private long lastLogTime = 0;
+
+  @Override
+  public void run() {
+    try {
+      long startTime = System.currentTimeMillis();
+      isAliveCounter++;
+      TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+      txnHandler.countOpenTxns();
+      if (System.currentTimeMillis() - lastLogTime > 60 * 1000) {
+        LOG.info("AcidOpenTxnsCounterService ran for " +
+            ((System.currentTimeMillis() - startTime) / 1000) +
+            " seconds.  isAliveCounter = " + isAliveCounter);
+        lastLogTime = System.currentTimeMillis();
+      }
+    }
+    catch(Throwable t) {
+      LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t);
+    }
+  }
+
+  @Override
+  public void setConf(Configuration configuration) {
+    conf = configuration;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
new file mode 100644
index 0000000..41e428b
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.common.ValidCompactorTxnList;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+/**
+ * Information on a possible or running compaction.
+ */
+public class CompactionInfo implements Comparable<CompactionInfo> {
+  public long id;
+  public String dbname;
+  public String tableName;
+  public String partName;
+  char state;
+  public CompactionType type;
+  String workerId;
+  long start;
+  public String runAs;
+  public String properties;
+  public boolean tooManyAborts = false;
+  /**
+   * {@code 0} means it wasn't set (e.g. in case of upgrades, since ResultSet.getLong() will return 0 if field is NULL) 
+   * See {@link TxnStore#setCompactionHighestTxnId(CompactionInfo, long)} for precise definition.
+   * See also {@link TxnUtils#createValidCompactTxnList(GetOpenTxnsInfoResponse)} and
+   * {@link ValidCompactorTxnList#highWatermark}
+   */
+  public long highestTxnId;
+  byte[] metaInfo;
+  String hadoopJobId;
+
+  private String fullPartitionName = null;
+  private String fullTableName = null;
+
+  public CompactionInfo(String dbname, String tableName, String partName, CompactionType type) {
+    this.dbname = dbname;
+    this.tableName = tableName;
+    this.partName = partName;
+    this.type = type;
+  }
+  CompactionInfo(long id, String dbname, String tableName, String partName, char state) {
+    this(dbname, tableName, partName, null);
+    this.id = id;
+    this.state = state;
+  }
+  CompactionInfo() {}
+  
+  public String getFullPartitionName() {
+    if (fullPartitionName == null) {
+      StringBuilder buf = new StringBuilder(dbname);
+      buf.append('.');
+      buf.append(tableName);
+      if (partName != null) {
+        buf.append('.');
+        buf.append(partName);
+      }
+      fullPartitionName = buf.toString();
+    }
+    return fullPartitionName;
+  }
+
+  public String getFullTableName() {
+    if (fullTableName == null) {
+      StringBuilder buf = new StringBuilder(dbname);
+      buf.append('.');
+      buf.append(tableName);
+      fullTableName = buf.toString();
+    }
+    return fullTableName;
+  }
+  public boolean isMajorCompaction() {
+    return CompactionType.MAJOR == type;
+  }
+
+  @Override
+  public int compareTo(CompactionInfo o) {
+    return getFullPartitionName().compareTo(o.getFullPartitionName());
+  }
+  public String toString() {
+    return "id:" + id + "," +
+      "dbname:" + dbname + "," +
+      "tableName:" + tableName + "," +
+      "partName:" + partName + "," +
+      "state:" + state + "," +
+      "type:" + type + "," +
+      "properties:" + properties + "," +
+      "runAs:" + runAs + "," +
+      "tooManyAborts:" + tooManyAborts + "," +
+      "highestTxnId:" + highestTxnId;
+  }
+
+  /**
+   * loads object from a row in Select * from COMPACTION_QUEUE
+   * @param rs ResultSet after call to rs.next()
+   * @throws SQLException
+   */
+  static CompactionInfo loadFullFromCompactionQueue(ResultSet rs) throws SQLException {
+    CompactionInfo fullCi = new CompactionInfo();
+    fullCi.id = rs.getLong(1);
+    fullCi.dbname = rs.getString(2);
+    fullCi.tableName = rs.getString(3);
+    fullCi.partName = rs.getString(4);
+    fullCi.state = rs.getString(5).charAt(0);//cq_state
+    fullCi.type = TxnHandler.dbCompactionType2ThriftType(rs.getString(6).charAt(0));
+    fullCi.properties = rs.getString(7);
+    fullCi.workerId = rs.getString(8);
+    fullCi.start = rs.getLong(9);
+    fullCi.runAs = rs.getString(10);
+    fullCi.highestTxnId = rs.getLong(11);
+    fullCi.metaInfo = rs.getBytes(12);
+    fullCi.hadoopJobId = rs.getString(13);
+    return fullCi;
+  }
+  static void insertIntoCompletedCompactions(PreparedStatement pStmt, CompactionInfo ci, long endTime) throws SQLException {
+    pStmt.setLong(1, ci.id);
+    pStmt.setString(2, ci.dbname);
+    pStmt.setString(3, ci.tableName);
+    pStmt.setString(4, ci.partName);
+    pStmt.setString(5, Character.toString(ci.state));
+    pStmt.setString(6, Character.toString(TxnHandler.thriftCompactionType2DbType(ci.type)));
+    pStmt.setString(7, ci.properties);
+    pStmt.setString(8, ci.workerId);
+    pStmt.setLong(9, ci.start);
+    pStmt.setLong(10, endTime);
+    pStmt.setString(11, ci.runAs);
+    pStmt.setLong(12, ci.highestTxnId);
+    pStmt.setBytes(13, ci.metaInfo);
+    pStmt.setString(14, ci.hadoopJobId);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
new file mode 100644
index 0000000..e676b91
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
@@ -0,0 +1,961 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.hive.common.classification.RetrySemantics;
+import org.apache.hadoop.hive.metastore.api.CompactionType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Extends the transaction handler with methods needed only by the compactor threads.  These
+ * methods are not available through the thrift interface.
+ */
+class CompactionTxnHandler extends TxnHandler {
+  static final private String CLASS_NAME = CompactionTxnHandler.class.getName();
+  static final private Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
+
+  public CompactionTxnHandler() {
+  }
+
+  /**
+   * This will look through the completed_txn_components table and look for partitions or tables
+   * that may be ready for compaction.  Also, look through txns and txn_components tables for
+   * aborted transactions that we should add to the list.
+   * @param maxAborted Maximum number of aborted queries to allow before marking this as a
+   *                   potential compaction.
+   * @return list of CompactionInfo structs.  These will not have id, type,
+   * or runAs set since these are only potential compactions not actual ones.
+   */
+  @Override
+  @RetrySemantics.ReadOnly
+  public Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException {
+    Connection dbConn = null;
+    Set<CompactionInfo> response = new HashSet<>();
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        // Check for completed transactions
+        String s = "select distinct ctc_database, ctc_table, " +
+          "ctc_partition from COMPLETED_TXN_COMPONENTS";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          CompactionInfo info = new CompactionInfo();
+          info.dbname = rs.getString(1);
+          info.tableName = rs.getString(2);
+          info.partName = rs.getString(3);
+          response.add(info);
+        }
+        rs.close();
+
+        // Check for aborted txns
+        s = "select tc_database, tc_table, tc_partition " +
+          "from TXNS, TXN_COMPONENTS " +
+          "where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' " +
+          "group by tc_database, tc_table, tc_partition " +
+          "having count(*) > " + maxAborted;
+
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          CompactionInfo info = new CompactionInfo();
+          info.dbname = rs.getString(1);
+          info.tableName = rs.getString(2);
+          info.partName = rs.getString(3);
+          info.tooManyAborts = true;
+          response.add(info);
+        }
+
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+      } catch (SQLException e) {
+        LOG.error("Unable to connect to transaction database " + e.getMessage());
+        checkRetryable(dbConn, e, "findPotentialCompactions(maxAborted:" + maxAborted + ")");
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+      return response;
+    }
+    catch (RetryException e) {
+      return findPotentialCompactions(maxAborted);
+    }
+  }
+
+  /**
+   * Sets the user to run as.  This is for the case
+   * where the request was generated by the user and so the worker must set this value later.
+   * @param cq_id id of this entry in the queue
+   * @param user user to run the jobs as
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public void setRunAs(long cq_id, String user) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "update COMPACTION_QUEUE set cq_run_as = '" + user + "' where cq_id = " + cq_id;
+        LOG.debug("Going to execute update <" + s + ">");
+        int updCnt = stmt.executeUpdate(s);
+        if (updCnt != 1) {
+          LOG.error("Unable to set cq_run_as=" + user + " for compaction record with cq_id=" + cq_id + ".  updCnt=" + updCnt);
+          LOG.debug("Going to rollback");
+          dbConn.rollback();
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to update compaction queue, " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "setRunAs(cq_id:" + cq_id + ",user:" + user +")");
+      } finally {
+        closeDbConn(dbConn);
+        closeStmt(stmt);
+      }
+    } catch (RetryException e) {
+      setRunAs(cq_id, user);
+    }
+  }
+
+  /**
+   * This will grab the next compaction request off of
+   * the queue, and assign it to the worker.
+   * @param workerId id of the worker calling this, will be recorded in the db
+   * @return an info element for this compaction request, or null if there is no work to do now.
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public CompactionInfo findNextToCompact(String workerId) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      //need a separate stmt for executeUpdate() otherwise it will close the ResultSet(HIVE-12725)
+      Statement updStmt = null;
+      ResultSet rs = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select cq_id, cq_database, cq_table, cq_partition, " +
+          "cq_type, cq_tblproperties from COMPACTION_QUEUE where cq_state = '" + INITIATED_STATE + "'";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        if (!rs.next()) {
+          LOG.debug("No compactions found ready to compact");
+          dbConn.rollback();
+          return null;
+        }
+        updStmt = dbConn.createStatement();
+        do {
+          CompactionInfo info = new CompactionInfo();
+          info.id = rs.getLong(1);
+          info.dbname = rs.getString(2);
+          info.tableName = rs.getString(3);
+          info.partName = rs.getString(4);
+          info.type = dbCompactionType2ThriftType(rs.getString(5).charAt(0));
+          info.properties = rs.getString(6);
+          // Now, update this record as being worked on by this worker.
+          long now = getDbTime(dbConn);
+          s = "update COMPACTION_QUEUE set cq_worker_id = '" + workerId + "', " +
+            "cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id +
+            " AND cq_state='" + INITIATED_STATE + "'";
+          LOG.debug("Going to execute update <" + s + ">");
+          int updCount = updStmt.executeUpdate(s);
+          if(updCount == 1) {
+            dbConn.commit();
+            return info;
+          }
+          if(updCount == 0) {
+            LOG.debug("Another Worker picked up " + info);
+            continue;
+          }
+          LOG.error("Unable to set to cq_state=" + WORKING_STATE + " for compaction record: " +
+            info + ". updCnt=" + updCount + ".");
+          dbConn.rollback();
+          return null;
+        } while( rs.next());
+        dbConn.rollback();
+        return null;
+      } catch (SQLException e) {
+        LOG.error("Unable to select next element for compaction, " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "findNextToCompact(workerId:" + workerId + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(updStmt);
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return findNextToCompact(workerId);
+    }
+  }
+
+  /**
+   * This will mark an entry in the queue as compacted
+   * and put it in the ready to clean state.
+   * @param info info on the compaction entry to mark as compacted.
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void markCompacted(CompactionInfo info) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "update COMPACTION_QUEUE set cq_state = '" + READY_FOR_CLEANING + "', " +
+          "cq_worker_id = null where cq_id = " + info.id;
+        LOG.debug("Going to execute update <" + s + ">");
+        int updCnt = stmt.executeUpdate(s);
+        if (updCnt != 1) {
+          LOG.error("Unable to set cq_state=" + READY_FOR_CLEANING + " for compaction record: " + info + ". updCnt=" + updCnt);
+          LOG.debug("Going to rollback");
+          dbConn.rollback();
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to update compaction queue " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "markCompacted(" + info + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      markCompacted(info);
+    }
+  }
+
+  /**
+   * Find entries in the queue that are ready to
+   * be cleaned.
+   * @return information on the entry in the queue.
+   */
+  @Override
+  @RetrySemantics.ReadOnly
+  public List<CompactionInfo> findReadyToClean() throws MetaException {
+    Connection dbConn = null;
+    List<CompactionInfo> rc = new ArrayList<>();
+
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select cq_id, cq_database, cq_table, cq_partition, " +
+          "cq_type, cq_run_as, cq_highest_txn_id from COMPACTION_QUEUE where cq_state = '" + READY_FOR_CLEANING + "'";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          CompactionInfo info = new CompactionInfo();
+          info.id = rs.getLong(1);
+          info.dbname = rs.getString(2);
+          info.tableName = rs.getString(3);
+          info.partName = rs.getString(4);
+          switch (rs.getString(5).charAt(0)) {
+            case MAJOR_TYPE: info.type = CompactionType.MAJOR; break;
+            case MINOR_TYPE: info.type = CompactionType.MINOR; break;
+            default: throw new MetaException("Unexpected compaction type " + rs.getString(5));
+          }
+          info.runAs = rs.getString(6);
+          info.highestTxnId = rs.getLong(7);
+          rc.add(info);
+        }
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+        return rc;
+      } catch (SQLException e) {
+        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "findReadyToClean");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return findReadyToClean();
+    }
+  }
+
+  /**
+   * This will remove an entry from the queue after
+   * it has been compacted.
+   * 
+   * @param info info on the compaction entry to remove
+   */
+  @Override
+  @RetrySemantics.CannotRetry
+  public void markCleaned(CompactionInfo info) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      PreparedStatement pStmt = null;
+      ResultSet rs = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + info.id);
+        if(rs.next()) {
+          info = CompactionInfo.loadFullFromCompactionQueue(rs);
+        }
+        else {
+          throw new IllegalStateException("No record with CQ_ID=" + info.id + " found in COMPACTION_QUEUE");
+        }
+        close(rs);
+        String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id;
+        LOG.debug("Going to execute update <" + s + ">");
+        int updCount = stmt.executeUpdate(s);
+        if (updCount != 1) {
+          LOG.error("Unable to delete compaction record: " + info +  ".  Update count=" + updCount);
+          LOG.debug("Going to rollback");
+          dbConn.rollback();
+        }
+        pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
+        info.state = SUCCEEDED_STATE;
+        CompactionInfo.insertIntoCompletedCompactions(pStmt, info, getDbTime(dbConn));
+        updCount = pStmt.executeUpdate();
+
+        // Remove entries from completed_txn_components as well, so we don't start looking there
+        // again but only up to the highest txn ID include in this compaction job.
+        //highestTxnId will be NULL in upgrade scenarios
+        s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " +
+          "ctc_table = '" + info.tableName + "'";
+        if (info.partName != null) {
+          s += " and ctc_partition = '" + info.partName + "'";
+        }
+        if(info.highestTxnId != 0) {
+          s += " and ctc_txnid <= " + info.highestTxnId;
+        }
+        LOG.debug("Going to execute update <" + s + ">");
+        if (stmt.executeUpdate(s) < 1) {
+          LOG.error("Expected to remove at least one row from completed_txn_components when " +
+            "marking compaction entry as clean!");
+        }
+
+        s = "select distinct txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" +
+          TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" +
+          info.tableName + "'" + (info.highestTxnId == 0 ? "" : " and txn_id <= " + info.highestTxnId);
+        if (info.partName != null) s += " and tc_partition = '" + info.partName + "'";
+        LOG.debug("Going to execute update <" + s + ">");
+        rs = stmt.executeQuery(s);
+        List<Long> txnids = new ArrayList<>();
+        while (rs.next()) txnids.add(rs.getLong(1));
+        // Remove entries from txn_components, as there may be aborted txn components
+        if (txnids.size() > 0) {
+          List<String> queries = new ArrayList<>();
+
+          // Prepare prefix and suffix
+          StringBuilder prefix = new StringBuilder();
+          StringBuilder suffix = new StringBuilder();
+
+          prefix.append("delete from TXN_COMPONENTS where ");
+
+          //because 1 txn may include different partitions/tables even in auto commit mode
+          suffix.append(" and tc_database = ");
+          suffix.append(quoteString(info.dbname));
+          suffix.append(" and tc_table = ");
+          suffix.append(quoteString(info.tableName));
+          if (info.partName != null) {
+            suffix.append(" and tc_partition = ");
+            suffix.append(quoteString(info.partName));
+          }
+
+          // Populate the complete query with provided prefix and suffix
+          TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "tc_txnid", true, false);
+
+          for (String query : queries) {
+            LOG.debug("Going to execute update <" + query + ">");
+            int rc = stmt.executeUpdate(query);
+            LOG.debug("Removed " + rc + " records from txn_components");
+
+            // Don't bother cleaning from the txns table.  A separate call will do that.  We don't
+            // know here which txns still have components from other tables or partitions in the
+            // table, so we don't know which ones we can and cannot clean.
+          }
+        }
+
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to delete from compaction queue " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "markCleaned(" + info + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(pStmt);
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      markCleaned(info);
+    }
+  }
+
+  /**
+   * Clean up aborted transactions from txns that have no components in txn_components.  The reason such
+   * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
+   * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void cleanEmptyAbortedTxns() throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      ResultSet rs = null;
+      try {
+        //Aborted is a terminal state, so nothing about the txn can change
+        //after that, so READ COMMITTED is sufficient.
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select txn_id from TXNS where " +
+          "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " +
+          "txn_state = '" + TXN_ABORTED + "'";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        List<Long> txnids = new ArrayList<>();
+        while (rs.next()) txnids.add(rs.getLong(1));
+        close(rs);
+        if(txnids.size() <= 0) {
+          return;
+        }
+        Collections.sort(txnids);//easier to read logs
+        List<String> queries = new ArrayList<>();
+        StringBuilder prefix = new StringBuilder();
+        StringBuilder suffix = new StringBuilder();
+
+        prefix.append("delete from TXNS where ");
+        suffix.append("");
+
+        TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "txn_id", false, false);
+
+        for (String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          int rc = stmt.executeUpdate(query);
+          LOG.info("Removed " + rc + "  empty Aborted transactions from TXNS");
+        }
+        LOG.info("Aborted transactions removed from TXNS: " + txnids);
+
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to delete from txns table " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "cleanEmptyAbortedTxns");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      cleanEmptyAbortedTxns();
+    }
+  }
+
+  /**
+   * This will take all entries assigned to workers
+   * on a host return them to INITIATED state.  The initiator should use this at start up to
+   * clean entries from any workers that were in the middle of compacting when the metastore
+   * shutdown.  It does not reset entries from worker threads on other hosts as those may still
+   * be working.
+   * @param hostname Name of this host.  It is assumed this prefixes the thread's worker id,
+   *                 so that like hostname% will match the worker id.
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public void revokeFromLocalWorkers(String hostname) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
+          + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_worker_id like '"
+          +  hostname + "%'";
+        LOG.debug("Going to execute update <" + s + ">");
+        // It isn't an error if the following returns no rows, as the local workers could have died
+        // with  nothing assigned to them.
+        stmt.executeUpdate(s);
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to change dead worker's records back to initiated state " +
+          e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "revokeFromLocalWorkers(hostname:" + hostname +")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      revokeFromLocalWorkers(hostname);
+    }
+  }
+
+  /**
+   * This call will return all compaction queue
+   * entries assigned to a worker but over the timeout back to the initiated state.
+   * This should be called by the initiator on start up and occasionally when running to clean up
+   * after dead threads.  At start up {@link #revokeFromLocalWorkers(String)} should be called
+   * first.
+   * @param timeout number of milliseconds since start time that should elapse before a worker is
+   *                declared dead.
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public void revokeTimedoutWorkers(long timeout) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        long latestValidStart = getDbTime(dbConn) - timeout;
+        stmt = dbConn.createStatement();
+        String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
+          + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_start < "
+          +  latestValidStart;
+        LOG.debug("Going to execute update <" + s + ">");
+        // It isn't an error if the following returns no rows, as the local workers could have died
+        // with  nothing assigned to them.
+        stmt.executeUpdate(s);
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.error("Unable to change dead worker's records back to initiated state " +
+          e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "revokeTimedoutWorkers(timeout:" + timeout + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      revokeTimedoutWorkers(timeout);
+    }
+  }
+
+  /**
+   * Queries metastore DB directly to find columns in the table which have statistics information.
+   * If {@code ci} includes partition info then per partition stats info is examined, otherwise
+   * table level stats are examined.
+   * @throws MetaException
+   */
+  @Override
+  @RetrySemantics.ReadOnly
+  public List<String> findColumnsWithStats(CompactionInfo ci) throws MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        String quote = getIdentifierQuoteString(dbConn);
+        stmt = dbConn.createStatement();
+        StringBuilder bldr = new StringBuilder();
+        bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote)
+          .append(" FROM ")
+          .append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS"))
+          .append(quote)
+          .append(" WHERE ")
+          .append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname)
+          .append("' AND ").append(quote).append("TABLE_NAME").append(quote)
+          .append(" = '").append(ci.tableName).append("'");
+        if (ci.partName != null) {
+          bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '")
+            .append(ci.partName).append("'");
+        }
+        String s = bldr.toString();
+
+      /*String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" :
+          "PART_COL_STATS")
+         + " WHERE DB_NAME='" + ci.dbname + "' AND TABLE_NAME='" + ci.tableName + "'"
+        + (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'");*/
+        LOG.debug("Going to execute <" + s + ">");
+        rs = stmt.executeQuery(s);
+        List<String> columns = new ArrayList<>();
+        while (rs.next()) {
+          columns.add(rs.getString(1));
+        }
+        LOG.debug("Found columns to update stats: " + columns + " on " + ci.tableName +
+          (ci.partName == null ? "" : "/" + ci.partName));
+        dbConn.commit();
+        return columns;
+      } catch (SQLException e) {
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "findColumnsWithStats(" + ci.tableName +
+          (ci.partName == null ? "" : "/" + ci.partName) + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException ex) {
+      return findColumnsWithStats(ci);
+    }
+  }
+
+  /**
+   * Record the highest txn id that the {@code ci} compaction job will pay attention to.
+   * This is the highest resolved txn id, i.e. such that there are no open txns with lower ids.
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public void setCompactionHighestTxnId(CompactionInfo ci, long highestTxnId) throws MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        int updCount = stmt.executeUpdate("UPDATE COMPACTION_QUEUE SET CQ_HIGHEST_TXN_ID = " + highestTxnId +
+          " WHERE CQ_ID = " + ci.id);
+        if(updCount != 1) {
+          throw new IllegalStateException("Could not find record in COMPACTION_QUEUE for " + ci);
+        }
+        dbConn.commit();
+      } catch (SQLException e) {
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "setCompactionHighestTxnId(" + ci + "," + highestTxnId + ")");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(null, stmt, dbConn);
+      }
+    } catch (RetryException ex) {
+      setCompactionHighestTxnId(ci, highestTxnId);
+    }
+  }
+  private static class RetentionCounters {
+    int attemptedRetention = 0;
+    int failedRetention = 0;
+    int succeededRetention = 0;
+    RetentionCounters(int attemptedRetention, int failedRetention, int succeededRetention) {
+      this.attemptedRetention = attemptedRetention;
+      this.failedRetention = failedRetention;
+      this.succeededRetention = succeededRetention;
+    }
+  }
+  private void checkForDeletion(List<Long> deleteSet, CompactionInfo ci, RetentionCounters rc) {
+    switch (ci.state) {
+      case ATTEMPTED_STATE:
+        if(--rc.attemptedRetention < 0) {
+          deleteSet.add(ci.id);
+        }
+        break;
+      case FAILED_STATE:
+        if(--rc.failedRetention < 0) {
+          deleteSet.add(ci.id);
+        }
+        break;
+      case SUCCEEDED_STATE:
+        if(--rc.succeededRetention < 0) {
+          deleteSet.add(ci.id);
+        }
+        break;
+      default:
+        //do nothing to hanlde future RU/D where we may want to add new state types
+    }
+  }
+
+  /**
+   * For any given compactable entity (partition; table if not partitioned) the history of compactions
+   * may look like "sssfffaaasffss", for example.  The idea is to retain the tail (most recent) of the
+   * history such that a configurable number of each type of state is present.  Any other entries
+   * can be purged.  This scheme has advantage of always retaining the last failure/success even if
+   * it's not recent.
+   * @throws MetaException
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void purgeCompactionHistory() throws MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    List<Long> deleteSet = new ArrayList<>();
+    RetentionCounters rc = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        /*cc_id is monotonically increasing so for any entity sorts in order of compaction history,
+        thus this query groups by entity and withing group sorts most recent first*/
+        rs = stmt.executeQuery("select cc_id, cc_database, cc_table, cc_partition, cc_state from " +
+          "COMPLETED_COMPACTIONS order by cc_database, cc_table, cc_partition, cc_id desc");
+        String lastCompactedEntity = null;
+        /*In each group, walk from most recent and count occurences of each state type.  Once you
+        * have counted enough (for each state) to satisfy retention policy, delete all other
+        * instances of this status.*/
+        while(rs.next()) {
+          CompactionInfo ci = new CompactionInfo(rs.getLong(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5).charAt(0));
+          if(!ci.getFullPartitionName().equals(lastCompactedEntity)) {
+            lastCompactedEntity = ci.getFullPartitionName();
+            rc = new RetentionCounters(MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED),
+              getFailedCompactionRetention(),
+              MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED));
+          }
+          checkForDeletion(deleteSet, ci, rc);
+        }
+        close(rs);
+
+        if (deleteSet.size() <= 0) {
+          return;
+        }
+
+        List<String> queries = new ArrayList<>();
+
+        StringBuilder prefix = new StringBuilder();
+        StringBuilder suffix = new StringBuilder();
+
+        prefix.append("delete from COMPLETED_COMPACTIONS where ");
+        suffix.append("");
+
+        TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, deleteSet, "cc_id", false, false);
+
+        for (String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          int count = stmt.executeUpdate(query);
+          LOG.debug("Removed " + count + " records from COMPLETED_COMPACTIONS");
+        }
+        dbConn.commit();
+      } catch (SQLException e) {
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "purgeCompactionHistory()");
+        throw new MetaException("Unable to connect to transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException ex) {
+      purgeCompactionHistory();
+    }
+  }
+  /**
+   * this ensures that the number of failed compaction entries retained is > than number of failed
+   * compaction threshold which prevents new compactions from being scheduled.
+   */
+  private int getFailedCompactionRetention() {
+    int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
+    int failedRetention = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED);
+    if(failedRetention < failedThreshold) {
+      LOG.warn("Invalid configuration " + ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.varname +
+        "=" + failedRetention + " < " + ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED + "=" +
+        failedRetention + ".  Will use " + ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.varname +
+        "=" + failedRetention);
+      failedRetention = failedThreshold;
+    }
+    return failedRetention;
+  }
+  /**
+   * Returns {@code true} if there already exists sufficient number of consecutive failures for
+   * this table/partition so that no new automatic compactions will be scheduled.
+   * User initiated compactions don't do this check.
+   *
+   * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
+   * That would be a meta operations, i.e. first find all partitions for this table (which have 
+   * txn info) and schedule each compaction separately.  This avoids complications in this logic.
+   */
+  @Override
+  @RetrySemantics.ReadOnly
+  public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        rs = stmt.executeQuery("select CC_STATE from COMPLETED_COMPACTIONS where " +
+          "CC_DATABASE = " + quoteString(ci.dbname) + " and " +
+          "CC_TABLE = " + quoteString(ci.tableName) +
+          (ci.partName != null ? "and CC_PARTITION = " + quoteString(ci.partName) : "") +
+          " and CC_STATE != " + quoteChar(ATTEMPTED_STATE) + " order by CC_ID desc");
+        int numFailed = 0;
+        int numTotal = 0;
+        int failedThreshold = MetastoreConf.getIntVar(conf, ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
+        while(rs.next() && ++numTotal <= failedThreshold) {
+          if(rs.getString(1).charAt(0) == FAILED_STATE) {
+            numFailed++;
+          }
+          else {
+            numFailed--;
+          }
+        }
+        return numFailed == failedThreshold;
+      }
+      catch (SQLException e) {
+        LOG.error("Unable to delete from compaction queue " + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "checkFailedCompactions(" + ci + ")");
+        LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(e));
+        return false;//weren't able to check
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return checkFailedCompactions(ci);
+    }
+  }
+  /**
+   * If there is an entry in compaction_queue with ci.id, remove it
+   * Make entry in completed_compactions with status 'f'.
+   * If there is no entry in compaction_queue, it means Initiator failed to even schedule a compaction,
+   * which we record as ATTEMPTED_STATE entry in history.
+   */
+  @Override
+  @RetrySemantics.CannotRetry
+  public void markFailed(CompactionInfo ci) throws MetaException {//todo: this should not throw
+    //todo: this should take "comment" as parameter to set in CC_META_INFO to provide some context for the failure
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      PreparedStatement pStmt = null;
+      ResultSet rs = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + ci.id);
+        if(rs.next()) {
+          ci = CompactionInfo.loadFullFromCompactionQueue(rs);
+          String s = "delete from COMPACTION_QUEUE where cq_id = " + ci.id;
+          LOG.debug("Going to execute update <" + s + ">");
+          int updCnt = stmt.executeUpdate(s);
+        }
+        else {
+          if(ci.id > 0) {
+            //the record with valid CQ_ID has disappeared - this is a sign of something wrong
+            throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
+          }
+        }
+        if(ci.id == 0) {
+          //The failure occurred before we even made an entry in COMPACTION_QUEUE
+          //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
+          ci.id = generateCompactionQueueId(stmt);
+          //mostly this indicates that the Initiator is paying attention to some table even though
+          //compactions are not happening.
+          ci.state = ATTEMPTED_STATE;
+          //this is not strictly accurate, but 'type' cannot be null.
+          if(ci.type == null) { ci.type = CompactionType.MINOR; }
+          ci.start = getDbTime(dbConn);
+        }
+        else {
+          ci.state = FAILED_STATE;
+        }
+        close(rs, stmt, null);
+
+        pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
+        CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
+        int updCount = pStmt.executeUpdate();
+        LOG.debug("Going to commit");
+        closeStmt(pStmt);
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.warn("markFailed(" + ci.id + "):" + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        try {
+          checkRetryable(dbConn, e, "markFailed(" + ci + ")");
+        }
+        catch(MetaException ex) {
+          LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(ex));
+        }
+        LOG.error("markFailed(" + ci + ") failed: " + e.getMessage(), e);
+      } finally {
+        close(rs, stmt, null);
+        close(null, pStmt, dbConn);
+      }
+    } catch (RetryException e) {
+      markFailed(ci);
+    }
+  }
+  @Override
+  @RetrySemantics.Idempotent
+  public void setHadoopJobId(String hadoopJobId, long id) {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "update COMPACTION_QUEUE set CQ_HADOOP_JOB_ID = " + quoteString(hadoopJobId) + " WHERE CQ_ID = " + id;
+        LOG.debug("Going to execute <" + s + ">");
+        int updateCount = stmt.executeUpdate(s);
+        LOG.debug("Going to commit");
+        closeStmt(stmt);
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.warn("setHadoopJobId(" + hadoopJobId + "," + id + "):" + e.getMessage());
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        try {
+          checkRetryable(dbConn, e, "setHadoopJobId(" + hadoopJobId + "," + id + ")");
+        }
+        catch(MetaException ex) {
+          LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(ex));
+        }
+        LOG.error("setHadoopJobId(" + hadoopJobId + "," + id + ") failed: " + e.getMessage(), e);
+      } finally {
+        close(null, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      setHadoopJobId(hadoopJobId, id);
+    }
+  }
+}
+
+

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
new file mode 100644
index 0000000..d09c958
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
@@ -0,0 +1,381 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.SQLTransactionRollbackException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility methods for creating and destroying txn database/schema, plus methods for
+ * querying against metastore tables.
+ * Placed here in a separate class so it can be shared across unit tests.
+ */
+public final class TxnDbUtil {
+
+  static final private Logger LOG = LoggerFactory.getLogger(TxnDbUtil.class.getName());
+  private static final String TXN_MANAGER = "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager";
+
+  private static int deadlockCnt = 0;
+
+  private TxnDbUtil() {
+    throw new UnsupportedOperationException("Can't initialize class");
+  }
+
+  /**
+   * Set up the configuration so it will use the DbTxnManager, concurrency will be set to true,
+   * and the JDBC configs will be set for putting the transaction and lock info in the embedded
+   * metastore.
+   *
+   * @param conf HiveConf to add these values to
+   */
+  public static void setConfValues(Configuration conf) {
+    MetastoreConf.setVar(conf, ConfVars.HIVE_TXN_MANAGER, TXN_MANAGER);
+    MetastoreConf.setBoolVar(conf, ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
+  }
+
+  public static void prepDb(Configuration conf) throws Exception {
+    // This is a bogus hack because it copies the contents of the SQL file
+    // intended for creating derby databases, and thus will inexorably get
+    // out of date with it.  I'm open to any suggestions on how to make this
+    // read the file in a build friendly way.
+
+    Connection conn = null;
+    Statement stmt = null;
+    try {
+      conn = getConnection(conf);
+      stmt = conn.createStatement();
+      stmt.execute("CREATE TABLE TXNS (" +
+          "  TXN_ID bigint PRIMARY KEY," +
+          "  TXN_STATE char(1) NOT NULL," +
+          "  TXN_STARTED bigint NOT NULL," +
+          "  TXN_LAST_HEARTBEAT bigint NOT NULL," +
+          "  TXN_USER varchar(128) NOT NULL," +
+          "  TXN_HOST varchar(128) NOT NULL)");
+
+      stmt.execute("CREATE TABLE TXN_COMPONENTS (" +
+          "  TC_TXNID bigint REFERENCES TXNS (TXN_ID)," +
+          "  TC_DATABASE varchar(128) NOT NULL," +
+          "  TC_TABLE varchar(128)," +
+          "  TC_PARTITION varchar(767)," +
+          "  TC_OPERATION_TYPE char(1) NOT NULL)");
+      stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" +
+          "  CTC_TXNID bigint," +
+          "  CTC_DATABASE varchar(128) NOT NULL," +
+          "  CTC_TABLE varchar(128)," +
+          "  CTC_PARTITION varchar(767))");
+      stmt.execute("CREATE TABLE NEXT_TXN_ID (" + "  NTXN_NEXT bigint NOT NULL)");
+      stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)");
+      stmt.execute("CREATE TABLE HIVE_LOCKS (" +
+          " HL_LOCK_EXT_ID bigint NOT NULL," +
+          " HL_LOCK_INT_ID bigint NOT NULL," +
+          " HL_TXNID bigint," +
+          " HL_DB varchar(128) NOT NULL," +
+          " HL_TABLE varchar(128)," +
+          " HL_PARTITION varchar(767)," +
+          " HL_LOCK_STATE char(1) NOT NULL," +
+          " HL_LOCK_TYPE char(1) NOT NULL," +
+          " HL_LAST_HEARTBEAT bigint NOT NULL," +
+          " HL_ACQUIRED_AT bigint," +
+          " HL_USER varchar(128) NOT NULL," +
+          " HL_HOST varchar(128) NOT NULL," +
+          " HL_HEARTBEAT_COUNT integer," +
+          " HL_AGENT_INFO varchar(128)," +
+          " HL_BLOCKEDBY_EXT_ID bigint," +
+          " HL_BLOCKEDBY_INT_ID bigint," +
+        " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))");
+      stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)");
+
+      stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)");
+      stmt.execute("INSERT INTO NEXT_LOCK_ID VALUES(1)");
+
+      stmt.execute("CREATE TABLE COMPACTION_QUEUE (" +
+          " CQ_ID bigint PRIMARY KEY," +
+          " CQ_DATABASE varchar(128) NOT NULL," +
+          " CQ_TABLE varchar(128) NOT NULL," +
+          " CQ_PARTITION varchar(767)," +
+          " CQ_STATE char(1) NOT NULL," +
+          " CQ_TYPE char(1) NOT NULL," +
+          " CQ_TBLPROPERTIES varchar(2048)," +
+          " CQ_WORKER_ID varchar(128)," +
+          " CQ_START bigint," +
+          " CQ_RUN_AS varchar(128)," +
+          " CQ_HIGHEST_TXN_ID bigint," +
+          " CQ_META_INFO varchar(2048) for bit data," +
+          " CQ_HADOOP_JOB_ID varchar(32))");
+
+      stmt.execute("CREATE TABLE NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT bigint NOT NULL)");
+      stmt.execute("INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1)");
+      
+      stmt.execute("CREATE TABLE COMPLETED_COMPACTIONS (" +
+        " CC_ID bigint PRIMARY KEY," +
+        " CC_DATABASE varchar(128) NOT NULL," +
+        " CC_TABLE varchar(128) NOT NULL," +
+        " CC_PARTITION varchar(767)," +
+        " CC_STATE char(1) NOT NULL," +
+        " CC_TYPE char(1) NOT NULL," +
+        " CC_TBLPROPERTIES varchar(2048)," +
+        " CC_WORKER_ID varchar(128)," +
+        " CC_START bigint," +
+        " CC_END bigint," +
+        " CC_RUN_AS varchar(128)," +
+        " CC_HIGHEST_TXN_ID bigint," +
+        " CC_META_INFO varchar(2048) for bit data," +
+        " CC_HADOOP_JOB_ID varchar(32))");
+      
+      stmt.execute("CREATE TABLE AUX_TABLE (" +
+        " MT_KEY1 varchar(128) NOT NULL," +
+        " MT_KEY2 bigint NOT NULL," +
+        " MT_COMMENT varchar(255)," +
+        " PRIMARY KEY(MT_KEY1, MT_KEY2))");
+      
+      stmt.execute("CREATE TABLE WRITE_SET (" +
+        " WS_DATABASE varchar(128) NOT NULL," +
+        " WS_TABLE varchar(128) NOT NULL," +
+        " WS_PARTITION varchar(767)," +
+        " WS_TXNID bigint NOT NULL," +
+        " WS_COMMIT_ID bigint NOT NULL," +
+        " WS_OPERATION_TYPE char(1) NOT NULL)"
+      );
+    } catch (SQLException e) {
+      try {
+        conn.rollback();
+      } catch (SQLException re) {
+        LOG.error("Error rolling back: " + re.getMessage());
+      }
+
+      // This might be a deadlock, if so, let's retry
+      if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) {
+        LOG.warn("Caught deadlock, retrying db creation");
+        prepDb(conf);
+      } else {
+        throw e;
+      }
+    } finally {
+      deadlockCnt = 0;
+      closeResources(conn, stmt, null);
+    }
+  }
+
+  public static void cleanDb(Configuration conf) throws Exception {
+    int retryCount = 0;
+    while(++retryCount <= 3) {
+      boolean success = true;
+      Connection conn = null;
+      Statement stmt = null;
+      try {
+        conn = getConnection(conf);
+        stmt = conn.createStatement();
+
+        // We want to try these, whether they succeed or fail.
+        try {
+          stmt.execute("DROP INDEX HL_TXNID_INDEX");
+        } catch (SQLException e) {
+          if(!("42X65".equals(e.getSQLState()) && 30000 == e.getErrorCode())) {
+            //42X65/3000 means index doesn't exist
+            LOG.error("Unable to drop index HL_TXNID_INDEX " + e.getMessage() +
+              "State=" + e.getSQLState() + " code=" + e.getErrorCode() + " retryCount=" + retryCount);
+            success = false;
+          }
+        }
+
+        success &= dropTable(stmt, "TXN_COMPONENTS", retryCount);
+        success &= dropTable(stmt, "COMPLETED_TXN_COMPONENTS", retryCount);
+        success &= dropTable(stmt, "TXNS", retryCount);
+        success &= dropTable(stmt, "NEXT_TXN_ID", retryCount);
+        success &= dropTable(stmt, "HIVE_LOCKS", retryCount);
+        success &= dropTable(stmt, "NEXT_LOCK_ID", retryCount);
+        success &= dropTable(stmt, "COMPACTION_QUEUE", retryCount);
+        success &= dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID", retryCount);
+        success &= dropTable(stmt, "COMPLETED_COMPACTIONS", retryCount);
+        success &= dropTable(stmt, "AUX_TABLE", retryCount);
+        success &= dropTable(stmt, "WRITE_SET", retryCount);
+      } finally {
+        closeResources(conn, stmt, null);
+      }
+      if(success) {
+        return;
+      }
+    }
+    throw new RuntimeException("Failed to clean up txn tables");
+  }
+
+  private static boolean dropTable(Statement stmt, String name, int retryCount) throws SQLException {
+    for (int i = 0; i < 3; i++) {
+      try {
+        stmt.execute("DROP TABLE " + name);
+        LOG.debug("Successfully dropped table " + name);
+        return true;
+      } catch (SQLException e) {
+        if ("42Y55".equals(e.getSQLState()) && 30000 == e.getErrorCode()) {
+          LOG.debug("Not dropping " + name + " because it doesn't exist");
+          //failed because object doesn't exist
+          return true;
+        }
+        if ("X0Y25".equals(e.getSQLState()) && 30000 == e.getErrorCode()) {
+          // Intermittent failure
+          LOG.warn("Intermittent drop failure, retrying, try number " + i);
+          continue;
+        }
+        LOG.error("Unable to drop table " + name + ": " + e.getMessage() +
+            " State=" + e.getSQLState() + " code=" + e.getErrorCode() + " retryCount=" + retryCount);
+      }
+    }
+    LOG.error("Failed to drop table, don't know why");
+    return false;
+  }
+
+  /**
+   * A tool to count the number of partitions, tables,
+   * and databases locked by a particular lockId.
+   *
+   * @param lockId lock id to look for lock components
+   *
+   * @return number of components, or 0 if there is no lock
+   */
+  public static int countLockComponents(Configuration conf, long lockId) throws Exception {
+    Connection conn = null;
+    PreparedStatement stmt = null;
+    ResultSet rs = null;
+    try {
+      conn = getConnection(conf);
+      stmt = conn.prepareStatement("SELECT count(*) FROM hive_locks WHERE hl_lock_ext_id = ?");
+      stmt.setLong(1, lockId);
+      rs = stmt.executeQuery();
+      if (!rs.next()) {
+        return 0;
+      }
+      return rs.getInt(1);
+    } finally {
+      closeResources(conn, stmt, rs);
+    }
+  }
+
+  /**
+   * Utility method used to run COUNT queries like "select count(*) from ..." against metastore tables
+   * @param countQuery countQuery text
+   * @return count countQuery result
+   * @throws Exception
+   */
+  public static int countQueryAgent(Configuration conf, String countQuery) throws Exception {
+    Connection conn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      conn = getConnection(conf);
+      stmt = conn.createStatement();
+      rs = stmt.executeQuery(countQuery);
+      if (!rs.next()) {
+        return 0;
+      }
+      return rs.getInt(1);
+    } finally {
+      closeResources(conn, stmt, rs);
+    }
+  }
+  @VisibleForTesting
+  public static String queryToString(Configuration conf, String query) throws Exception {
+    return queryToString(conf, query, true);
+  }
+  public static String queryToString(Configuration conf, String query, boolean includeHeader)
+      throws Exception {
+    Connection conn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    StringBuilder sb = new StringBuilder();
+    try {
+      conn = getConnection(conf);
+      stmt = conn.createStatement();
+      rs = stmt.executeQuery(query);
+      ResultSetMetaData rsmd = rs.getMetaData();
+      if(includeHeader) {
+        for (int colPos = 1; colPos <= rsmd.getColumnCount(); colPos++) {
+          sb.append(rsmd.getColumnName(colPos)).append("   ");
+        }
+        sb.append('\n');
+      }
+      while(rs.next()) {
+        for (int colPos = 1; colPos <= rsmd.getColumnCount(); colPos++) {
+          sb.append(rs.getObject(colPos)).append("   ");
+        }
+        sb.append('\n');
+      }
+    } finally {
+      closeResources(conn, stmt, rs);
+    }
+    return sb.toString();
+  }
+
+  static Connection getConnection(Configuration conf) throws Exception {
+    String jdbcDriver = MetastoreConf.getVar(conf, ConfVars.CONNECTION_DRIVER);
+    Driver driver = (Driver) Class.forName(jdbcDriver).newInstance();
+    Properties prop = new Properties();
+    String driverUrl = MetastoreConf.getVar(conf, ConfVars.CONNECTURLKEY);
+    String user = MetastoreConf.getVar(conf, ConfVars.CONNECTION_USER_NAME);
+    String passwd = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
+    prop.setProperty("user", user);
+    prop.setProperty("password", passwd);
+    Connection conn = driver.connect(driverUrl, prop);
+    conn.setAutoCommit(true);
+    return conn;
+  }
+
+  static void closeResources(Connection conn, Statement stmt, ResultSet rs) {
+    if (rs != null) {
+      try {
+        rs.close();
+      } catch (SQLException e) {
+        LOG.error("Error closing ResultSet: " + e.getMessage());
+      }
+    }
+
+    if (stmt != null) {
+      try {
+        stmt.close();
+      } catch (SQLException e) {
+        System.err.println("Error closing Statement: " + e.getMessage());
+      }
+    }
+
+    if (conn != null) {
+      try {
+        conn.rollback();
+      } catch (SQLException e) {
+        System.err.println("Error rolling back: " + e.getMessage());
+      }
+      try {
+        conn.close();
+      } catch (SQLException e) {
+        System.err.println("Error closing Connection: " + e.getMessage());
+      }
+    }
+  }
+}


[08/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
deleted file mode 100644
index 3eb3827..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnStore.java
+++ /dev/null
@@ -1,433 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.common.classification.RetrySemantics;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.*;
-
-import java.sql.SQLException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-/**
- * A handler to answer transaction related calls that come into the metastore
- * server.
- *
- * Note on log messages:  Please include txnid:X and lockid info using
- * {@link org.apache.hadoop.hive.common.JavaUtils#txnIdToString(long)}
- * and {@link org.apache.hadoop.hive.common.JavaUtils#lockIdToString(long)} in all messages.
- * The txnid:X and lockid:Y matches how Thrift object toString() methods are generated,
- * so keeping the format consistent makes grep'ing the logs much easier.
- *
- * Note on HIVE_LOCKS.hl_last_heartbeat.
- * For locks that are part of transaction, we set this 0 (would rather set it to NULL but
- * Currently the DB schema has this NOT NULL) and only update/read heartbeat from corresponding
- * transaction in TXNS.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface TxnStore {
-
-  enum MUTEX_KEY {Initiator, Cleaner, HouseKeeper, CompactionHistory, CheckLock,
-    WriteSetCleaner, CompactionScheduler}
-  // Compactor states (Should really be enum)
-  String INITIATED_RESPONSE = "initiated";
-  String WORKING_RESPONSE = "working";
-  String CLEANING_RESPONSE = "ready for cleaning";
-  String FAILED_RESPONSE = "failed";
-  String SUCCEEDED_RESPONSE = "succeeded";
-  String ATTEMPTED_RESPONSE = "attempted";
-
-  public static final int TIMED_OUT_TXN_ABORT_BATCH_SIZE = 50000;
-
-  public void setConf(HiveConf conf);
-
-  /**
-   * Get information about open transactions.  This gives extensive information about the
-   * transactions rather than just the list of transactions.  This should be used when the need
-   * is to see information about the transactions (e.g. show transactions).
-   * @return information about open transactions
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException;
-
-  /**
-   * Get list of valid transactions.  This gives just the list of transactions that are open.
-   * @return list of open transactions, as well as a high water mark.
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public GetOpenTxnsResponse getOpenTxns() throws MetaException;
-
-  /**
-   * Get the count for open transactions.
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public void countOpenTxns() throws MetaException;
-
-  /**
-   * Open a set of transactions
-   * @param rqst request to open transactions
-   * @return information on opened transactions
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException;
-
-  /**
-   * Abort (rollback) a transaction.
-   * @param rqst info on transaction to abort
-   * @throws NoSuchTxnException
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException, TxnAbortedException;
-
-  /**
-   * Abort (rollback) a list of transactions in one request.
-   * @param rqst info on transactions to abort
-   * @throws NoSuchTxnException
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaException;
-
-  /**
-   * Commit a transaction
-   * @param rqst info on transaction to commit
-   * @throws NoSuchTxnException
-   * @throws TxnAbortedException
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public void commitTxn(CommitTxnRequest rqst)
-    throws NoSuchTxnException, TxnAbortedException,  MetaException;
-
-  /**
-   * Obtain a lock.
-   * @param rqst information on the lock to obtain.  If the requester is part of a transaction
-   *             the txn information must be included in the lock request.
-   * @return info on the lock, including whether it was obtained.
-   * @throws NoSuchTxnException
-   * @throws TxnAbortedException
-   * @throws MetaException
-   */
-  @RetrySemantics.CannotRetry
-  public LockResponse lock(LockRequest rqst)
-    throws NoSuchTxnException, TxnAbortedException, MetaException;
-
-  /**
-   * Check whether a lock has been obtained.  This is used after {@link #lock} returned a wait
-   * state.
-   * @param rqst info on the lock to check
-   * @return info on the state of the lock
-   * @throws NoSuchTxnException
-   * @throws NoSuchLockException
-   * @throws TxnAbortedException
-   * @throws MetaException
-   */
-  @RetrySemantics.SafeToRetry
-  public LockResponse checkLock(CheckLockRequest rqst)
-    throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException;
-
-  /**
-   * Unlock a lock.  It is not legal to call this if the caller is part of a txn.  In that case
-   * the txn should be committed or aborted instead.  (Note someday this will change since
-   * multi-statement transactions will allow unlocking in the transaction.)
-   * @param rqst lock to unlock
-   * @throws NoSuchLockException
-   * @throws TxnOpenException
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public void unlock(UnlockRequest rqst)
-    throws NoSuchLockException, TxnOpenException, MetaException;
-
-  /**
-   * Get information on current locks.
-   * @param rqst lock information to retrieve
-   * @return lock information.
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException;
-
-  /**
-   * Send a heartbeat for a lock or a transaction
-   * @param ids lock and/or txn id to heartbeat
-   * @throws NoSuchTxnException
-   * @throws NoSuchLockException
-   * @throws TxnAbortedException
-   * @throws MetaException
-   */
-  @RetrySemantics.SafeToRetry
-  public void heartbeat(HeartbeatRequest ids)
-    throws NoSuchTxnException,  NoSuchLockException, TxnAbortedException, MetaException;
-
-  /**
-   * Heartbeat a group of transactions together
-   * @param rqst set of transactions to heartbat
-   * @return info on txns that were heartbeated
-   * @throws MetaException
-   */
-  @RetrySemantics.SafeToRetry
-  public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst)
-    throws MetaException;
-
-  /**
-   * Submit a compaction request into the queue.  This is called when a user manually requests a
-   * compaction.
-   * @param rqst information on what to compact
-   * @return id of the compaction that has been started or existing id if this resource is already scheduled
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public CompactionResponse compact(CompactionRequest rqst) throws MetaException;
-
-  /**
-   * Show list of current compactions
-   * @param rqst info on which compactions to show
-   * @return compaction information
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException;
-
-  /**
-   * Add information on a set of dynamic partitions that participated in a transaction.
-   * @param rqst dynamic partition info.
-   * @throws NoSuchTxnException
-   * @throws TxnAbortedException
-   * @throws MetaException
-   */
-  @RetrySemantics.SafeToRetry
-  public void addDynamicPartitions(AddDynamicPartitions rqst)
-      throws NoSuchTxnException,  TxnAbortedException, MetaException;
-
-  /**
-   * Clean up corresponding records in metastore tables
-   * @param type Hive object type
-   * @param db database object
-   * @param table table object
-   * @param partitionIterator partition iterator
-   * @throws MetaException
-   */
-  @RetrySemantics.Idempotent
-  public void cleanupRecords(HiveObjectType type, Database db, Table table,
-                             Iterator<Partition> partitionIterator) throws MetaException;
-
-  /**
-   * Timeout transactions and/or locks.  This should only be called by the compactor.
-   */
-  @RetrySemantics.Idempotent
-  public void performTimeOuts();
-
-  /**
-   * This will look through the completed_txn_components table and look for partitions or tables
-   * that may be ready for compaction.  Also, look through txns and txn_components tables for
-   * aborted transactions that we should add to the list.
-   * @param maxAborted Maximum number of aborted queries to allow before marking this as a
-   *                   potential compaction.
-   * @return list of CompactionInfo structs.  These will not have id, type,
-   * or runAs set since these are only potential compactions not actual ones.
-   */
-  @RetrySemantics.ReadOnly
-  public Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException;
-
-  /**
-   * Sets the user to run as.  This is for the case
-   * where the request was generated by the user and so the worker must set this value later.
-   * @param cq_id id of this entry in the queue
-   * @param user user to run the jobs as
-   */
-  @RetrySemantics.Idempotent
-  public void setRunAs(long cq_id, String user) throws MetaException;
-
-  /**
-   * This will grab the next compaction request off of
-   * the queue, and assign it to the worker.
-   * @param workerId id of the worker calling this, will be recorded in the db
-   * @return an info element for this compaction request, or null if there is no work to do now.
-   */
-  @RetrySemantics.ReadOnly
-  public CompactionInfo findNextToCompact(String workerId) throws MetaException;
-
-  /**
-   * This will mark an entry in the queue as compacted
-   * and put it in the ready to clean state.
-   * @param info info on the compaction entry to mark as compacted.
-   */
-  @RetrySemantics.SafeToRetry
-  public void markCompacted(CompactionInfo info) throws MetaException;
-
-  /**
-   * Find entries in the queue that are ready to
-   * be cleaned.
-   * @return information on the entry in the queue.
-   */
-  @RetrySemantics.ReadOnly
-  public List<CompactionInfo> findReadyToClean() throws MetaException;
-
-  /**
-   * This will remove an entry from the queue after
-   * it has been compacted.
-   * 
-   * @param info info on the compaction entry to remove
-   */
-  @RetrySemantics.CannotRetry
-  public void markCleaned(CompactionInfo info) throws MetaException;
-
-  /**
-   * Mark a compaction entry as failed.  This will move it to the compaction history queue with a
-   * failed status.  It will NOT clean up aborted transactions in the table/partition associated
-   * with this compaction.
-   * @param info information on the compaction that failed.
-   * @throws MetaException
-   */
-  @RetrySemantics.CannotRetry
-  public void markFailed(CompactionInfo info) throws MetaException;
-
-  /**
-   * Clean up aborted transactions from txns that have no components in txn_components.  The reson such
-   * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
-   * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
-   */
-  @RetrySemantics.SafeToRetry
-  public void cleanEmptyAbortedTxns() throws MetaException;
-
-  /**
-   * This will take all entries assigned to workers
-   * on a host return them to INITIATED state.  The initiator should use this at start up to
-   * clean entries from any workers that were in the middle of compacting when the metastore
-   * shutdown.  It does not reset entries from worker threads on other hosts as those may still
-   * be working.
-   * @param hostname Name of this host.  It is assumed this prefixes the thread's worker id,
-   *                 so that like hostname% will match the worker id.
-   */
-  @RetrySemantics.Idempotent
-  public void revokeFromLocalWorkers(String hostname) throws MetaException;
-
-  /**
-   * This call will return all compaction queue
-   * entries assigned to a worker but over the timeout back to the initiated state.
-   * This should be called by the initiator on start up and occasionally when running to clean up
-   * after dead threads.  At start up {@link #revokeFromLocalWorkers(String)} should be called
-   * first.
-   * @param timeout number of milliseconds since start time that should elapse before a worker is
-   *                declared dead.
-   */
-  @RetrySemantics.Idempotent
-  public void revokeTimedoutWorkers(long timeout) throws MetaException;
-
-  /**
-   * Queries metastore DB directly to find columns in the table which have statistics information.
-   * If {@code ci} includes partition info then per partition stats info is examined, otherwise
-   * table level stats are examined.
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public List<String> findColumnsWithStats(CompactionInfo ci) throws MetaException;
-
-  /**
-   * Record the highest txn id that the {@code ci} compaction job will pay attention to.
-   */
-  @RetrySemantics.Idempotent
-  public void setCompactionHighestTxnId(CompactionInfo ci, long highestTxnId) throws MetaException;
-
-  /**
-   * For any given compactable entity (partition, table if not partitioned) the history of compactions
-   * may look like "sssfffaaasffss", for example.  The idea is to retain the tail (most recent) of the
-   * history such that a configurable number of each type of state is present.  Any other entries
-   * can be purged.  This scheme has advantage of always retaining the last failure/success even if
-   * it's not recent.
-   * @throws MetaException
-   */
-  @RetrySemantics.SafeToRetry
-  public void purgeCompactionHistory() throws MetaException;
-
-  /**
-   * WriteSet tracking is used to ensure proper transaction isolation.  This method deletes the 
-   * transaction metadata once it becomes unnecessary.  
-   */
-  @RetrySemantics.SafeToRetry
-  public void performWriteSetGC();
-
-  /**
-   * Determine if there are enough consecutive failures compacting a table or partition that no
-   * new automatic compactions should be scheduled.  User initiated compactions do not do this
-   * check.
-   * @param ci  Table or partition to check.
-   * @return true if it is ok to compact, false if there have been too many failures.
-   * @throws MetaException
-   */
-  @RetrySemantics.ReadOnly
-  public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException;
-
-  @VisibleForTesting
-  public int numLocksInLockTable() throws SQLException, MetaException;
-
-  @VisibleForTesting
-  long setTimeout(long milliseconds);
-
-  @RetrySemantics.Idempotent
-  public MutexAPI getMutexAPI();
-
-  /**
-   * This is primarily designed to provide coarse grained mutex support to operations running
-   * inside the Metastore (of which there could be several instances).  The initial goal is to 
-   * ensure that various sub-processes of the Compactor don't step on each other.
-   * 
-   * In RDMBS world each {@code LockHandle} uses a java.sql.Connection so use it sparingly.
-   */
-  public static interface MutexAPI {
-    /**
-     * The {@code key} is name of the lock. Will acquire and exclusive lock or block.  It retuns
-     * a handle which must be used to release the lock.  Each invocation returns a new handle.
-     */
-    public LockHandle acquireLock(String key) throws MetaException;
-
-    /**
-     * Same as {@link #acquireLock(String)} but takes an already existing handle as input.  This 
-     * will associate the lock on {@code key} with the same handle.  All locks associated with
-     * the same handle will be released together.
-     * @param handle not NULL
-     */
-    public void acquireLock(String key, LockHandle handle) throws MetaException;
-    public static interface LockHandle {
-      /**
-       * Releases all locks associated with this handle.
-       */
-      public void releaseLocks();
-    }
-  }
-
-  /**
-   * Once a {@link java.util.concurrent.ThreadPoolExecutor} Worker submits a job to the cluster,
-   * it calls this to update the metadata.
-   * @param id {@link CompactionInfo#id}
-   */
-  @RetrySemantics.Idempotent
-  public void setHadoopJobId(String hadoopJobId, long id);
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
deleted file mode 100644
index 30b155f..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.common.ValidCompactorTxnList;
-import org.apache.hadoop.hive.common.ValidReadTxnList;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.MetaStoreUtils;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.api.TxnInfo;
-import org.apache.hadoop.hive.metastore.api.TxnState;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.BitSet;
-import java.util.List;
-import java.util.Map;
-
-public class TxnUtils {
-  private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class);
-
-  /**
-   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a
-   * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
-   * read the files, and thus treats both open and aborted transactions as invalid.
-   * @param txns txn list from the metastore
-   * @param currentTxn Current transaction that the user has open.  If this is greater than 0 it
-   *                   will be removed from the exceptions list so that the user sees his own
-   *                   transaction as valid.
-   * @return a valid txn list.
-   */
-  public static ValidTxnList createValidReadTxnList(GetOpenTxnsResponse txns, long currentTxn) {
-    /*todo: should highWater be min(currentTxn,txns.getTxn_high_water_mark()) assuming currentTxn>0
-     * otherwise if currentTxn=7 and 8 commits before 7, then 7 will see result of 8 which
-     * doesn't make sense for Snapshot Isolation.  Of course for Read Committed, the list should
-     * inlude the latest committed set.*/
-    long highWater = txns.getTxn_high_water_mark();
-    List<Long> open = txns.getOpen_txns();
-    BitSet abortedBits = BitSet.valueOf(txns.getAbortedBits());
-    long[] exceptions = new long[open.size() - (currentTxn > 0 ? 1 : 0)];
-    int i = 0;
-    for(long txn: open) {
-      if (currentTxn > 0 && currentTxn == txn) continue;
-      exceptions[i++] = txn;
-    }
-    if(txns.isSetMin_open_txn()) {
-      return new ValidReadTxnList(exceptions, abortedBits, highWater, txns.getMin_open_txn());
-    }
-    else {
-      return new ValidReadTxnList(exceptions, abortedBits, highWater);
-    }
-  }
-
-  /**
-   * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse} to a
-   * {@link org.apache.hadoop.hive.common.ValidTxnList}.  This assumes that the caller intends to
-   * compact the files, and thus treats only open transactions as invalid.  Additionally any
-   * txnId &gt; highestOpenTxnId is also invalid.  This is to avoid creating something like
-   * delta_17_120 where txnId 80, for example, is still open.
-   * @param txns txn list from the metastore
-   * @return a valid txn list.
-   */
-  public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
-    //highWater is the last txn id that has been allocated
-    long highWater = txns.getTxn_high_water_mark();
-    long minOpenTxn = Long.MAX_VALUE;
-    long[] exceptions = new long[txns.getOpen_txnsSize()];
-    int i = 0;
-    for (TxnInfo txn : txns.getOpen_txns()) {
-      if (txn.getState() == TxnState.OPEN) {
-        minOpenTxn = Math.min(minOpenTxn, txn.getId());
-      }
-      else {
-        //only need aborted since we don't consider anything above minOpenTxn
-        exceptions[i++] = txn.getId();
-      }
-    }
-    if(i < exceptions.length) {
-      exceptions = Arrays.copyOf(exceptions, i);
-    }
-    highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
-    BitSet bitSet = new BitSet(exceptions.length);
-    bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted
-    if(minOpenTxn == Long.MAX_VALUE) {
-      return new ValidCompactorTxnList(exceptions, bitSet, highWater);
-    }
-    else {
-      return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
-    }
-  }
-
-  /**
-   * Get an instance of the TxnStore that is appropriate for this store
-   * @param conf configuration
-   * @return txn store
-   */
-  public static TxnStore getTxnStore(HiveConf conf) {
-    String className = conf.getVar(HiveConf.ConfVars.METASTORE_TXN_STORE_IMPL);
-    try {
-      TxnStore handler = ((Class<? extends TxnHandler>) MetaStoreUtils.getClass(
-        className)).newInstance();
-      handler.setConf(conf);
-      return handler;
-    } catch (Exception e) {
-      LOG.error("Unable to instantiate raw store directly in fastpath mode", e);
-      throw new RuntimeException(e);
-    }
-  }
-
-  /** Checks if a table is a valid ACID table.
-   * Note, users are responsible for using the correct TxnManager. We do not look at
-   * SessionState.get().getTxnMgr().supportsAcid() here
-   * @param table table
-   * @return true if table is a legit ACID table, false otherwise
-   */
-  public static boolean isAcidTable(Table table) {
-    if (table == null) {
-      return false;
-    }
-    Map<String, String> parameters = table.getParameters();
-    String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL);
-    return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
-  }
-
-  /**
-   * Build a query (or queries if one query is too big) with specified "prefix" and "suffix",
-   * while populating the IN list into multiple OR clauses, e.g. id in (1,2,3) OR id in (4,5,6)
-   * For NOT IN case, NOT IN list is broken into multiple AND clauses.
-   * @param queries array of complete query strings
-   * @param prefix part of the query that comes before IN list
-   * @param suffix part of the query that comes after IN list
-   * @param inList the list containing IN list values
-   * @param inColumn column name of IN list operator
-   * @param addParens add a pair of parenthesis outside the IN lists
-   *                  e.g. ( id in (1,2,3) OR id in (4,5,6) )
-   * @param notIn clause to be broken up is NOT IN
-   */
-  public static void buildQueryWithINClause(HiveConf conf, List<String> queries, StringBuilder prefix,
-                                            StringBuilder suffix, List<Long> inList,
-                                            String inColumn, boolean addParens, boolean notIn) {
-    if (inList == null || inList.size() == 0) {
-      throw new IllegalArgumentException("The IN list is empty!");
-    }
-    int batchSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE);
-    int numWholeBatches = inList.size() / batchSize;
-    StringBuilder buf = new StringBuilder();
-    buf.append(prefix);
-    if (addParens) {
-      buf.append("(");
-    }
-    buf.append(inColumn);
-    if (notIn) {
-      buf.append(" not in (");
-    } else {
-      buf.append(" in (");
-    }
-
-    for (int i = 0; i <= numWholeBatches; i++) {
-      if (i * batchSize == inList.size()) {
-        // At this point we just realized we don't need another query
-        break;
-      }
-
-      if (needNewQuery(conf, buf)) {
-        // Wrap up current query string
-        if (addParens) {
-          buf.append(")");
-        }
-        buf.append(suffix);
-        queries.add(buf.toString());
-
-        // Prepare a new query string
-        buf.setLength(0);
-      }
-
-      if (i > 0) {
-        if (notIn) {
-          if (buf.length() == 0) {
-            buf.append(prefix);
-            if (addParens) {
-              buf.append("(");
-            }
-          } else {
-            buf.append(" and ");
-          }
-          buf.append(inColumn);
-          buf.append(" not in (");
-        } else {
-          if (buf.length() == 0) {
-            buf.append(prefix);
-            if (addParens) {
-              buf.append("(");
-            }
-          } else {
-            buf.append(" or ");
-          }
-          buf.append(inColumn);
-          buf.append(" in (");
-        }
-      }
-
-      for (int j = i * batchSize; j < (i + 1) * batchSize && j < inList.size(); j++) {
-        buf.append(inList.get(j)).append(",");
-      }
-      buf.setCharAt(buf.length() - 1, ')');
-    }
-
-    if (addParens) {
-      buf.append(")");
-    }
-    buf.append(suffix);
-    queries.add(buf.toString());
-  }
-
-  /** Estimate if the size of a string will exceed certain limit */
-  private static boolean needNewQuery(HiveConf conf, StringBuilder sb) {
-    int queryMemoryLimit = conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH);
-    // http://www.javamex.com/tutorials/memory/string_memory_usage.shtml
-    long sizeInBytes = 8 * (((sb.length() * 2) + 45) / 8);
-    return sizeInBytes / 1024 > queryMemoryLimit;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java b/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
deleted file mode 100644
index daea544..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/datasource/TestDataSourceProviderFactory.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.jolbox.bonecp.BoneCPDataSource;
-import com.zaxxer.hikari.HikariDataSource;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import javax.sql.DataSource;
-import java.sql.SQLException;
-
-public class TestDataSourceProviderFactory {
-
-  private HiveConf conf;
-
-  @Before
-  public void init() {
-    conf = new HiveConf();
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, "dummyUser");
-    conf.setVar(HiveConf.ConfVars.METASTOREPWD, "dummyPass");
-  }
-
-  @Test
-  public void testNoDataSourceCreatedWithoutProps() throws SQLException {
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNull(dsp);
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-
-    dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNull(dsp);
-  }
-
-  @Test
-  public void testCreateBoneCpDataSource() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-    conf.set(BoneCPDataSourceProvider.BONECP + ".firstProp", "value");
-    conf.set(BoneCPDataSourceProvider.BONECP + ".secondProp", "value");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof BoneCPDataSource);
-  }
-
-  @Test
-  public void testSetBoneCpStringProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-    conf.set(BoneCPDataSourceProvider.BONECP + ".initSQL", "select 1 from dual");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof BoneCPDataSource);
-    Assert.assertEquals("select 1 from dual", ((BoneCPDataSource)ds).getInitSQL());
-  }
-
-  @Test
-  public void testSetBoneCpNumberProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-    conf.set(BoneCPDataSourceProvider.BONECP + ".acquireRetryDelayInMs", "599");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof BoneCPDataSource);
-    Assert.assertEquals(599L, ((BoneCPDataSource)ds).getAcquireRetryDelayInMs());
-  }
-
-  @Test
-  public void testSetBoneCpBooleanProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, BoneCPDataSourceProvider.BONECP);
-    conf.set(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof BoneCPDataSource);
-    Assert.assertEquals(true, ((BoneCPDataSource)ds).isDisableJMX());
-  }
-
-  @Test
-  public void testCreateHikariCpDataSource() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
-    // This is needed to prevent the HikariDataSource from trying to connect to the DB
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof HikariDataSource);
-  }
-
-  @Test
-  public void testSetHikariCpStringProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".connectionInitSql", "select 1 from dual");
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof HikariDataSource);
-    Assert.assertEquals("select 1 from dual", ((HikariDataSource)ds).getConnectionInitSql());
-  }
-
-  @Test
-  public void testSetHikariCpNumberProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".idleTimeout", "59999");
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof HikariDataSource);
-    Assert.assertEquals(59999L, ((HikariDataSource)ds).getIdleTimeout());
-  }
-
-  @Test
-  public void testSetHikariCpBooleanProperty() throws SQLException {
-
-    conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, HikariCPDataSourceProvider.HIKARI);
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".allowPoolSuspension", "false");
-    conf.set(HikariCPDataSourceProvider.HIKARI + ".initializationFailTimeout", "-1");
-
-    DataSourceProvider dsp = DataSourceProviderFactory.getDataSourceProvider(conf);
-    Assert.assertNotNull(dsp);
-
-    DataSource ds = dsp.create(conf);
-    Assert.assertTrue(ds instanceof HikariDataSource);
-    Assert.assertEquals(false, ((HikariDataSource)ds).isAllowPoolSuspension());
-  }
-  @Test(expected = IllegalArgumentException.class)
-  public void testBoneCPConfigCannotBeSet() {
-    conf.addToRestrictList(BoneCPDataSourceProvider.BONECP);
-    conf.verifyAndSet(BoneCPDataSourceProvider.BONECP + ".disableJMX", "true");
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
deleted file mode 100644
index 639669e..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNegative.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.junit.Test;
-
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertTrue;
-
-public class TestTxnHandlerNegative {
-  static final private Logger LOG = LoggerFactory.getLogger(TestTxnHandlerNegative.class);
-
-  /**
-   * this intentionally sets a bad URL for connection to test error handling logic
-   * in TxnHandler
-   * @throws Exception
-   */
-  @Test
-  public void testBadConnection() throws Exception {
-    HiveConf conf = new HiveConf();
-    conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, "blah");
-    RuntimeException e = null;
-    try {
-      TxnUtils.getTxnStore(conf);
-    }
-    catch(RuntimeException ex) {
-      LOG.info("Expected error: " + ex.getMessage(), ex);
-      e = ex;
-    }
-    assertNotNull(e);
-    assertTrue(
-        e.getMessage().contains("No suitable driver found for blah")
-        || e.getMessage().contains("Failed to get driver instance for jdbcUrl=blah")
-    );
-  }
-}


[11/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4a12a56
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4a12a56
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4a12a56

Branch: refs/heads/master
Commit: f4a12a56122a9b72dd60852617cca4e8c9d84cc9
Parents: c1f3d9a
Author: Alan Gates <ga...@hortonworks.com>
Authored: Fri Oct 6 09:56:44 2017 -0700
Committer: Alan Gates <ga...@hortonworks.com>
Committed: Fri Oct 6 09:56:44 2017 -0700

----------------------------------------------------------------------
 .../hive/common/ValidCompactorTxnList.java      |   89 -
 .../hadoop/hive/common/ValidReadTxnList.java    |  237 --
 .../apache/hadoop/hive/common/ValidTxnList.java |  112 -
 .../hive/common/TestValidReadTxnList.java       |  109 -
 .../hive/hcatalog/streaming/TestStreaming.java  |    4 +-
 .../streaming/mutate/StreamingTestUtils.java    |    4 +-
 .../hive/metastore/TestHiveMetaStoreTxns.java   |    4 +-
 .../apache/hadoop/hive/ql/TestAcidOnTez.java    |    5 +-
 .../hive/ql/txn/compactor/TestCompactor.java    |    4 +-
 .../hadoop/hive/metastore/HiveMetaStore.java    |   10 +
 .../datasource/BoneCPDataSourceProvider.java    |   95 -
 .../datasource/DataSourceProvider.java          |   79 -
 .../datasource/DataSourceProviderFactory.java   |   47 -
 .../datasource/HikariCPDataSourceProvider.java  |   97 -
 .../hive/metastore/datasource/package-info.java |   23 -
 .../hive/metastore/tools/SQLGenerator.java      |  172 -
 .../hive/metastore/txn/CompactionInfo.java      |  151 -
 .../metastore/txn/CompactionTxnHandler.java     |  960 -----
 .../hadoop/hive/metastore/txn/TxnDbUtil.java    |  369 --
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 3667 ------------------
 .../hadoop/hive/metastore/txn/TxnStore.java     |  433 ---
 .../hadoop/hive/metastore/txn/TxnUtils.java     |  242 --
 .../TestDataSourceProviderFactory.java          |  173 -
 .../metastore/txn/TestTxnHandlerNegative.java   |   54 -
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |  215 -
 .../txn/TestValidCompactorTxnList.java          |  136 -
 .../hive/ql/txn/AcidOpenTxnsCounterService.java |   75 -
 .../metastore/txn/TestCompactionTxnHandler.java |    4 +-
 .../hive/metastore/txn/TestTxnHandler.java      |    4 +-
 .../txn/TestTxnHandlerNoConnectionPool.java     |    6 +-
 .../apache/hadoop/hive/ql/TestTxnCommands.java  |   23 +-
 .../apache/hadoop/hive/ql/TestTxnCommands2.java |   16 +-
 .../hadoop/hive/ql/TxnCommandsBaseForTests.java |    5 +-
 .../hive/ql/lockmgr/TestDbTxnManager.java       |   28 +-
 .../hive/ql/lockmgr/TestDbTxnManager2.java      |  356 +-
 .../hive/ql/txn/compactor/CompactorTest.java    |    2 +-
 standalone-metastore/pom.xml                    |   20 +
 .../hive/metastore/RunnableConfigurable.java    |   26 +
 .../hive/metastore/conf/MetastoreConf.java      |   63 +-
 .../datasource/BoneCPDataSourceProvider.java    |   94 +
 .../datasource/DataSourceProvider.java          |   79 +
 .../datasource/DataSourceProviderFactory.java   |   47 +
 .../datasource/HikariCPDataSourceProvider.java  |   96 +
 .../hive/metastore/datasource/package-info.java |   23 +
 .../hive/metastore/tools/SQLGenerator.java      |  172 +
 .../txn/AcidOpenTxnsCounterService.java         |   63 +
 .../hive/metastore/txn/CompactionInfo.java      |  151 +
 .../metastore/txn/CompactionTxnHandler.java     |  961 +++++
 .../hadoop/hive/metastore/txn/TxnDbUtil.java    |  381 ++
 .../hadoop/hive/metastore/txn/TxnHandler.java   | 3643 +++++++++++++++++
 .../hadoop/hive/metastore/txn/TxnStore.java     |  420 ++
 .../hadoop/hive/metastore/txn/TxnUtils.java     |  243 ++
 .../hadoop/hive/metastore/utils/JavaUtils.java  |   46 +
 .../hive/metastore/utils/StringableMap.java     |   80 +
 .../hive/metastore/conf/TestMetastoreConf.java  |   17 +
 .../TestDataSourceProviderFactory.java          |  170 +
 .../metastore/txn/TestTxnHandlerNegative.java   |   55 +
 .../hadoop/hive/metastore/txn/TestTxnUtils.java |  216 ++
 storage-api/pom.xml                             |    7 +
 .../hive/common/ValidCompactorTxnList.java      |   89 +
 .../hadoop/hive/common/ValidReadTxnList.java    |  235 ++
 .../apache/hadoop/hive/common/ValidTxnList.java |  112 +
 .../hive/common/TestValidCompactorTxnList.java  |  134 +
 .../hive/common/TestValidReadTxnList.java       |  109 +
 64 files changed, 7979 insertions(+), 7783 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
deleted file mode 100644
index eaa0b34..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common;
-
-import java.util.Arrays;
-import java.util.BitSet;
-
-/**
- * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by the compactor.
- * 
- * Compaction should only include txns up to smallest open txn (exclussive).
- * There may be aborted txns in the snapshot represented by this ValidCompactorTxnList.
- * Thus {@link #isTxnRangeValid(long, long)} returns NONE for any range that inluces any unresolved
- * transactions.  Any txn above {@code highWatermark} is unresolved.
- * These produce the logic we need to assure that the compactor only sees records less than the lowest
- * open transaction when choosing which files to compact, but that it still ignores aborted
- * records when compacting.
- * 
- * See org.apache.hadoop.hive.metastore.txn.TxnUtils#createValidCompactTxnList() for proper
- * way to construct this.
- */
-public class ValidCompactorTxnList extends ValidReadTxnList {
-  public ValidCompactorTxnList() {
-    super();
-  }
-  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
-    this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
-  }
-  /**
-   * @param abortedTxnList list of all aborted transactions
-   * @param abortedBits bitset marking whether the corresponding transaction is aborted
-   * @param highWatermark highest committed transaction to be considered for compaction,
-   *                      equivalently (lowest_open_txn - 1).
-   */
-  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) {
-    // abortedBits should be all true as everything in exceptions are aborted txns
-    super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
-    if(this.exceptions.length <= 0) {
-      return;
-    }
-    //now that exceptions (aka abortedTxnList) is sorted
-    int idx = Arrays.binarySearch(this.exceptions, highWatermark);
-    int lastElementPos;
-    if(idx < 0) {
-      int insertionPoint = -idx - 1 ;//see Arrays.binarySearch() JavaDoc
-      lastElementPos = insertionPoint - 1;
-    }
-    else {
-      lastElementPos = idx;
-    }
-    /**
-     * ensure that we throw out any exceptions above highWatermark to make
-     * {@link #isTxnValid(long)} faster 
-     */
-    this.exceptions = Arrays.copyOf(this.exceptions, lastElementPos + 1);
-  }
-  public ValidCompactorTxnList(String value) {
-    super(value);
-  }
-  /**
-   * Returns org.apache.hadoop.hive.common.ValidTxnList.RangeResponse.ALL if all txns in
-   * the range are resolved and RangeResponse.NONE otherwise
-   */
-  @Override
-  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
-    return highWatermark >= maxTxnId ? RangeResponse.ALL : RangeResponse.NONE;
-  }
-
-  @Override
-  public boolean isTxnAborted(long txnid) {
-    return Arrays.binarySearch(exceptions, txnid) >= 0;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
deleted file mode 100644
index 002afd6..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import java.util.Arrays;
-import java.util.BitSet;
-
-/**
- * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by readers.
- * This class will view a transaction as valid only if it is committed.  Both open and aborted
- * transactions will be seen as invalid.
- */
-public class ValidReadTxnList implements ValidTxnList {
-
-  protected long[] exceptions;
-  protected BitSet abortedBits; // BitSet for flagging aborted transactions. Bit is true if aborted, false if open
-  //default value means there are no open txn in the snapshot
-  private long minOpenTxn = Long.MAX_VALUE;
-  protected long highWatermark;
-
-  public ValidReadTxnList() {
-    this(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE);
-  }
-
-  /**
-   * Used if there are no open transactions in the snapshot
-   */
-  public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark) {
-    this(exceptions, abortedBits, highWatermark, Long.MAX_VALUE);
-  }
-  public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark, long minOpenTxn) {
-    if (exceptions.length > 0) {
-      this.minOpenTxn = minOpenTxn;
-    }
-    this.exceptions = exceptions;
-    this.abortedBits = abortedBits;
-    this.highWatermark = highWatermark;
-  }
-
-  public ValidReadTxnList(String value) {
-    readFromString(value);
-  }
-
-  @Override
-  public boolean isTxnValid(long txnid) {
-    if (highWatermark < txnid) {
-      return false;
-    }
-    return Arrays.binarySearch(exceptions, txnid) < 0;
-  }
-
-  /**
-   * We cannot use a base file if its range contains an open txn.
-   * @param txnid from base_xxxx
-   */
-  @Override
-  public boolean isValidBase(long txnid) {
-    return minOpenTxn > txnid && txnid <= highWatermark;
-  }
-  @Override
-  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
-    // check the easy cases first
-    if (highWatermark < minTxnId) {
-      return RangeResponse.NONE;
-    } else if (exceptions.length > 0 && exceptions[0] > maxTxnId) {
-      return RangeResponse.ALL;
-    }
-
-    // since the exceptions and the range in question overlap, count the
-    // exceptions in the range
-    long count = Math.max(0, maxTxnId - highWatermark);
-    for(long txn: exceptions) {
-      if (minTxnId <= txn && txn <= maxTxnId) {
-        count += 1;
-      }
-    }
-
-    if (count == 0) {
-      return RangeResponse.ALL;
-    } else if (count == (maxTxnId - minTxnId + 1)) {
-      return RangeResponse.NONE;
-    } else {
-      return RangeResponse.SOME;
-    }
-  }
-
-  @Override
-  public String toString() {
-    return writeToString();
-  }
-
-  @Override
-  public String writeToString() {
-    StringBuilder buf = new StringBuilder();
-    buf.append(highWatermark);
-    buf.append(':');
-    buf.append(minOpenTxn);
-    if (exceptions.length == 0) {
-      buf.append(':');  // separator for open txns
-      buf.append(':');  // separator for aborted txns
-    } else {
-      StringBuilder open = new StringBuilder();
-      StringBuilder abort = new StringBuilder();
-      for (int i = 0; i < exceptions.length; i++) {
-        if (abortedBits.get(i)) {
-          if (abort.length() > 0) {
-            abort.append(',');
-          }
-          abort.append(exceptions[i]);
-        } else {
-          if (open.length() > 0) {
-            open.append(',');
-          }
-          open.append(exceptions[i]);
-        }
-      }
-      buf.append(':');
-      buf.append(open);
-      buf.append(':');
-      buf.append(abort);
-    }
-    return buf.toString();
-  }
-
-  @Override
-  public void readFromString(String src) {
-    if (src == null || src.length() == 0) {
-      highWatermark = Long.MAX_VALUE;
-      exceptions = new long[0];
-      abortedBits = new BitSet();
-    } else {
-      String[] values = src.split(":");
-      highWatermark = Long.parseLong(values[0]);
-      minOpenTxn = Long.parseLong(values[1]);
-      String[] openTxns = new String[0];
-      String[] abortedTxns = new String[0];
-      if (values.length < 3) {
-        openTxns = new String[0];
-        abortedTxns = new String[0];
-      } else if (values.length == 3) {
-        if (!values[2].isEmpty()) {
-          openTxns = values[2].split(",");
-        }
-      } else {
-        if (!values[2].isEmpty()) {
-          openTxns = values[2].split(",");
-        }
-        if (!values[3].isEmpty()) {
-          abortedTxns = values[3].split(",");
-        }
-      }
-      exceptions = new long[openTxns.length + abortedTxns.length];
-      int i = 0;
-      for (String open : openTxns) {
-        exceptions[i++] = Long.parseLong(open);
-      }
-      for (String abort : abortedTxns) {
-        exceptions[i++] = Long.parseLong(abort);
-      }
-      Arrays.sort(exceptions);
-      abortedBits = new BitSet(exceptions.length);
-      for (String abort : abortedTxns) {
-        int index = Arrays.binarySearch(exceptions, Long.parseLong(abort));
-        abortedBits.set(index);
-      }
-    }
-  }
-
-  @Override
-  public long getHighWatermark() {
-    return highWatermark;
-  }
-
-  @Override
-  public long[] getInvalidTransactions() {
-    return exceptions;
-  }
-
-  @Override
-  public Long getMinOpenTxn() {
-    return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
-  }
-
-  @Override
-  public boolean isTxnAborted(long txnid) {
-    int index = Arrays.binarySearch(exceptions, txnid);
-    return index >= 0 && abortedBits.get(index);
-  }
-
-  @Override
-  public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId) {
-    // check the easy cases first
-    if (highWatermark < minTxnId) {
-      return RangeResponse.NONE;
-    }
-
-    int count = 0;  // number of aborted txns found in exceptions
-
-    // traverse the aborted txns list, starting at first aborted txn index
-    for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) {
-      long abortedTxnId = exceptions[i];
-      if (abortedTxnId > maxTxnId) {  // we've already gone beyond the specified range
-        break;
-      }
-      if (abortedTxnId >= minTxnId && abortedTxnId <= maxTxnId) {
-        count++;
-      }
-    }
-
-    if (count == 0) {
-      return RangeResponse.NONE;
-    } else if (count == (maxTxnId - minTxnId + 1)) {
-      return RangeResponse.ALL;
-    } else {
-      return RangeResponse.SOME;
-    }
-  }
-}
-

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
deleted file mode 100644
index 108e5ca..0000000
--- a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.common;
-
-/**
- * Models the list of transactions that should be included in a snapshot.
- * It is modelled as a high water mark, which is the largest transaction id that
- * has been committed and a list of transactions that are not included.
- */
-public interface ValidTxnList {
-
-  /**
-   * Key used to store valid txn list in a
-   * {@link org.apache.hadoop.conf.Configuration} object.
-   */
-  public static final String VALID_TXNS_KEY = "hive.txn.valid.txns";
-
-  /**
-   * The response to a range query.  NONE means no values in this range match,
-   * SOME mean that some do, and ALL means that every value does.
-   */
-  public enum RangeResponse {NONE, SOME, ALL};
-
-  /**
-   * Indicates whether a given transaction is valid. Note that valid may have different meanings
-   * for different implementations, as some will only want to see committed transactions and some
-   * both committed and aborted.
-   * @param txnid id for the transaction
-   * @return true if valid, false otherwise
-   */
-  public boolean isTxnValid(long txnid);
-
-  /**
-   * Returns {@code true} if such base file can be used to materialize the snapshot represented by
-   * this {@code ValidTxnList}.
-   * @param txnid highest txn in a given base_xxxx file
-   */
-  public boolean isValidBase(long txnid);
-
-  /**
-   * Find out if a range of transaction ids are valid.  Note that valid may have different meanings
-   * for different implementations, as some will only want to see committed transactions and some
-   * both committed and aborted.
-   * @param minTxnId minimum txnid to look for, inclusive
-   * @param maxTxnId maximum txnid to look for, inclusive
-   * @return Indicate whether none, some, or all of these transactions are valid.
-   */
-  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId);
-
-  /**
-   * Write this validTxnList into a string. This should produce a string that
-   * can be used by {@link #readFromString(String)} to populate a validTxnsList.
-   */
-  public String writeToString();
-
-  /**
-   * Populate this validTxnList from the string.  It is assumed that the string
-   * was created via {@link #writeToString()} and the exceptions list is sorted.
-   * @param src source string.
-   */
-  public void readFromString(String src);
-
-  /**
-   * Get the largest transaction id used.
-   * @return largest transaction id used
-   */
-  public long getHighWatermark();
-
-  /**
-   * Get the list of transactions under the high water mark that are not valid.  Note that invalid
-   * may have different meanings for different implementations, as some will only want to see open
-   * transactions and some both open and aborted.
-   * @return a list of invalid transaction ids
-   */
-  public long[] getInvalidTransactions();
-
-  /**
-   * Indicates whether a given transaction is aborted.
-   * @param txnid id for the transaction
-   * @return true if aborted, false otherwise
-   */
-  public boolean isTxnAborted(long txnid);
-
-  /**
-   * Find out if a range of transaction ids are aborted.
-   * @param minTxnId minimum txnid to look for, inclusive
-   * @param maxTxnId maximum txnid to look for, inclusive
-   * @return Indicate whether none, some, or all of these transactions are aborted.
-   */
-  public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
-
-  /**
-   * Returns smallest Open transaction in this set, {@code null} if there is none.
-   */
-  Long getMinOpenTxn();
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java b/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
deleted file mode 100644
index 00ee820..0000000
--- a/common/src/test/org/apache/hadoop/hive/common/TestValidReadTxnList.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.common;
-
-import junit.framework.Assert;
-import org.apache.hadoop.conf.Configuration;
-import org.junit.Test;
-
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.util.BitSet;
-
-/**
- * Tests for {@link ValidReadTxnList}
- */
-public class TestValidReadTxnList {
-
-  @Test
-  public void noExceptions() throws Exception {
-    ValidTxnList txnList = new ValidReadTxnList(new long[0], new BitSet(), 1, Long.MAX_VALUE);
-    String str = txnList.writeToString();
-    Assert.assertEquals("1:" + Long.MAX_VALUE + "::", str);
-    ValidTxnList newList = new ValidReadTxnList();
-    newList.readFromString(str);
-    Assert.assertTrue(newList.isTxnValid(1));
-    Assert.assertFalse(newList.isTxnValid(2));
-  }
-
-  @Test
-  public void exceptions() throws Exception {
-    ValidTxnList txnList = new ValidReadTxnList(new long[]{2L,4L}, new BitSet(), 5, 4L);
-    String str = txnList.writeToString();
-    Assert.assertEquals("5:4:2,4:", str);
-    ValidTxnList newList = new ValidReadTxnList();
-    newList.readFromString(str);
-    Assert.assertTrue(newList.isTxnValid(1));
-    Assert.assertFalse(newList.isTxnValid(2));
-    Assert.assertTrue(newList.isTxnValid(3));
-    Assert.assertFalse(newList.isTxnValid(4));
-    Assert.assertTrue(newList.isTxnValid(5));
-    Assert.assertFalse(newList.isTxnValid(6));
-  }
-
-  @Test
-  public void longEnoughToCompress() throws Exception {
-    long[] exceptions = new long[1000];
-    for (int i = 0; i < 1000; i++) exceptions[i] = i + 100;
-    ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900);
-    String str = txnList.writeToString();
-    ValidTxnList newList = new ValidReadTxnList();
-    newList.readFromString(str);
-    for (int i = 0; i < 100; i++) Assert.assertTrue(newList.isTxnValid(i));
-    for (int i = 100; i < 1100; i++) Assert.assertFalse(newList.isTxnValid(i));
-    for (int i = 1100; i < 2001; i++) Assert.assertTrue(newList.isTxnValid(i));
-    Assert.assertFalse(newList.isTxnValid(2001));
-  }
-
-  @Test
-  public void readWriteConfig() throws Exception {
-    long[] exceptions = new long[1000];
-    for (int i = 0; i < 1000; i++) exceptions[i] = i + 100;
-    ValidTxnList txnList = new ValidReadTxnList(exceptions, new BitSet(), 2000, 900);
-    String str = txnList.writeToString();
-    Configuration conf = new Configuration();
-    conf.set(ValidTxnList.VALID_TXNS_KEY, str);
-    File tmpFile = File.createTempFile("TestValidTxnImpl", "readWriteConfig");
-    DataOutputStream out = new DataOutputStream(new FileOutputStream(tmpFile));
-    conf.write(out);
-    out.close();
-    DataInputStream in = new DataInputStream(new FileInputStream(tmpFile));
-    Configuration newConf = new Configuration();
-    newConf.readFields(in);
-    Assert.assertEquals(str, newConf.get(ValidTxnList.VALID_TXNS_KEY));
-  }
-
-  @Test
-  public void testAbortedTxn() throws Exception {
-    long[] exceptions = {2L, 4L, 6L, 8L, 10L};
-    BitSet bitSet = new BitSet(exceptions.length);
-    bitSet.set(0);  // mark txn "2L" aborted
-    bitSet.set(3);  // mark txn "8L" aborted
-    ValidTxnList txnList = new ValidReadTxnList(exceptions, bitSet, 11, 4L);
-    String str = txnList.writeToString();
-    Assert.assertEquals("11:4:4,6,10:2,8", str);
-    Assert.assertTrue(txnList.isTxnAborted(2L));
-    Assert.assertFalse(txnList.isTxnAborted(4L));
-    Assert.assertFalse(txnList.isTxnAborted(6L));
-    Assert.assertTrue(txnList.isTxnAborted(8L));
-    Assert.assertFalse(txnList.isTxnAborted(10L));
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
index 921bbd3..3b5066f 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/TestStreaming.java
@@ -211,8 +211,8 @@ public class TestStreaming {
 
 
     //1) Start from a clean slate (metastore)
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(conf);
+    TxnDbUtil.prepDb(conf);
 
     //2) obtain metastore clients
     msClient = new HiveMetaStoreClient(conf);

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingTestUtils.java
----------------------------------------------------------------------
diff --git a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingTestUtils.java b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingTestUtils.java
index 87a72b5..63690f9 100644
--- a/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingTestUtils.java
+++ b/hcatalog/streaming/src/test/org/apache/hive/hcatalog/streaming/mutate/StreamingTestUtils.java
@@ -66,8 +66,8 @@ public class StreamingTestUtils {
 
   public void prepareTransactionDatabase(HiveConf conf) throws Exception {
     TxnDbUtil.setConfValues(conf);
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(conf);
+    TxnDbUtil.prepDb(conf);
   }
 
   public IMetaStoreClient newMetaStoreClient(HiveConf conf) throws Exception {

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index 1002be7..a19cc86 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -259,12 +259,12 @@ public class TestHiveMetaStoreTxns {
 
   @Before
   public void setUp() throws Exception {
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(conf);
     client = new HiveMetaStoreClient(conf);
   }
 
   @After
   public void tearDown() throws Exception {
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
   }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
index ea0aadf..dbfc235 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestAcidOnTez.java
@@ -96,7 +96,6 @@ public class TestAcidOnTez {
 
   @Before
   public void setUp() throws Exception {
-    tearDown();
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
@@ -107,7 +106,7 @@ public class TestAcidOnTez {
         .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER,
             "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
     TxnDbUtil.setConfValues(hiveConf);
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(hiveConf);
     File f = new File(TEST_WAREHOUSE_DIR);
     if (f.exists()) {
       FileUtil.fullyDelete(f);
@@ -152,7 +151,7 @@ public class TestAcidOnTez {
         d.close();
         d = null;
       }
-      TxnDbUtil.cleanDb();
+      TxnDbUtil.cleanDb(hiveConf);
     } finally {
       FileUtils.deleteDirectory(new File(TEST_DATA_DIR));
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index aea1dfc..707bcd1 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -118,8 +118,8 @@ public class TestCompactor {
     //"org.apache.hadoop.hive.ql.io.HiveInputFormat"
 
     TxnDbUtil.setConfValues(hiveConf);
-    TxnDbUtil.cleanDb();
-    TxnDbUtil.prepDb();
+    TxnDbUtil.cleanDb(hiveConf);
+    TxnDbUtil.prepDb(hiveConf);
 
     conf = hiveConf;
     msClient = new HiveMetaStoreClient(conf);

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f4af6f4..5617e1c 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -131,6 +131,7 @@ import org.apache.hadoop.hive.metastore.repl.DumpDirCleanerTask;
 import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
 import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager;
 import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport;
+import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.serde2.Deserializer;
@@ -7624,6 +7625,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
                 + e.getMessage(), e);
             }
           }
+          ThreadPool.shutdown();
         }
       });
 
@@ -8002,6 +8004,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
     startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidHouseKeeperService"));
     startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService"));
     startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidWriteSetService"));
+
+    ThreadPool.initialize(conf);
+    RunnableConfigurable rc = new AcidOpenTxnsCounterService();
+    rc.setConf(conf);
+    ThreadPool.getPool().scheduleAtFixedRate(rc, 100, MetastoreConf.getTimeVar(conf,
+        MetastoreConf.ConfVars.COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS),
+        TimeUnit.MILLISECONDS);
+
   }
   private static void startHouseKeeperService(HiveConf conf, Class c) throws Exception {
     //todo: when metastore adds orderly-shutdown logic, houseKeeper.stop()

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
deleted file mode 100644
index 34765b0..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/BoneCPDataSourceProvider.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.jolbox.bonecp.BoneCPConfig;
-import com.jolbox.bonecp.BoneCPDataSource;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.sql.DataSource;
-import java.sql.SQLException;
-import java.util.Properties;
-
-/**
- * DataSourceProvider for the BoneCP connection pool.
- */
-public class BoneCPDataSourceProvider implements DataSourceProvider {
-
-  private static final Logger LOG = LoggerFactory.getLogger(BoneCPDataSourceProvider.class);
-
-  public static final String BONECP = "bonecp";
-  private static final String CONNECTION_TIMEOUT_PROPERTY= "bonecp.connectionTimeoutInMs";
-  private static final String PARTITION_COUNT_PROPERTY= "bonecp.partitionCount";
-
-  @Override
-  public DataSource create(Configuration hdpConfig) throws SQLException {
-
-    LOG.debug("Creating BoneCP connection pool for the MetaStore");
-
-    String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
-    String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
-    String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
-    int maxPoolSize = hdpConfig.getInt(
-        MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.varname,
-        ((Long)MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.defaultVal).intValue());
-
-    Properties properties = DataSourceProvider.getPrefixedProperties(hdpConfig, BONECP);
-    long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
-    String partitionCount = properties.getProperty(PARTITION_COUNT_PROPERTY, "1");
-
-    BoneCPConfig config = null;
-    try {
-      config = new BoneCPConfig(properties);
-    } catch (Exception e) {
-      throw new SQLException("Cannot create BoneCP configuration: ", e);
-    }
-    config.setJdbcUrl(driverUrl);
-    //if we are waiting for connection for a long time, something is really wrong
-    //better raise an error than hang forever
-    //see DefaultConnectionStrategy.getConnectionInternal()
-    config.setConnectionTimeoutInMs(connectionTimeout);
-    config.setMaxConnectionsPerPartition(maxPoolSize);
-    config.setPartitionCount(Integer.parseInt(partitionCount));
-    config.setUser(user);
-    config.setPassword(passwd);
-    return new BoneCPDataSource(config);
-  }
-
-  @Override
-  public boolean mayReturnClosedConnection() {
-    // See HIVE-11915 for details
-    return true;
-  }
-
-  @Override
-  public boolean supports(Configuration configuration) {
-    String poolingType =
-        configuration.get(
-            MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE.varname).toLowerCase();
-    if (BONECP.equals(poolingType)) {
-      int boneCpPropsNr = DataSourceProvider.getPrefixedProperties(configuration, BONECP).size();
-      LOG.debug("Found " + boneCpPropsNr + " nr. of bonecp specific configurations");
-      return boneCpPropsNr > 0;
-    }
-    LOG.debug("Configuration requested " + poolingType + " pooling, BoneCpDSProvider exiting");
-    return false;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
deleted file mode 100644
index ad1763e..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProvider.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-
-import javax.sql.DataSource;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.util.Properties;
-
-public interface DataSourceProvider {
-
-  /**
-   * @param hdpConfig
-   * @return the new connection pool
-   */
-  DataSource create(Configuration hdpConfig) throws SQLException;
-
-  /**
-   * BoneCp has a bug which causes closed connections to be returned to the pool
-   * under certain conditions. (HIVE-11915)
-   * @return true if the factory creates BoneCp pools which need "special attention"
-   */
-  boolean mayReturnClosedConnection();
-
-  /**
-   * @param configuration Hadoop configuration object
-   * @return factory able to create a connection pool for the implementation
-   * specified in the configuration
-   */
-  boolean supports(Configuration configuration);
-
-  /**
-   * @param hdpConfig
-   * @return subset of properties prefixed by a connection pool specific substring
-   */
-  static Properties getPrefixedProperties(Configuration hdpConfig, String factoryPrefix) {
-    Properties dataSourceProps = new Properties();
-    Iterables.filter(
-        hdpConfig, (entry -> entry.getKey() != null && entry.getKey().startsWith(factoryPrefix)))
-        .forEach(entry -> dataSourceProps.put(entry.getKey(), entry.getValue()));
-    return dataSourceProps;
-  }
-
-  static String getMetastoreJdbcUser(Configuration conf) {
-    return conf.get(MetastoreConf.ConfVars.CONNECTION_USER_NAME.varname);
-  }
-
-  static String getMetastoreJdbcPasswd(Configuration conf) throws SQLException {
-    try {
-      return MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
-    } catch (IOException err) {
-      throw new SQLException("Error getting metastore password", err);
-    }
-  }
-
-  static String getMetastoreJdbcDriverUrl(Configuration conf) throws SQLException {
-    return conf.get(MetastoreConf.ConfVars.CONNECTURLKEY.varname);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
deleted file mode 100644
index 1eb792c..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/DataSourceProviderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Create a DataSourceProvider for a connectionPool configured in a hadoop
- * Configuration object.
- */
-public abstract  class DataSourceProviderFactory {
-
-  private static final ImmutableList<DataSourceProvider> FACTORIES =
-      ImmutableList.<DataSourceProvider>builder().add(new HikariCPDataSourceProvider(), new BoneCPDataSourceProvider()).build();
-
-  /**
-   * @param hdpConfig hadoop configuration
-   * @return factory for the configured datanucleus.connectionPoolingType
-   */
-  public static DataSourceProvider getDataSourceProvider(Configuration hdpConfig) {
-
-    for (DataSourceProvider factory : FACTORIES) {
-
-      if (factory.supports(hdpConfig)) {
-        return factory;
-      }
-    }
-    return null;
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
deleted file mode 100644
index 9b3d6d5..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/HikariCPDataSourceProvider.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.datasource;
-
-import com.zaxxer.hikari.HikariConfig;
-import com.zaxxer.hikari.HikariDataSource;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.sql.DataSource;
-import java.sql.SQLException;
-import java.util.Properties;
-
-/**
- * DataSourceProvider for the HikariCP connection pool.
- */
-public class HikariCPDataSourceProvider implements DataSourceProvider {
-
-  private static final Logger LOG = LoggerFactory.getLogger(HikariCPDataSourceProvider.class);
-
-  public static final String HIKARI = "hikari";
-  private static final String CONNECTION_TIMEOUT_PROPERTY= "hikari.connectionTimeout";
-
-  @Override
-  public DataSource create(Configuration hdpConfig) throws SQLException {
-
-    LOG.debug("Creating Hikari connection pool for the MetaStore");
-
-    String driverUrl = DataSourceProvider.getMetastoreJdbcDriverUrl(hdpConfig);
-    String user = DataSourceProvider.getMetastoreJdbcUser(hdpConfig);
-    String passwd = DataSourceProvider.getMetastoreJdbcPasswd(hdpConfig);
-    int maxPoolSize = hdpConfig.getInt(
-        MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.varname,
-        ((Long)MetastoreConf.ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS.defaultVal).intValue());
-
-    Properties properties = replacePrefix(
-        DataSourceProvider.getPrefixedProperties(hdpConfig, HIKARI));
-    long connectionTimeout = hdpConfig.getLong(CONNECTION_TIMEOUT_PROPERTY, 30000L);
-    HikariConfig config = null;
-    try {
-      config = new HikariConfig(properties);
-    } catch (Exception e) {
-      throw new SQLException("Cannot create HikariCP configuration: ", e);
-    }
-    config.setMaximumPoolSize(maxPoolSize);
-    config.setJdbcUrl(driverUrl);
-    config.setUsername(user);
-    config.setPassword(passwd);
-    //https://github.com/brettwooldridge/HikariCP
-    config.setConnectionTimeout(connectionTimeout);
-    return new HikariDataSource(config);
-  }
-
-  @Override
-  public boolean mayReturnClosedConnection() {
-    // Only BoneCP should return true
-    return false;
-  }
-
-  @Override
-  public boolean supports(Configuration configuration) {
-    String poolingType =
-        configuration.get(
-            MetastoreConf.ConfVars.CONNECTION_POOLING_TYPE.varname).toLowerCase();
-    if (HIKARI.equals(poolingType)) {
-      int hikariPropsNr = DataSourceProvider.getPrefixedProperties(configuration, HIKARI).size();
-      LOG.debug("Found " + hikariPropsNr + " nr. of hikari specific configurations");
-      return hikariPropsNr > 0;
-    }
-    LOG.debug("Configuration requested " + poolingType + " pooling, HikariCpDSProvider exiting");
-    return false;
-  }
-
-  private Properties replacePrefix(Properties props) {
-    Properties newProps = new Properties();
-    props.forEach((key,value) ->
-        newProps.put(key.toString().replaceFirst(HIKARI + ".", ""), value));
-    return newProps;
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java b/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
deleted file mode 100644
index 86d6a26..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/datasource/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * DataSource providers that can construct a connection pool from configuration
- * properties in a hadoop configuration object.
- */
-package org.apache.hadoop.hive.metastore.datasource;

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java b/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
deleted file mode 100644
index 0c0bfef..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/tools/SQLGenerator.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
-  Licensed to the Apache Software Foundation (ASF) under one
-  or more contributor license agreements.  See the NOTICE file
-  distributed with this work for additional information
-  regarding copyright ownership.  The ASF licenses this file
-  to you under the Apache License, Version 2.0 (the
-  "License"); you may not use this file except in compliance
-  with the License.  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.tools;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-/**
- * Helper class that generates SQL queries with syntax specific to target DB
- * todo: why throw MetaException?
- */
-@VisibleForTesting
-public final class SQLGenerator {
-  static final private Logger LOG = LoggerFactory.getLogger(SQLGenerator.class.getName());
-  private final DatabaseProduct dbProduct;
-  private final HiveConf conf;
-
-  public SQLGenerator(DatabaseProduct dbProduct, HiveConf conf) {
-    this.dbProduct = dbProduct;
-    this.conf = conf;
-  }
-
-  /**
-   * Genereates "Insert into T(a,b,c) values(1,2,'f'),(3,4,'c')" for appropriate DB
-   *
-   * @param tblColumns e.g. "T(a,b,c)"
-   * @param rows       e.g. list of Strings like 3,4,'d'
-   * @return fully formed INSERT INTO ... statements
-   */
-  public List<String> createInsertValuesStmt(String tblColumns, List<String> rows) {
-    if (rows == null || rows.size() == 0) {
-      return Collections.emptyList();
-    }
-    List<String> insertStmts = new ArrayList<>();
-    StringBuilder sb = new StringBuilder();
-    switch (dbProduct) {
-    case ORACLE:
-      if (rows.size() > 1) {
-        //http://www.oratable.com/oracle-insert-all/
-        //https://livesql.oracle.com/apex/livesql/file/content_BM1LJQ87M5CNIOKPOWPV6ZGR3.html
-        for (int numRows = 0; numRows < rows.size(); numRows++) {
-          if (numRows % conf
-              .getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
-            if (numRows > 0) {
-              sb.append(" select * from dual");
-              insertStmts.add(sb.toString());
-            }
-            sb.setLength(0);
-            sb.append("insert all ");
-          }
-          sb.append("into ").append(tblColumns).append(" values(").append(rows.get(numRows))
-              .append(") ");
-        }
-        sb.append("select * from dual");
-        insertStmts.add(sb.toString());
-        return insertStmts;
-      }
-      //fall through
-    case DERBY:
-    case MYSQL:
-    case POSTGRES:
-    case SQLSERVER:
-      for (int numRows = 0; numRows < rows.size(); numRows++) {
-        if (numRows % conf
-            .getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE) == 0) {
-          if (numRows > 0) {
-            insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
-          }
-          sb.setLength(0);
-          sb.append("insert into ").append(tblColumns).append(" values");
-        }
-        sb.append('(').append(rows.get(numRows)).append("),");
-      }
-      insertStmts.add(sb.substring(0, sb.length() - 1));//exclude trailing comma
-      return insertStmts;
-    default:
-      String msg = "Unrecognized database product name <" + dbProduct + ">";
-      LOG.error(msg);
-      throw new IllegalStateException(msg);
-    }
-  }
-
-  /**
-   * Given a {@code selectStatement}, decorated it with FOR UPDATE or semantically equivalent
-   * construct.  If the DB doesn't support, return original select.
-   */
-  public String addForUpdateClause(String selectStatement) throws MetaException {
-    switch (dbProduct) {
-    case DERBY:
-      //https://db.apache.org/derby/docs/10.1/ref/rrefsqlj31783.html
-      //sadly in Derby, FOR UPDATE doesn't meant what it should
-      return selectStatement;
-    case MYSQL:
-      //http://dev.mysql.com/doc/refman/5.7/en/select.html
-    case ORACLE:
-      //https://docs.oracle.com/cd/E17952_01/refman-5.6-en/select.html
-    case POSTGRES:
-      //http://www.postgresql.org/docs/9.0/static/sql-select.html
-      return selectStatement + " for update";
-    case SQLSERVER:
-      //https://msdn.microsoft.com/en-us/library/ms189499.aspx
-      //https://msdn.microsoft.com/en-us/library/ms187373.aspx
-      String modifier = " with (updlock)";
-      int wherePos = selectStatement.toUpperCase().indexOf(" WHERE ");
-      if (wherePos < 0) {
-        return selectStatement + modifier;
-      }
-      return selectStatement.substring(0, wherePos) + modifier +
-          selectStatement.substring(wherePos, selectStatement.length());
-    default:
-      String msg = "Unrecognized database product name <" + dbProduct + ">";
-      LOG.error(msg);
-      throw new MetaException(msg);
-    }
-  }
-
-  /**
-   * Suppose you have a query "select a,b from T" and you want to limit the result set
-   * to the first 5 rows.  The mechanism to do that differs in different DBs.
-   * Make {@code noSelectsqlQuery} to be "a,b from T" and this method will return the
-   * appropriately modified row limiting query.
-   * <p>
-   * Note that if {@code noSelectsqlQuery} contains a join, you must make sure that
-   * all columns are unique for Oracle.
-   */
-  public String addLimitClause(int numRows, String noSelectsqlQuery) throws MetaException {
-    switch (dbProduct) {
-    case DERBY:
-      //http://db.apache.org/derby/docs/10.7/ref/rrefsqljoffsetfetch.html
-      return "select " + noSelectsqlQuery + " fetch first " + numRows + " rows only";
-    case MYSQL:
-      //http://www.postgresql.org/docs/7.3/static/queries-limit.html
-    case POSTGRES:
-      //https://dev.mysql.com/doc/refman/5.0/en/select.html
-      return "select " + noSelectsqlQuery + " limit " + numRows;
-    case ORACLE:
-      //newer versions (12c and later) support OFFSET/FETCH
-      return "select * from (select " + noSelectsqlQuery + ") where rownum <= " + numRows;
-    case SQLSERVER:
-      //newer versions (2012 and later) support OFFSET/FETCH
-      //https://msdn.microsoft.com/en-us/library/ms189463.aspx
-      return "select TOP(" + numRows + ") " + noSelectsqlQuery;
-    default:
-      String msg = "Unrecognized database product name <" + dbProduct + ">";
-      LOG.error(msg);
-      throw new MetaException(msg);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
deleted file mode 100644
index 413ce3b..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.common.ValidCompactorTxnList;
-import org.apache.hadoop.hive.metastore.api.CompactionType;
-import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-/**
- * Information on a possible or running compaction.
- */
-public class CompactionInfo implements Comparable<CompactionInfo> {
-  public long id;
-  public String dbname;
-  public String tableName;
-  public String partName;
-  char state;
-  public CompactionType type;
-  String workerId;
-  long start;
-  public String runAs;
-  public String properties;
-  public boolean tooManyAborts = false;
-  /**
-   * {@code 0} means it wasn't set (e.g. in case of upgrades, since ResultSet.getLong() will return 0 if field is NULL) 
-   * See {@link TxnStore#setCompactionHighestTxnId(CompactionInfo, long)} for precise definition.
-   * See also {@link TxnUtils#createValidCompactTxnList(GetOpenTxnsInfoResponse)} and
-   * {@link ValidCompactorTxnList#highWatermark}
-   */
-  public long highestTxnId;
-  byte[] metaInfo;
-  String hadoopJobId;
-
-  private String fullPartitionName = null;
-  private String fullTableName = null;
-
-  public CompactionInfo(String dbname, String tableName, String partName, CompactionType type) {
-    this.dbname = dbname;
-    this.tableName = tableName;
-    this.partName = partName;
-    this.type = type;
-  }
-  CompactionInfo(long id, String dbname, String tableName, String partName, char state) {
-    this(dbname, tableName, partName, null);
-    this.id = id;
-    this.state = state;
-  }
-  CompactionInfo() {}
-  
-  public String getFullPartitionName() {
-    if (fullPartitionName == null) {
-      StringBuilder buf = new StringBuilder(dbname);
-      buf.append('.');
-      buf.append(tableName);
-      if (partName != null) {
-        buf.append('.');
-        buf.append(partName);
-      }
-      fullPartitionName = buf.toString();
-    }
-    return fullPartitionName;
-  }
-
-  public String getFullTableName() {
-    if (fullTableName == null) {
-      StringBuilder buf = new StringBuilder(dbname);
-      buf.append('.');
-      buf.append(tableName);
-      fullTableName = buf.toString();
-    }
-    return fullTableName;
-  }
-  public boolean isMajorCompaction() {
-    return CompactionType.MAJOR == type;
-  }
-
-  @Override
-  public int compareTo(CompactionInfo o) {
-    return getFullPartitionName().compareTo(o.getFullPartitionName());
-  }
-  public String toString() {
-    return "id:" + id + "," +
-      "dbname:" + dbname + "," +
-      "tableName:" + tableName + "," +
-      "partName:" + partName + "," +
-      "state:" + state + "," +
-      "type:" + type + "," +
-      "properties:" + properties + "," +
-      "runAs:" + runAs + "," +
-      "tooManyAborts:" + tooManyAborts + "," +
-      "highestTxnId:" + highestTxnId;
-  }
-
-  /**
-   * loads object from a row in Select * from COMPACTION_QUEUE
-   * @param rs ResultSet after call to rs.next()
-   * @throws SQLException
-   */
-  static CompactionInfo loadFullFromCompactionQueue(ResultSet rs) throws SQLException {
-    CompactionInfo fullCi = new CompactionInfo();
-    fullCi.id = rs.getLong(1);
-    fullCi.dbname = rs.getString(2);
-    fullCi.tableName = rs.getString(3);
-    fullCi.partName = rs.getString(4);
-    fullCi.state = rs.getString(5).charAt(0);//cq_state
-    fullCi.type = TxnHandler.dbCompactionType2ThriftType(rs.getString(6).charAt(0));
-    fullCi.properties = rs.getString(7);
-    fullCi.workerId = rs.getString(8);
-    fullCi.start = rs.getLong(9);
-    fullCi.runAs = rs.getString(10);
-    fullCi.highestTxnId = rs.getLong(11);
-    fullCi.metaInfo = rs.getBytes(12);
-    fullCi.hadoopJobId = rs.getString(13);
-    return fullCi;
-  }
-  static void insertIntoCompletedCompactions(PreparedStatement pStmt, CompactionInfo ci, long endTime) throws SQLException {
-    pStmt.setLong(1, ci.id);
-    pStmt.setString(2, ci.dbname);
-    pStmt.setString(3, ci.tableName);
-    pStmt.setString(4, ci.partName);
-    pStmt.setString(5, Character.toString(ci.state));
-    pStmt.setString(6, Character.toString(TxnHandler.thriftCompactionType2DbType(ci.type)));
-    pStmt.setString(7, ci.properties);
-    pStmt.setString(8, ci.workerId);
-    pStmt.setLong(9, ci.start);
-    pStmt.setLong(10, endTime);
-    pStmt.setString(11, ci.runAs);
-    pStmt.setLong(12, ci.highestTxnId);
-    pStmt.setBytes(13, ci.metaInfo);
-    pStmt.setString(14, ci.hadoopJobId);
-  }
-}


[10/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
deleted file mode 100644
index 60839fa..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java
+++ /dev/null
@@ -1,960 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.common.classification.RetrySemantics;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.CompactionType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Extends the transaction handler with methods needed only by the compactor threads.  These
- * methods are not available through the thrift interface.
- */
-class CompactionTxnHandler extends TxnHandler {
-  static final private String CLASS_NAME = CompactionTxnHandler.class.getName();
-  static final private Logger LOG = LoggerFactory.getLogger(CLASS_NAME);
-
-  public CompactionTxnHandler() {
-  }
-
-  /**
-   * This will look through the completed_txn_components table and look for partitions or tables
-   * that may be ready for compaction.  Also, look through txns and txn_components tables for
-   * aborted transactions that we should add to the list.
-   * @param maxAborted Maximum number of aborted queries to allow before marking this as a
-   *                   potential compaction.
-   * @return list of CompactionInfo structs.  These will not have id, type,
-   * or runAs set since these are only potential compactions not actual ones.
-   */
-  @Override
-  @RetrySemantics.ReadOnly
-  public Set<CompactionInfo> findPotentialCompactions(int maxAborted) throws MetaException {
-    Connection dbConn = null;
-    Set<CompactionInfo> response = new HashSet<CompactionInfo>();
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        // Check for completed transactions
-        String s = "select distinct ctc_database, ctc_table, " +
-          "ctc_partition from COMPLETED_TXN_COMPONENTS";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          response.add(info);
-        }
-        rs.close();
-
-        // Check for aborted txns
-        s = "select tc_database, tc_table, tc_partition " +
-          "from TXNS, TXN_COMPONENTS " +
-          "where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' " +
-          "group by tc_database, tc_table, tc_partition " +
-          "having count(*) > " + maxAborted;
-
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.dbname = rs.getString(1);
-          info.tableName = rs.getString(2);
-          info.partName = rs.getString(3);
-          info.tooManyAborts = true;
-          response.add(info);
-        }
-
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } catch (SQLException e) {
-        LOG.error("Unable to connect to transaction database " + e.getMessage());
-        checkRetryable(dbConn, e, "findPotentialCompactions(maxAborted:" + maxAborted + ")");
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-      return response;
-    }
-    catch (RetryException e) {
-      return findPotentialCompactions(maxAborted);
-    }
-  }
-
-  /**
-   * Sets the user to run as.  This is for the case
-   * where the request was generated by the user and so the worker must set this value later.
-   * @param cq_id id of this entry in the queue
-   * @param user user to run the jobs as
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public void setRunAs(long cq_id, String user) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "update COMPACTION_QUEUE set cq_run_as = '" + user + "' where cq_id = " + cq_id;
-        LOG.debug("Going to execute update <" + s + ">");
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_run_as=" + user + " for compaction record with cq_id=" + cq_id + ".  updCnt=" + updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue, " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "setRunAs(cq_id:" + cq_id + ",user:" + user +")");
-      } finally {
-        closeDbConn(dbConn);
-        closeStmt(stmt);
-      }
-    } catch (RetryException e) {
-      setRunAs(cq_id, user);
-    }
-  }
-
-  /**
-   * This will grab the next compaction request off of
-   * the queue, and assign it to the worker.
-   * @param workerId id of the worker calling this, will be recorded in the db
-   * @return an info element for this compaction request, or null if there is no work to do now.
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public CompactionInfo findNextToCompact(String workerId) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      //need a separate stmt for executeUpdate() otherwise it will close the ResultSet(HIVE-12725)
-      Statement updStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select cq_id, cq_database, cq_table, cq_partition, " +
-          "cq_type, cq_tblproperties from COMPACTION_QUEUE where cq_state = '" + INITIATED_STATE + "'";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          LOG.debug("No compactions found ready to compact");
-          dbConn.rollback();
-          return null;
-        }
-        updStmt = dbConn.createStatement();
-        do {
-          CompactionInfo info = new CompactionInfo();
-          info.id = rs.getLong(1);
-          info.dbname = rs.getString(2);
-          info.tableName = rs.getString(3);
-          info.partName = rs.getString(4);
-          info.type = dbCompactionType2ThriftType(rs.getString(5).charAt(0));
-          info.properties = rs.getString(6);
-          // Now, update this record as being worked on by this worker.
-          long now = getDbTime(dbConn);
-          s = "update COMPACTION_QUEUE set cq_worker_id = '" + workerId + "', " +
-            "cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id +
-            " AND cq_state='" + INITIATED_STATE + "'";
-          LOG.debug("Going to execute update <" + s + ">");
-          int updCount = updStmt.executeUpdate(s);
-          if(updCount == 1) {
-            dbConn.commit();
-            return info;
-          }
-          if(updCount == 0) {
-            LOG.debug("Another Worker picked up " + info);
-            continue;
-          }
-          LOG.error("Unable to set to cq_state=" + WORKING_STATE + " for compaction record: " +
-            info + ". updCnt=" + updCount + ".");
-          dbConn.rollback();
-          return null;
-        } while( rs.next());
-        dbConn.rollback();
-        return null;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for compaction, " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "findNextToCompact(workerId:" + workerId + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(updStmt);
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return findNextToCompact(workerId);
-    }
-  }
-
-  /**
-   * This will mark an entry in the queue as compacted
-   * and put it in the ready to clean state.
-   * @param info info on the compaction entry to mark as compacted.
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void markCompacted(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "update COMPACTION_QUEUE set cq_state = '" + READY_FOR_CLEANING + "', " +
-          "cq_worker_id = null where cq_id = " + info.id;
-        LOG.debug("Going to execute update <" + s + ">");
-        int updCnt = stmt.executeUpdate(s);
-        if (updCnt != 1) {
-          LOG.error("Unable to set cq_state=" + READY_FOR_CLEANING + " for compaction record: " + info + ". updCnt=" + updCnt);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to update compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "markCompacted(" + info + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      markCompacted(info);
-    }
-  }
-
-  /**
-   * Find entries in the queue that are ready to
-   * be cleaned.
-   * @return information on the entry in the queue.
-   */
-  @Override
-  @RetrySemantics.ReadOnly
-  public List<CompactionInfo> findReadyToClean() throws MetaException {
-    Connection dbConn = null;
-    List<CompactionInfo> rc = new ArrayList<CompactionInfo>();
-
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select cq_id, cq_database, cq_table, cq_partition, " +
-          "cq_type, cq_run_as, cq_highest_txn_id from COMPACTION_QUEUE where cq_state = '" + READY_FOR_CLEANING + "'";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          CompactionInfo info = new CompactionInfo();
-          info.id = rs.getLong(1);
-          info.dbname = rs.getString(2);
-          info.tableName = rs.getString(3);
-          info.partName = rs.getString(4);
-          switch (rs.getString(5).charAt(0)) {
-            case MAJOR_TYPE: info.type = CompactionType.MAJOR; break;
-            case MINOR_TYPE: info.type = CompactionType.MINOR; break;
-            default: throw new MetaException("Unexpected compaction type " + rs.getString(5));
-          }
-          info.runAs = rs.getString(6);
-          info.highestTxnId = rs.getLong(7);
-          rc.add(info);
-        }
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-        return rc;
-      } catch (SQLException e) {
-        LOG.error("Unable to select next element for cleaning, " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "findReadyToClean");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return findReadyToClean();
-    }
-  }
-
-  /**
-   * This will remove an entry from the queue after
-   * it has been compacted.
-   * 
-   * @param info info on the compaction entry to remove
-   */
-  @Override
-  @RetrySemantics.CannotRetry
-  public void markCleaned(CompactionInfo info) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + info.id);
-        if(rs.next()) {
-          info = CompactionInfo.loadFullFromCompactionQueue(rs);
-        }
-        else {
-          throw new IllegalStateException("No record with CQ_ID=" + info.id + " found in COMPACTION_QUEUE");
-        }
-        close(rs);
-        String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id;
-        LOG.debug("Going to execute update <" + s + ">");
-        int updCount = stmt.executeUpdate(s);
-        if (updCount != 1) {
-          LOG.error("Unable to delete compaction record: " + info +  ".  Update count=" + updCount);
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-        }
-        pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
-        info.state = SUCCEEDED_STATE;
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, info, getDbTime(dbConn));
-        updCount = pStmt.executeUpdate();
-
-        // Remove entries from completed_txn_components as well, so we don't start looking there
-        // again but only up to the highest txn ID include in this compaction job.
-        //highestTxnId will be NULL in upgrade scenarios
-        s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " +
-          "ctc_table = '" + info.tableName + "'";
-        if (info.partName != null) {
-          s += " and ctc_partition = '" + info.partName + "'";
-        }
-        if(info.highestTxnId != 0) {
-          s += " and ctc_txnid <= " + info.highestTxnId;
-        }
-        LOG.debug("Going to execute update <" + s + ">");
-        if (stmt.executeUpdate(s) < 1) {
-          LOG.error("Expected to remove at least one row from completed_txn_components when " +
-            "marking compaction entry as clean!");
-        }
-
-        s = "select distinct txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" +
-          TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" +
-          info.tableName + "'" + (info.highestTxnId == 0 ? "" : " and txn_id <= " + info.highestTxnId);
-        if (info.partName != null) s += " and tc_partition = '" + info.partName + "'";
-        LOG.debug("Going to execute update <" + s + ">");
-        rs = stmt.executeQuery(s);
-        List<Long> txnids = new ArrayList<>();
-        while (rs.next()) txnids.add(rs.getLong(1));
-        // Remove entries from txn_components, as there may be aborted txn components
-        if (txnids.size() > 0) {
-          List<String> queries = new ArrayList<String>();
-
-          // Prepare prefix and suffix
-          StringBuilder prefix = new StringBuilder();
-          StringBuilder suffix = new StringBuilder();
-
-          prefix.append("delete from TXN_COMPONENTS where ");
-
-          //because 1 txn may include different partitions/tables even in auto commit mode
-          suffix.append(" and tc_database = ");
-          suffix.append(quoteString(info.dbname));
-          suffix.append(" and tc_table = ");
-          suffix.append(quoteString(info.tableName));
-          if (info.partName != null) {
-            suffix.append(" and tc_partition = ");
-            suffix.append(quoteString(info.partName));
-          }
-
-          // Populate the complete query with provided prefix and suffix
-          TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "tc_txnid", true, false);
-
-          for (String query : queries) {
-            LOG.debug("Going to execute update <" + query + ">");
-            int rc = stmt.executeUpdate(query);
-            LOG.debug("Removed " + rc + " records from txn_components");
-
-            // Don't bother cleaning from the txns table.  A separate call will do that.  We don't
-            // know here which txns still have components from other tables or partitions in the
-            // table, so we don't know which ones we can and cannot clean.
-          }
-        }
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "markCleaned(" + info + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(pStmt);
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markCleaned(info);
-    }
-  }
-
-  /**
-   * Clean up aborted transactions from txns that have no components in txn_components.  The reason such
-   * txns exist can be that now work was done in this txn (e.g. Streaming opened TransactionBatch and
-   * abandoned it w/o doing any work) or due to {@link #markCleaned(CompactionInfo)} being called.
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void cleanEmptyAbortedTxns() throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-      try {
-        //Aborted is a terminal state, so nothing about the txn can change
-        //after that, so READ COMMITTED is sufficient.
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select txn_id from TXNS where " +
-          "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " +
-          "txn_state = '" + TXN_ABORTED + "'";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        List<Long> txnids = new ArrayList<>();
-        while (rs.next()) txnids.add(rs.getLong(1));
-        close(rs);
-        if(txnids.size() <= 0) {
-          return;
-        }
-        Collections.sort(txnids);//easier to read logs
-        List<String> queries = new ArrayList<String>();
-        StringBuilder prefix = new StringBuilder();
-        StringBuilder suffix = new StringBuilder();
-
-        prefix.append("delete from TXNS where ");
-        suffix.append("");
-
-        TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, txnids, "txn_id", false, false);
-
-        for (String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          int rc = stmt.executeUpdate(query);
-          LOG.info("Removed " + rc + "  empty Aborted transactions from TXNS");
-        }
-        LOG.info("Aborted transactions removed from TXNS: " + txnids);
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to delete from txns table " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "cleanEmptyAbortedTxns");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      cleanEmptyAbortedTxns();
-    }
-  }
-
-  /**
-   * This will take all entries assigned to workers
-   * on a host return them to INITIATED state.  The initiator should use this at start up to
-   * clean entries from any workers that were in the middle of compacting when the metastore
-   * shutdown.  It does not reset entries from worker threads on other hosts as those may still
-   * be working.
-   * @param hostname Name of this host.  It is assumed this prefixes the thread's worker id,
-   *                 so that like hostname% will match the worker id.
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public void revokeFromLocalWorkers(String hostname) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
-          + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_worker_id like '"
-          +  hostname + "%'";
-        LOG.debug("Going to execute update <" + s + ">");
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        stmt.executeUpdate(s);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "revokeFromLocalWorkers(hostname:" + hostname +")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeFromLocalWorkers(hostname);
-    }
-  }
-
-  /**
-   * This call will return all compaction queue
-   * entries assigned to a worker but over the timeout back to the initiated state.
-   * This should be called by the initiator on start up and occasionally when running to clean up
-   * after dead threads.  At start up {@link #revokeFromLocalWorkers(String)} should be called
-   * first.
-   * @param timeout number of milliseconds since start time that should elapse before a worker is
-   *                declared dead.
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public void revokeTimedoutWorkers(long timeout) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        long latestValidStart = getDbTime(dbConn) - timeout;
-        stmt = dbConn.createStatement();
-        String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
-          + INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_start < "
-          +  latestValidStart;
-        LOG.debug("Going to execute update <" + s + ">");
-        // It isn't an error if the following returns no rows, as the local workers could have died
-        // with  nothing assigned to them.
-        stmt.executeUpdate(s);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.error("Unable to change dead worker's records back to initiated state " +
-          e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "revokeTimedoutWorkers(timeout:" + timeout + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      revokeTimedoutWorkers(timeout);
-    }
-  }
-
-  /**
-   * Queries metastore DB directly to find columns in the table which have statistics information.
-   * If {@code ci} includes partition info then per partition stats info is examined, otherwise
-   * table level stats are examined.
-   * @throws MetaException
-   */
-  @Override
-  @RetrySemantics.ReadOnly
-  public List<String> findColumnsWithStats(CompactionInfo ci) throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        String quote = getIdentifierQuoteString(dbConn);
-        stmt = dbConn.createStatement();
-        StringBuilder bldr = new StringBuilder();
-        bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote)
-          .append(" FROM ")
-          .append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS"))
-          .append(quote)
-          .append(" WHERE ")
-          .append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname)
-          .append("' AND ").append(quote).append("TABLE_NAME").append(quote)
-          .append(" = '").append(ci.tableName).append("'");
-        if (ci.partName != null) {
-          bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '")
-            .append(ci.partName).append("'");
-        }
-        String s = bldr.toString();
-
-      /*String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" :
-          "PART_COL_STATS")
-         + " WHERE DB_NAME='" + ci.dbname + "' AND TABLE_NAME='" + ci.tableName + "'"
-        + (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'");*/
-        LOG.debug("Going to execute <" + s + ">");
-        rs = stmt.executeQuery(s);
-        List<String> columns = new ArrayList<String>();
-        while (rs.next()) {
-          columns.add(rs.getString(1));
-        }
-        LOG.debug("Found columns to update stats: " + columns + " on " + ci.tableName +
-          (ci.partName == null ? "" : "/" + ci.partName));
-        dbConn.commit();
-        return columns;
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "findColumnsWithStats(" + ci.tableName +
-          (ci.partName == null ? "" : "/" + ci.partName) + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException ex) {
-      return findColumnsWithStats(ci);
-    }
-  }
-
-  /**
-   * Record the highest txn id that the {@code ci} compaction job will pay attention to.
-   * This is the highest resolved txn id, i.e. such that there are no open txns with lower ids.
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public void setCompactionHighestTxnId(CompactionInfo ci, long highestTxnId) throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        int updCount = stmt.executeUpdate("UPDATE COMPACTION_QUEUE SET CQ_HIGHEST_TXN_ID = " + highestTxnId +
-          " WHERE CQ_ID = " + ci.id);
-        if(updCount != 1) {
-          throw new IllegalStateException("Could not find record in COMPACTION_QUEUE for " + ci);
-        }
-        dbConn.commit();
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "setCompactionHighestTxnId(" + ci + "," + highestTxnId + ")");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(null, stmt, dbConn);
-      }
-    } catch (RetryException ex) {
-      setCompactionHighestTxnId(ci, highestTxnId);
-    }
-  }
-  private static class RetentionCounters {
-    int attemptedRetention = 0;
-    int failedRetention = 0;
-    int succeededRetention = 0;
-    RetentionCounters(int attemptedRetention, int failedRetention, int succeededRetention) {
-      this.attemptedRetention = attemptedRetention;
-      this.failedRetention = failedRetention;
-      this.succeededRetention = succeededRetention;
-    }
-  }
-  private void checkForDeletion(List<Long> deleteSet, CompactionInfo ci, RetentionCounters rc) {
-    switch (ci.state) {
-      case ATTEMPTED_STATE:
-        if(--rc.attemptedRetention < 0) {
-          deleteSet.add(ci.id);
-        }
-        break;
-      case FAILED_STATE:
-        if(--rc.failedRetention < 0) {
-          deleteSet.add(ci.id);
-        }
-        break;
-      case SUCCEEDED_STATE:
-        if(--rc.succeededRetention < 0) {
-          deleteSet.add(ci.id);
-        }
-        break;
-      default:
-        //do nothing to hanlde future RU/D where we may want to add new state types
-    }
-  }
-
-  /**
-   * For any given compactable entity (partition; table if not partitioned) the history of compactions
-   * may look like "sssfffaaasffss", for example.  The idea is to retain the tail (most recent) of the
-   * history such that a configurable number of each type of state is present.  Any other entries
-   * can be purged.  This scheme has advantage of always retaining the last failure/success even if
-   * it's not recent.
-   * @throws MetaException
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void purgeCompactionHistory() throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    List<Long> deleteSet = new ArrayList<>();
-    RetentionCounters rc = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        /*cc_id is monotonically increasing so for any entity sorts in order of compaction history,
-        thus this query groups by entity and withing group sorts most recent first*/
-        rs = stmt.executeQuery("select cc_id, cc_database, cc_table, cc_partition, cc_state from " +
-          "COMPLETED_COMPACTIONS order by cc_database, cc_table, cc_partition, cc_id desc");
-        String lastCompactedEntity = null;
-        /*In each group, walk from most recent and count occurences of each state type.  Once you
-        * have counted enough (for each state) to satisfy retention policy, delete all other
-        * instances of this status.*/
-        while(rs.next()) {
-          CompactionInfo ci = new CompactionInfo(rs.getLong(1), rs.getString(2), rs.getString(3), rs.getString(4), rs.getString(5).charAt(0));
-          if(!ci.getFullPartitionName().equals(lastCompactedEntity)) {
-            lastCompactedEntity = ci.getFullPartitionName();
-            rc = new RetentionCounters(conf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_ATTEMPTED),
-              getFailedCompactionRetention(),
-              conf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_SUCCEEDED));
-          }
-          checkForDeletion(deleteSet, ci, rc);
-        }
-        close(rs);
-
-        if (deleteSet.size() <= 0) {
-          return;
-        }
-
-        List<String> queries = new ArrayList<String>();
-
-        StringBuilder prefix = new StringBuilder();
-        StringBuilder suffix = new StringBuilder();
-
-        prefix.append("delete from COMPLETED_COMPACTIONS where ");
-        suffix.append("");
-
-        TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, deleteSet, "cc_id", false, false);
-
-        for (String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          int count = stmt.executeUpdate(query);
-          LOG.debug("Removed " + count + " records from COMPLETED_COMPACTIONS");
-        }
-        dbConn.commit();
-      } catch (SQLException e) {
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "purgeCompactionHistory()");
-        throw new MetaException("Unable to connect to transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException ex) {
-      purgeCompactionHistory();
-    }
-  }
-  /**
-   * this ensures that the number of failed compaction entries retained is > than number of failed
-   * compaction threshold which prevents new compactions from being scheduled.
-   */
-  private int getFailedCompactionRetention() {
-    int failedThreshold = conf.getIntVar(HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-    int failedRetention = conf.getIntVar(HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED);
-    if(failedRetention < failedThreshold) {
-      LOG.warn("Invalid configuration " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.varname +
-        "=" + failedRetention + " < " + HiveConf.ConfVars.COMPACTOR_HISTORY_RETENTION_FAILED + "=" +
-        failedRetention + ".  Will use " + HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD.varname +
-        "=" + failedRetention);
-      failedRetention = failedThreshold;
-    }
-    return failedRetention;
-  }
-  /**
-   * Returns {@code true} if there already exists sufficient number of consecutive failures for
-   * this table/partition so that no new automatic compactions will be scheduled.
-   * User initiated compactions don't do this check.
-   *
-   * Do we allow compacting whole table (when it's partitioned)?  No, though perhaps we should.
-   * That would be a meta operations, i.e. first find all partitions for this table (which have 
-   * txn info) and schedule each compaction separately.  This avoids complications in this logic.
-   */
-  @Override
-  @RetrySemantics.ReadOnly
-  public boolean checkFailedCompactions(CompactionInfo ci) throws MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        rs = stmt.executeQuery("select CC_STATE from COMPLETED_COMPACTIONS where " +
-          "CC_DATABASE = " + quoteString(ci.dbname) + " and " +
-          "CC_TABLE = " + quoteString(ci.tableName) +
-          (ci.partName != null ? "and CC_PARTITION = " + quoteString(ci.partName) : "") +
-          " and CC_STATE != " + quoteChar(ATTEMPTED_STATE) + " order by CC_ID desc");
-        int numFailed = 0;
-        int numTotal = 0;
-        int failedThreshold = conf.getIntVar(HiveConf.ConfVars.COMPACTOR_INITIATOR_FAILED_THRESHOLD);
-        while(rs.next() && ++numTotal <= failedThreshold) {
-          if(rs.getString(1).charAt(0) == FAILED_STATE) {
-            numFailed++;
-          }
-          else {
-            numFailed--;
-          }
-        }
-        return numFailed == failedThreshold;
-      }
-      catch (SQLException e) {
-        LOG.error("Unable to delete from compaction queue " + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "checkFailedCompactions(" + ci + ")");
-        LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(e));
-        return false;//weren't able to check
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return checkFailedCompactions(ci);
-    }
-  }
-  /**
-   * If there is an entry in compaction_queue with ci.id, remove it
-   * Make entry in completed_compactions with status 'f'.
-   * If there is no entry in compaction_queue, it means Initiator failed to even schedule a compaction,
-   * which we record as ATTEMPTED_STATE entry in history.
-   */
-  @Override
-  @RetrySemantics.CannotRetry
-  public void markFailed(CompactionInfo ci) throws MetaException {//todo: this should not throw
-    //todo: this should take "comment" as parameter to set in CC_META_INFO to provide some context for the failure
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      PreparedStatement pStmt = null;
-      ResultSet rs = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        rs = stmt.executeQuery("select CQ_ID, CQ_DATABASE, CQ_TABLE, CQ_PARTITION, CQ_STATE, CQ_TYPE, CQ_TBLPROPERTIES, CQ_WORKER_ID, CQ_START, CQ_RUN_AS, CQ_HIGHEST_TXN_ID, CQ_META_INFO, CQ_HADOOP_JOB_ID from COMPACTION_QUEUE WHERE CQ_ID = " + ci.id);
-        if(rs.next()) {
-          ci = CompactionInfo.loadFullFromCompactionQueue(rs);
-          String s = "delete from COMPACTION_QUEUE where cq_id = " + ci.id;
-          LOG.debug("Going to execute update <" + s + ">");
-          int updCnt = stmt.executeUpdate(s);
-        }
-        else {
-          if(ci.id > 0) {
-            //the record with valid CQ_ID has disappeared - this is a sign of something wrong
-            throw new IllegalStateException("No record with CQ_ID=" + ci.id + " found in COMPACTION_QUEUE");
-          }
-        }
-        if(ci.id == 0) {
-          //The failure occurred before we even made an entry in COMPACTION_QUEUE
-          //generate ID so that we can make an entry in COMPLETED_COMPACTIONS
-          ci.id = generateCompactionQueueId(stmt);
-          //mostly this indicates that the Initiator is paying attention to some table even though
-          //compactions are not happening.
-          ci.state = ATTEMPTED_STATE;
-          //this is not strictly accurate, but 'type' cannot be null.
-          if(ci.type == null) { ci.type = CompactionType.MINOR; }
-          ci.start = getDbTime(dbConn);
-        }
-        else {
-          ci.state = FAILED_STATE;
-        }
-        close(rs, stmt, null);
-
-        pStmt = dbConn.prepareStatement("insert into COMPLETED_COMPACTIONS(CC_ID, CC_DATABASE, CC_TABLE, CC_PARTITION, CC_STATE, CC_TYPE, CC_TBLPROPERTIES, CC_WORKER_ID, CC_START, CC_END, CC_RUN_AS, CC_HIGHEST_TXN_ID, CC_META_INFO, CC_HADOOP_JOB_ID) VALUES(?,?,?,?,?, ?,?,?,?,?, ?,?,?,?)");
-        CompactionInfo.insertIntoCompletedCompactions(pStmt, ci, getDbTime(dbConn));
-        int updCount = pStmt.executeUpdate();
-        LOG.debug("Going to commit");
-        closeStmt(pStmt);
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.warn("markFailed(" + ci.id + "):" + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        try {
-          checkRetryable(dbConn, e, "markFailed(" + ci + ")");
-        }
-        catch(MetaException ex) {
-          LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(ex));
-        }
-        LOG.error("markFailed(" + ci + ") failed: " + e.getMessage(), e);
-      } finally {
-        close(rs, stmt, null);
-        close(null, pStmt, dbConn);
-      }
-    } catch (RetryException e) {
-      markFailed(ci);
-    }
-  }
-  @Override
-  @RetrySemantics.Idempotent
-  public void setHadoopJobId(String hadoopJobId, long id) {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "update COMPACTION_QUEUE set CQ_HADOOP_JOB_ID = " + quoteString(hadoopJobId) + " WHERE CQ_ID = " + id;
-        LOG.debug("Going to execute <" + s + ">");
-        int updateCount = stmt.executeUpdate(s);
-        LOG.debug("Going to commit");
-        closeStmt(stmt);
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.warn("setHadoopJobId(" + hadoopJobId + "," + id + "):" + e.getMessage());
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        try {
-          checkRetryable(dbConn, e, "setHadoopJobId(" + hadoopJobId + "," + id + ")");
-        }
-        catch(MetaException ex) {
-          LOG.error("Unable to connect to transaction database " + StringUtils.stringifyException(ex));
-        }
-        LOG.error("setHadoopJobId(" + hadoopJobId + "," + id + ") failed: " + e.getMessage(), e);
-      } finally {
-        close(null, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      setHadoopJobId(hadoopJobId, id);
-    }
-  }
-}
-
-

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
deleted file mode 100644
index 0161894..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java
+++ /dev/null
@@ -1,369 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import java.sql.Connection;
-import java.sql.Driver;
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.ResultSetMetaData;
-import java.sql.SQLException;
-import java.sql.SQLTransactionRollbackException;
-import java.sql.Statement;
-import java.util.Properties;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.hive.conf.HiveConf;
-
-/**
- * Utility methods for creating and destroying txn database/schema, plus methods for
- * querying against metastore tables.
- * Placed here in a separate class so it can be shared across unit tests.
- */
-public final class TxnDbUtil {
-
-  static final private Logger LOG = LoggerFactory.getLogger(TxnDbUtil.class.getName());
-  private static final String TXN_MANAGER = "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager";
-
-  private static int deadlockCnt = 0;
-
-  private TxnDbUtil() {
-    throw new UnsupportedOperationException("Can't initialize class");
-  }
-
-  /**
-   * Set up the configuration so it will use the DbTxnManager, concurrency will be set to true,
-   * and the JDBC configs will be set for putting the transaction and lock info in the embedded
-   * metastore.
-   *
-   * @param conf HiveConf to add these values to
-   */
-  public static void setConfValues(HiveConf conf) {
-    conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, TXN_MANAGER);
-    conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
-  }
-
-  public static void prepDb() throws Exception {
-    // This is a bogus hack because it copies the contents of the SQL file
-    // intended for creating derby databases, and thus will inexorably get
-    // out of date with it.  I'm open to any suggestions on how to make this
-    // read the file in a build friendly way.
-
-    Connection conn = null;
-    Statement stmt = null;
-    try {
-      conn = getConnection();
-      stmt = conn.createStatement();
-      stmt.execute("CREATE TABLE TXNS (" +
-          "  TXN_ID bigint PRIMARY KEY," +
-          "  TXN_STATE char(1) NOT NULL," +
-          "  TXN_STARTED bigint NOT NULL," +
-          "  TXN_LAST_HEARTBEAT bigint NOT NULL," +
-          "  TXN_USER varchar(128) NOT NULL," +
-          "  TXN_HOST varchar(128) NOT NULL)");
-
-      stmt.execute("CREATE TABLE TXN_COMPONENTS (" +
-          "  TC_TXNID bigint REFERENCES TXNS (TXN_ID)," +
-          "  TC_DATABASE varchar(128) NOT NULL," +
-          "  TC_TABLE varchar(128)," +
-          "  TC_PARTITION varchar(767)," +
-          "  TC_OPERATION_TYPE char(1) NOT NULL)");
-      stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" +
-          "  CTC_TXNID bigint," +
-          "  CTC_DATABASE varchar(128) NOT NULL," +
-          "  CTC_TABLE varchar(128)," +
-          "  CTC_PARTITION varchar(767))");
-      stmt.execute("CREATE TABLE NEXT_TXN_ID (" + "  NTXN_NEXT bigint NOT NULL)");
-      stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)");
-      stmt.execute("CREATE TABLE HIVE_LOCKS (" +
-          " HL_LOCK_EXT_ID bigint NOT NULL," +
-          " HL_LOCK_INT_ID bigint NOT NULL," +
-          " HL_TXNID bigint," +
-          " HL_DB varchar(128) NOT NULL," +
-          " HL_TABLE varchar(128)," +
-          " HL_PARTITION varchar(767)," +
-          " HL_LOCK_STATE char(1) NOT NULL," +
-          " HL_LOCK_TYPE char(1) NOT NULL," +
-          " HL_LAST_HEARTBEAT bigint NOT NULL," +
-          " HL_ACQUIRED_AT bigint," +
-          " HL_USER varchar(128) NOT NULL," +
-          " HL_HOST varchar(128) NOT NULL," +
-          " HL_HEARTBEAT_COUNT integer," +
-          " HL_AGENT_INFO varchar(128)," +
-          " HL_BLOCKEDBY_EXT_ID bigint," +
-          " HL_BLOCKEDBY_INT_ID bigint," +
-        " PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))");
-      stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)");
-
-      stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)");
-      stmt.execute("INSERT INTO NEXT_LOCK_ID VALUES(1)");
-
-      stmt.execute("CREATE TABLE COMPACTION_QUEUE (" +
-          " CQ_ID bigint PRIMARY KEY," +
-          " CQ_DATABASE varchar(128) NOT NULL," +
-          " CQ_TABLE varchar(128) NOT NULL," +
-          " CQ_PARTITION varchar(767)," +
-          " CQ_STATE char(1) NOT NULL," +
-          " CQ_TYPE char(1) NOT NULL," +
-          " CQ_TBLPROPERTIES varchar(2048)," +
-          " CQ_WORKER_ID varchar(128)," +
-          " CQ_START bigint," +
-          " CQ_RUN_AS varchar(128)," +
-          " CQ_HIGHEST_TXN_ID bigint," +
-          " CQ_META_INFO varchar(2048) for bit data," +
-          " CQ_HADOOP_JOB_ID varchar(32))");
-
-      stmt.execute("CREATE TABLE NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT bigint NOT NULL)");
-      stmt.execute("INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1)");
-      
-      stmt.execute("CREATE TABLE COMPLETED_COMPACTIONS (" +
-        " CC_ID bigint PRIMARY KEY," +
-        " CC_DATABASE varchar(128) NOT NULL," +
-        " CC_TABLE varchar(128) NOT NULL," +
-        " CC_PARTITION varchar(767)," +
-        " CC_STATE char(1) NOT NULL," +
-        " CC_TYPE char(1) NOT NULL," +
-        " CC_TBLPROPERTIES varchar(2048)," +
-        " CC_WORKER_ID varchar(128)," +
-        " CC_START bigint," +
-        " CC_END bigint," +
-        " CC_RUN_AS varchar(128)," +
-        " CC_HIGHEST_TXN_ID bigint," +
-        " CC_META_INFO varchar(2048) for bit data," +
-        " CC_HADOOP_JOB_ID varchar(32))");
-      
-      stmt.execute("CREATE TABLE AUX_TABLE (" +
-        " MT_KEY1 varchar(128) NOT NULL," +
-        " MT_KEY2 bigint NOT NULL," +
-        " MT_COMMENT varchar(255)," +
-        " PRIMARY KEY(MT_KEY1, MT_KEY2))");
-      
-      stmt.execute("CREATE TABLE WRITE_SET (" +
-        " WS_DATABASE varchar(128) NOT NULL," +
-        " WS_TABLE varchar(128) NOT NULL," +
-        " WS_PARTITION varchar(767)," +
-        " WS_TXNID bigint NOT NULL," +
-        " WS_COMMIT_ID bigint NOT NULL," +
-        " WS_OPERATION_TYPE char(1) NOT NULL)"
-      );
-    } catch (SQLException e) {
-      try {
-        conn.rollback();
-      } catch (SQLException re) {
-        LOG.error("Error rolling back: " + re.getMessage());
-      }
-
-      // This might be a deadlock, if so, let's retry
-      if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) {
-        LOG.warn("Caught deadlock, retrying db creation");
-        prepDb();
-      } else {
-        throw e;
-      }
-    } finally {
-      deadlockCnt = 0;
-      closeResources(conn, stmt, null);
-    }
-  }
-
-  public static void cleanDb() throws Exception {
-    int retryCount = 0;
-    while(++retryCount <= 3) {
-      boolean success = true;
-      Connection conn = null;
-      Statement stmt = null;
-      try {
-        conn = getConnection();
-        stmt = conn.createStatement();
-
-        // We want to try these, whether they succeed or fail.
-        try {
-          stmt.execute("DROP INDEX HL_TXNID_INDEX");
-        } catch (SQLException e) {
-          if(!("42X65".equals(e.getSQLState()) && 30000 == e.getErrorCode())) {
-            //42X65/3000 means index doesn't exist
-            LOG.error("Unable to drop index HL_TXNID_INDEX " + e.getMessage() +
-              "State=" + e.getSQLState() + " code=" + e.getErrorCode() + " retryCount=" + retryCount);
-            success = false;
-          }
-        }
-
-        success &= dropTable(stmt, "TXN_COMPONENTS", retryCount);
-        success &= dropTable(stmt, "COMPLETED_TXN_COMPONENTS", retryCount);
-        success &= dropTable(stmt, "TXNS", retryCount);
-        success &= dropTable(stmt, "NEXT_TXN_ID", retryCount);
-        success &= dropTable(stmt, "HIVE_LOCKS", retryCount);
-        success &= dropTable(stmt, "NEXT_LOCK_ID", retryCount);
-        success &= dropTable(stmt, "COMPACTION_QUEUE", retryCount);
-        success &= dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID", retryCount);
-        success &= dropTable(stmt, "COMPLETED_COMPACTIONS", retryCount);
-        success &= dropTable(stmt, "AUX_TABLE", retryCount);
-        success &= dropTable(stmt, "WRITE_SET", retryCount);
-      } finally {
-        closeResources(conn, stmt, null);
-      }
-      if(success) {
-        return;
-      }
-    }
-  }
-
-  private static boolean dropTable(Statement stmt, String name, int retryCount) throws SQLException {
-    try {
-      stmt.execute("DROP TABLE " + name);
-      return true;
-    } catch (SQLException e) {
-      if("42Y55".equals(e.getSQLState()) && 30000 == e.getErrorCode()) {
-        //failed because object doesn't exist
-        return true;
-      }
-      LOG.error("Unable to drop table " + name + ": " + e.getMessage() +
-        " State=" + e.getSQLState() + " code=" + e.getErrorCode() + " retryCount=" + retryCount);
-    }
-    return false;
-  }
-
-  /**
-   * A tool to count the number of partitions, tables,
-   * and databases locked by a particular lockId.
-   *
-   * @param lockId lock id to look for lock components
-   *
-   * @return number of components, or 0 if there is no lock
-   */
-  public static int countLockComponents(long lockId) throws Exception {
-    Connection conn = null;
-    PreparedStatement stmt = null;
-    ResultSet rs = null;
-    try {
-      conn = getConnection();
-      stmt = conn.prepareStatement("SELECT count(*) FROM hive_locks WHERE hl_lock_ext_id = ?");
-      stmt.setLong(1, lockId);
-      rs = stmt.executeQuery();
-      if (!rs.next()) {
-        return 0;
-      }
-      return rs.getInt(1);
-    } finally {
-      closeResources(conn, stmt, rs);
-    }
-  }
-
-  /**
-   * Utility method used to run COUNT queries like "select count(*) from ..." against metastore tables
-   * @param countQuery countQuery text
-   * @return count countQuery result
-   * @throws Exception
-   */
-  public static int countQueryAgent(String countQuery) throws Exception {
-    Connection conn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      conn = getConnection();
-      stmt = conn.createStatement();
-      rs = stmt.executeQuery(countQuery);
-      if (!rs.next()) {
-        return 0;
-      }
-      return rs.getInt(1);
-    } finally {
-      closeResources(conn, stmt, rs);
-    }
-  }
-  @VisibleForTesting
-  public static String queryToString(String query) throws Exception {
-    return queryToString(query, true);
-  }
-  public static String queryToString(String query, boolean includeHeader) throws Exception {
-    Connection conn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    StringBuilder sb = new StringBuilder();
-    try {
-      conn = getConnection();
-      stmt = conn.createStatement();
-      rs = stmt.executeQuery(query);
-      ResultSetMetaData rsmd = rs.getMetaData();
-      if(includeHeader) {
-        for (int colPos = 1; colPos <= rsmd.getColumnCount(); colPos++) {
-          sb.append(rsmd.getColumnName(colPos)).append("   ");
-        }
-        sb.append('\n');
-      }
-      while(rs.next()) {
-        for (int colPos = 1; colPos <= rsmd.getColumnCount(); colPos++) {
-          sb.append(rs.getObject(colPos)).append("   ");
-        }
-        sb.append('\n');
-      }
-    } finally {
-      closeResources(conn, stmt, rs);
-    }
-    return sb.toString();
-  }
-
-  static Connection getConnection() throws Exception {
-    HiveConf conf = new HiveConf();
-    String jdbcDriver = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER);
-    Driver driver = (Driver) Class.forName(jdbcDriver).newInstance();
-    Properties prop = new Properties();
-    String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
-    String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
-    String passwd = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.PWD);
-    prop.setProperty("user", user);
-    prop.setProperty("password", passwd);
-    Connection conn = driver.connect(driverUrl, prop);
-    conn.setAutoCommit(true);
-    return conn;
-  }
-
-  static void closeResources(Connection conn, Statement stmt, ResultSet rs) {
-    if (rs != null) {
-      try {
-        rs.close();
-      } catch (SQLException e) {
-        LOG.error("Error closing ResultSet: " + e.getMessage());
-      }
-    }
-
-    if (stmt != null) {
-      try {
-        stmt.close();
-      } catch (SQLException e) {
-        System.err.println("Error closing Statement: " + e.getMessage());
-      }
-    }
-
-    if (conn != null) {
-      try {
-        conn.rollback();
-      } catch (SQLException e) {
-        System.err.println("Error rolling back: " + e.getMessage());
-      }
-      try {
-        conn.close();
-      } catch (SQLException e) {
-        System.err.println("Error closing Connection: " + e.getMessage());
-      }
-    }
-  }
-}


[09/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
deleted file mode 100644
index f77900d..0000000
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ /dev/null
@@ -1,3667 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.commons.dbcp.ConnectionFactory;
-import org.apache.commons.dbcp.DriverManagerConnectionFactory;
-import org.apache.commons.dbcp.PoolableConnectionFactory;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.hadoop.hive.common.ServerUtils;
-import org.apache.hadoop.hive.common.classification.InterfaceAudience;
-import org.apache.hadoop.hive.common.classification.InterfaceStability;
-import org.apache.hadoop.hive.common.classification.RetrySemantics;
-import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.HouseKeeperService;
-import org.apache.hadoop.hive.metastore.Warehouse;
-import org.apache.hadoop.hive.metastore.datasource.BoneCPDataSourceProvider;
-import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
-import org.apache.hadoop.hive.metastore.datasource.HikariCPDataSourceProvider;
-import org.apache.hadoop.hive.metastore.metrics.Metrics;
-import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
-import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.commons.dbcp.PoolingDataSource;
-
-import org.apache.commons.pool.impl.GenericObjectPool;
-import org.apache.hadoop.hive.common.JavaUtils;
-import org.apache.hadoop.hive.common.StringableMap;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.conf.HiveConfUtil;
-import org.apache.hadoop.hive.metastore.api.*;
-import org.apache.hadoop.util.StringUtils;
-
-import javax.sql.DataSource;
-
-import java.io.PrintWriter;
-import java.nio.ByteBuffer;
-import java.sql.*;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
-
-/**
- * A handler to answer transaction related calls that come into the metastore
- * server.
- *
- * Note on log messages:  Please include txnid:X and lockid info using
- * {@link org.apache.hadoop.hive.common.JavaUtils#txnIdToString(long)}
- * and {@link org.apache.hadoop.hive.common.JavaUtils#lockIdToString(long)} in all messages.
- * The txnid:X and lockid:Y matches how Thrift object toString() methods are generated,
- * so keeping the format consistent makes grep'ing the logs much easier.
- *
- * Note on HIVE_LOCKS.hl_last_heartbeat.
- * For locks that are part of transaction, we set this 0 (would rather set it to NULL but
- * Currently the DB schema has this NOT NULL) and only update/read heartbeat from corresponding
- * transaction in TXNS.
- *
- * In general there can be multiple metastores where this logic can execute, thus the DB is
- * used to ensure proper mutexing of operations.
- * Select ... For Update (or equivalent: either MsSql with(updlock) or actual Update stmt) is
- * used to properly sequence operations.  Most notably:
- * 1. various sequence IDs are generated with aid of this mutex
- * 2. ensuring that each (Hive) Transaction state is transitioned atomically.  Transaction state
- *  includes its actual state (Open, Aborted) as well as it's lock list/component list.  Thus all
- *  per transaction ops, either start by update/delete of the relevant TXNS row or do S4U on that row.
- *  This allows almost all operations to run at READ_COMMITTED and minimizes DB deadlocks.
- * 3. checkLock() - this is mutexted entirely since we must ensure that while we check if some lock
- *  can be granted, no other (strictly speaking "earlier") lock can change state.
- *
- * The exception to his is Derby which doesn't support proper S4U.  Derby is always running embedded
- * (this is the only supported configuration for Derby)
- * in the same JVM as HiveMetaStoreHandler thus we use JVM wide lock to properly sequnce the operations.
- *
- * {@link #derbyLock}
-
- * If we ever decide to run remote Derby server, according to
- * https://db.apache.org/derby/docs/10.0/manuals/develop/develop78.html all transactions will be
- * seriazlied, so that would also work though has not been tested.
- *
- * General design note:
- * It's imperative that any operation on a txn (e.g. commit), ensure (atomically) that this txn is
- * still valid and active.  In the code this is usually achieved at the same time the txn record
- * is locked for some operation.
- * 
- * Note on retry logic:
- * Metastore has retry logic in both {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient}
- * and {@link org.apache.hadoop.hive.metastore.RetryingHMSHandler}.  The retry logic there is very
- * generic and is not aware whether the operations are idempotent or not.  (This is separate from
- * retry logic here in TxnHander which can/does retry DB errors intelligently).  The worst case is
- * when an op here issues a successful commit against the RDBMS but the calling stack doesn't
- * receive the ack and retries.  (If an op fails before commit, it's trivially idempotent)
- * Thus the ops here need to be made idempotent as much as possible or
- * the metstore call stack should have logic not to retry.  There are {@link RetrySemantics}
- * annotations to document the behavior.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
-
-  static final protected char INITIATED_STATE = 'i';
-  static final protected char WORKING_STATE = 'w';
-  static final protected char READY_FOR_CLEANING = 'r';
-  static final char FAILED_STATE = 'f';
-  static final char SUCCEEDED_STATE = 's';
-  static final char ATTEMPTED_STATE = 'a';
-
-  // Compactor types
-  static final protected char MAJOR_TYPE = 'a';
-  static final protected char MINOR_TYPE = 'i';
-
-  // Transaction states
-  static final protected char TXN_ABORTED = 'a';
-  static final protected char TXN_OPEN = 'o';
-  //todo: make these like OperationType and remove above char constatns
-  enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN}
-
-  // Lock states
-  static final protected char LOCK_ACQUIRED = 'a';
-  static final protected char LOCK_WAITING = 'w';
-
-  // Lock types
-  static final protected char LOCK_EXCLUSIVE = 'e';
-  static final protected char LOCK_SHARED = 'r';
-  static final protected char LOCK_SEMI_SHARED = 'w';
-
-  static final private int ALLOWED_REPEATED_DEADLOCKS = 10;
-  static final private Logger LOG = LoggerFactory.getLogger(TxnHandler.class.getName());
-
-  static private DataSource connPool;
-  private static DataSource connPoolMutex;
-  static private boolean doRetryOnConnPool = false;
-  
-  private enum OpertaionType {
-    SELECT('s'), INSERT('i'), UPDATE('u'), DELETE('d');
-    private final char sqlConst;
-    OpertaionType(char sqlConst) {
-      this.sqlConst = sqlConst;
-    }
-    public String toString() {
-      return Character.toString(sqlConst);
-    }
-    public static OpertaionType fromString(char sqlConst) {
-      switch (sqlConst) {
-        case 's':
-          return SELECT;
-        case 'i':
-          return INSERT;
-        case 'u':
-          return UPDATE;
-        case 'd':
-          return DELETE;
-        default:
-          throw new IllegalArgumentException(quoteChar(sqlConst));
-      }
-    }
-    public static OpertaionType fromDataOperationType(DataOperationType dop) {
-      switch (dop) {
-        case SELECT:
-          return OpertaionType.SELECT;
-        case INSERT:
-          return OpertaionType.INSERT;
-        case UPDATE:
-          return OpertaionType.UPDATE;
-        case DELETE:
-          return OpertaionType.DELETE;
-        default:
-          throw new IllegalArgumentException("Unexpected value: " + dop);
-      }
-    }
-  }
-
-  // Maximum number of open transactions that's allowed
-  private static volatile int maxOpenTxns = 0;
-  // Whether number of open transactions reaches the threshold
-  private static volatile boolean tooManyOpenTxns = false;
-  // The AcidHouseKeeperService for counting open transactions
-  private static volatile HouseKeeperService openTxnsCounter = null;
-
-  /**
-   * Number of consecutive deadlocks we have seen
-   */
-  private int deadlockCnt;
-  private long deadlockRetryInterval;
-  protected HiveConf conf;
-  private static DatabaseProduct dbProduct;
-  private static SQLGenerator sqlGenerator;
-
-  // (End user) Transaction timeout, in milliseconds.
-  private long timeout;
-
-  private String identifierQuoteString; // quotes to use for quoting tables, where necessary
-  private long retryInterval;
-  private int retryLimit;
-  private int retryNum;
-  // Current number of open txns
-  private AtomicInteger numOpenTxns;
-
-  /**
-   * Derby specific concurrency control
-   */
-  private static final ReentrantLock derbyLock = new ReentrantLock(true);
-  /**
-   * must be static since even in UT there may be > 1 instance of TxnHandler
-   * (e.g. via Compactor services)
-   */
-  private final static ConcurrentHashMap<String, Semaphore> derbyKey2Lock = new ConcurrentHashMap<>();
-  private static final String hostname = ServerUtils.hostname();
-
-  // Private methods should never catch SQLException and then throw MetaException.  The public
-  // methods depend on SQLException coming back so they can detect and handle deadlocks.  Private
-  // methods should only throw MetaException when they explicitly know there's a logic error and
-  // they want to throw past the public methods.
-  //
-  // All public methods that write to the database have to check for deadlocks when a SQLException
-  // comes back and handle it if they see one.  This has to be done with the connection pooling
-  // in mind.  To do this they should call checkRetryable() AFTER rolling back the db transaction,
-  // and then they should catch RetryException and call themselves recursively. See commitTxn for an example.
-
-  public TxnHandler() {
-  }
-
-  /**
-   * This is logically part of c'tor and must be called prior to any other method.
-   * Not physically part of c'tor due to use of relfection
-   */
-  public void setConf(HiveConf conf) {
-    this.conf = conf;
-
-    checkQFileTestHack();
-
-    synchronized (TxnHandler.class) {
-      if (connPool == null) {
-        //only do this once per JVM; useful for support
-        LOG.info(HiveConfUtil.dumpConfig(conf).toString());
-
-        Connection dbConn = null;
-        // Set up the JDBC connection pool
-        try {
-          int maxPoolSize = conf.getIntVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_MAX_CONNECTIONS);
-          long getConnectionTimeoutMs = 30000;
-          connPool = setupJdbcConnectionPool(conf, maxPoolSize, getConnectionTimeoutMs);
-          /*the mutex pools should ideally be somewhat larger since some operations require 1
-           connection from each pool and we want to avoid taking a connection from primary pool
-           and then blocking because mutex pool is empty.  There is only 1 thread in any HMS trying
-           to mutex on each MUTEX_KEY except MUTEX_KEY.CheckLock.  The CheckLock operation gets a
-           connection from connPool first, then connPoolMutex.  All others, go in the opposite
-           order (not very elegant...).  So number of connection requests for connPoolMutex cannot
-           exceed (size of connPool + MUTEX_KEY.values().length - 1).*/
-          connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize + MUTEX_KEY.values().length, getConnectionTimeoutMs);
-          dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-          determineDatabaseProduct(dbConn);
-          sqlGenerator = new SQLGenerator(dbProduct, conf);
-        } catch (SQLException e) {
-          String msg = "Unable to instantiate JDBC connection pooling, " + e.getMessage();
-          LOG.error(msg);
-          throw new RuntimeException(e);
-        } finally {
-          closeDbConn(dbConn);
-        }
-      }
-    }
-
-    numOpenTxns = Metrics.getOrCreateGauge(MetricsConstants.NUM_OPEN_TXNS);
-
-    timeout = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HIVE_TXN_TIMEOUT, TimeUnit.MILLISECONDS);
-    buildJumpTable();
-    retryInterval = HiveConf.getTimeVar(conf, HiveConf.ConfVars.HMSHANDLERINTERVAL,
-        TimeUnit.MILLISECONDS);
-    retryLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.HMSHANDLERATTEMPTS);
-    deadlockRetryInterval = retryInterval / 10;
-    maxOpenTxns = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVE_MAX_OPEN_TXNS);
-  }
-  @Override
-  @RetrySemantics.ReadOnly
-  public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException {
-    try {
-      // We need to figure out the current transaction number and the list of
-      // open transactions.  To avoid needing a transaction on the underlying
-      // database we'll look at the current transaction number first.  If it
-      // subsequently shows up in the open list that's ok.
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-      try {
-        /**
-         * This method can run at READ_COMMITTED as long as long as
-         * {@link #openTxns(org.apache.hadoop.hive.metastore.api.OpenTxnRequest)} is atomic.
-         * More specifically, as long as advancing TransactionID in NEXT_TXN_ID is atomic with
-         * adding corresponding entries into TXNS.  The reason is that any txnid below HWM
-         * is either in TXNS and thus considered open (Open/Aborted) or it's considered Committed.
-         */
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select ntxn_next - 1 from NEXT_TXN_ID";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly " +
-            "initialized, no record found in next_txn_id");
-        }
-        long hwm = rs.getLong(1);
-        if (rs.wasNull()) {
-          throw new MetaException("Transaction tables not properly " +
-            "initialized, null record found in next_txn_id");
-        }
-        close(rs);
-        List<TxnInfo> txnInfos = new ArrayList<TxnInfo>();
-        //need the WHERE clause below to ensure consistent results with READ_COMMITTED
-        s = "select txn_id, txn_state, txn_user, txn_host, txn_started, txn_last_heartbeat from " +
-            "TXNS where txn_id <= " + hwm;
-        LOG.debug("Going to execute query<" + s + ">");
-        rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          char c = rs.getString(2).charAt(0);
-          TxnState state;
-          switch (c) {
-            case TXN_ABORTED:
-              state = TxnState.ABORTED;
-              break;
-
-            case TXN_OPEN:
-              state = TxnState.OPEN;
-              break;
-
-            default:
-              throw new MetaException("Unexpected transaction state " + c +
-                " found in txns table");
-          }
-          TxnInfo txnInfo = new TxnInfo(rs.getLong(1), state, rs.getString(3), rs.getString(4));
-          txnInfo.setStartedTime(rs.getLong(5));
-          txnInfo.setLastHeartbeatTime(rs.getLong(6));
-          txnInfos.add(txnInfo);
-        }
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-        return new GetOpenTxnsInfoResponse(hwm, txnInfos);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "getOpenTxnsInfo");
-        throw new MetaException("Unable to select from transaction database: " + getMessage(e)
-          + StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return getOpenTxnsInfo();
-    }
-  }
-  @Override
-  @RetrySemantics.ReadOnly
-  public GetOpenTxnsResponse getOpenTxns() throws MetaException {
-    try {
-      // We need to figure out the current transaction number and the list of
-      // open transactions.  To avoid needing a transaction on the underlying
-      // database we'll look at the current transaction number first.  If it
-      // subsequently shows up in the open list that's ok.
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-      try {
-        /**
-         * This runs at READ_COMMITTED for exactly the same reason as {@link #getOpenTxnsInfo()}
-         */
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select ntxn_next - 1 from NEXT_TXN_ID";
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction tables not properly " +
-            "initialized, no record found in next_txn_id");
-        }
-        long hwm = rs.getLong(1);
-        if (rs.wasNull()) {
-          throw new MetaException("Transaction tables not properly " +
-            "initialized, null record found in next_txn_id");
-        }
-        close(rs);
-        List<Long> openList = new ArrayList<Long>();
-        //need the WHERE clause below to ensure consistent results with READ_COMMITTED
-        s = "select txn_id, txn_state from TXNS where txn_id <= " + hwm + " order by txn_id";
-        LOG.debug("Going to execute query<" + s + ">");
-        rs = stmt.executeQuery(s);
-        long minOpenTxn = Long.MAX_VALUE;
-        BitSet abortedBits = new BitSet();
-        while (rs.next()) {
-          long txnId = rs.getLong(1);
-          openList.add(txnId);
-          char c = rs.getString(2).charAt(0);
-          if(c == TXN_OPEN) {
-            minOpenTxn = Math.min(minOpenTxn, txnId);
-          } else if (c == TXN_ABORTED) {
-            abortedBits.set(openList.size() - 1);
-          }
-        }
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-        ByteBuffer byteBuffer = ByteBuffer.wrap(abortedBits.toByteArray());
-        GetOpenTxnsResponse otr = new GetOpenTxnsResponse(hwm, openList, byteBuffer);
-        if(minOpenTxn < Long.MAX_VALUE) {
-          otr.setMin_open_txn(minOpenTxn);
-        }
-        return otr;
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "getOpenTxns");
-        throw new MetaException("Unable to select from transaction database, "
-          + StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return getOpenTxns();
-    }
-  }
-
-  private static void startHouseKeeperService(HiveConf conf, Class c){
-    try {
-      openTxnsCounter = (HouseKeeperService)c.newInstance();
-      openTxnsCounter.start(conf);
-    } catch (Exception ex) {
-      LOG.error("Failed to start {}" , openTxnsCounter.getClass() +
-              ".  The system will not handle {} " , openTxnsCounter.getServiceDescription(),
-          ".  Root Cause: ", ex);
-    }
-  }
-
-  /**
-   * Retry-by-caller note:
-   * Worst case, it will leave an open txn which will timeout.
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException {
-    if (openTxnsCounter == null) {
-      synchronized (TxnHandler.class) {
-        try {
-          if (openTxnsCounter == null) {
-            startHouseKeeperService(conf, Class.forName("org.apache.hadoop.hive.ql.txn.AcidOpenTxnsCounterService"));
-          }
-        } catch (ClassNotFoundException e) {
-          throw new MetaException(e.getMessage());
-        }
-      }
-    }
-
-    if (!tooManyOpenTxns && numOpenTxns.get() >= maxOpenTxns) {
-      tooManyOpenTxns = true;
-    }
-    if (tooManyOpenTxns) {
-      if (numOpenTxns.get() < maxOpenTxns * 0.9) {
-        tooManyOpenTxns = false;
-      } else {
-        LOG.warn("Maximum allowed number of open transactions (" + maxOpenTxns + ") has been " +
-            "reached. Current number of open transactions: " + numOpenTxns);
-        throw new MetaException("Maximum allowed number of open transactions has been reached. " +
-            "See hive.max.open.txns.");
-      }
-    }
-
-    int numTxns = rqst.getNum_txns();
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet rs = null;
-      try {
-        lockInternal();
-        /**
-         * To make {@link #getOpenTxns()}/{@link #getOpenTxnsInfo()} work correctly, this operation must ensure
-         * that advancing the counter in NEXT_TXN_ID and adding appropriate entries to TXNS is atomic.
-         * Also, advancing the counter must work when multiple metastores are running.
-         * SELECT ... FOR UPDATE is used to prevent
-         * concurrent DB transactions being rolled back due to Write-Write conflict on NEXT_TXN_ID.
-         *
-         * In the current design, there can be several metastore instances running in a given Warehouse.
-         * This makes ideas like reserving a range of IDs to save trips to DB impossible.  For example,
-         * a client may go to MS1 and start a transaction with ID 500 to update a particular row.
-         * Now the same client will start another transaction, except it ends up on MS2 and may get
-         * transaction ID 400 and update the same row.  Now the merge that happens to materialize the snapshot
-         * on read will thing the version of the row from transaction ID 500 is the latest one.
-         *
-         * Longer term we can consider running Active-Passive MS (at least wrt to ACID operations).  This
-         * set could support a write-through cache for added performance.
-         */
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        // Make sure the user has not requested an insane amount of txns.
-        int maxTxns = HiveConf.getIntVar(conf,
-          HiveConf.ConfVars.HIVE_TXN_MAX_OPEN_BATCH);
-        if (numTxns > maxTxns) numTxns = maxTxns;
-
-        stmt = dbConn.createStatement();
-        String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID");
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          throw new MetaException("Transaction database not properly " +
-            "configured, can't find next transaction id.");
-        }
-        long first = rs.getLong(1);
-        s = "update NEXT_TXN_ID set ntxn_next = " + (first + numTxns);
-        LOG.debug("Going to execute update <" + s + ">");
-        stmt.executeUpdate(s);
-
-        long now = getDbTime(dbConn);
-        List<Long> txnIds = new ArrayList<Long>(numTxns);
-
-        List<String> rows = new ArrayList<>();
-        for (long i = first; i < first + numTxns; i++) {
-          txnIds.add(i);
-          rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()));
-        }
-        List<String> queries = sqlGenerator.createInsertValuesStmt(
-          "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows);
-        for (String q : queries) {
-          LOG.debug("Going to execute update <" + q + ">");
-          stmt.execute(q);
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-        return new OpenTxnsResponse(txnIds);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "openTxns(" + rqst + ")");
-        throw new MetaException("Unable to select from transaction database "
-          + StringUtils.stringifyException(e));
-      } finally {
-        close(rs, stmt, dbConn);
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      return openTxns(rqst);
-    }
-  }
-  @Override
-  @RetrySemantics.Idempotent
-  public void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException, TxnAbortedException {
-    long txnid = rqst.getTxnid();
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      try {
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        if (abortTxns(dbConn, Collections.singletonList(txnid), true) != 1) {
-          stmt = dbConn.createStatement();
-          TxnStatus status = findTxnState(txnid,stmt);
-          if(status == TxnStatus.ABORTED) {
-            LOG.info("abortTxn(" + JavaUtils.txnIdToString(txnid) +
-              ") requested by it is already " + TxnStatus.ABORTED);
-            return;
-          }
-          raiseTxnUnexpectedState(status, txnid);
-        }
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "abortTxn(" + rqst + ")");
-        throw new MetaException("Unable to update transaction database "
-          + StringUtils.stringifyException(e));
-      } finally {
-        close(null, stmt, dbConn);
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      abortTxn(rqst);
-    }
-  }
-  @Override
-  @RetrySemantics.Idempotent
-  public void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaException {
-    List<Long> txnids = rqst.getTxn_ids();
-    try {
-      Connection dbConn = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        int numAborted = abortTxns(dbConn, txnids, false);
-        if (numAborted != txnids.size()) {
-          LOG.warn("Abort Transactions command only aborted " + numAborted + " out of " +
-              txnids.size() + " transactions. It's possible that the other " +
-              (txnids.size() - numAborted) +
-              " transactions have been aborted or committed, or the transaction ids are invalid.");
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "abortTxns(" + rqst + ")");
-        throw new MetaException("Unable to update transaction database "
-            + StringUtils.stringifyException(e));
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      abortTxns(rqst);
-    }
-  }
-
-  /**
-   * Concurrency/isolation notes:
-   * This is mutexed with {@link #openTxns(OpenTxnRequest)} and other {@link #commitTxn(CommitTxnRequest)}
-   * operations using select4update on NEXT_TXN_ID.  Also, mutexes on TXNX table for specific txnid:X
-   * see more notes below.
-   * In order to prevent lost updates, we need to determine if any 2 transactions overlap.  Each txn
-   * is viewed as an interval [M,N]. M is the txnid and N is taken from the same NEXT_TXN_ID sequence
-   * so that we can compare commit time of txn T with start time of txn S.  This sequence can be thought of
-   * as a logical time counter.  If S.commitTime < T.startTime, T and S do NOT overlap.
-   *
-   * Motivating example:
-   * Suppose we have multi-statment transactions T and S both of which are attempting x = x + 1
-   * In order to prevent lost update problem, the the non-overlapping txns must lock in the snapshot
-   * that they read appropriately.  In particular, if txns do not overlap, then one follows the other
-   * (assumig they write the same entity), and thus the 2nd must see changes of the 1st.  We ensure
-   * this by locking in snapshot after 
-   * {@link #openTxns(OpenTxnRequest)} call is made (see {@link org.apache.hadoop.hive.ql.Driver#acquireLocksAndOpenTxn()})
-   * and mutexing openTxn() with commit().  In other words, once a S.commit() starts we must ensure
-   * that txn T which will be considered a later txn, locks in a snapshot that includes the result
-   * of S's commit (assuming no other txns).
-   * As a counter example, suppose we have S[3,3] and T[4,4] (commitId=txnid means no other transactions
-   * were running in parallel).  If T and S both locked in the same snapshot (for example commit of
-   * txnid:2, which is possible if commitTxn() and openTxnx() is not mutexed)
-   * 'x' would be updated to the same value by both, i.e. lost update. 
-   */
-  @Override
-  @RetrySemantics.Idempotent("No-op if already committed")
-  public void commitTxn(CommitTxnRequest rqst)
-    throws NoSuchTxnException, TxnAbortedException,  MetaException {
-    long txnid = rqst.getTxnid();
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      ResultSet lockHandle = null;
-      ResultSet commitIdRs = null, rs;
-      try {
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        /**
-         * Runs at READ_COMMITTED with S4U on TXNS row for "txnid".  S4U ensures that no other
-         * operation can change this txn (such acquiring locks). While lock() and commitTxn()
-         * should not normally run concurrently (for same txn) but could due to bugs in the client
-         * which could then corrupt internal transaction manager state.  Also competes with abortTxn().
-         */
-        lockHandle = lockTransactionRecord(stmt, txnid, TXN_OPEN);
-        if (lockHandle == null) {
-          //if here, txn was not found (in expected state)
-          TxnStatus actualTxnStatus = findTxnState(txnid, stmt);
-          if(actualTxnStatus == TxnStatus.COMMITTED) {
-            /**
-             * This makes the operation idempotent
-             * (assume that this is most likely due to retry logic)
-             */
-            LOG.info("Nth commitTxn(" + JavaUtils.txnIdToString(txnid) + ") msg");
-            return;
-          }
-          raiseTxnUnexpectedState(actualTxnStatus, txnid);
-          shouldNeverHappen(txnid);
-          //dbConn is rolled back in finally{}
-        }
-        String conflictSQLSuffix = "from TXN_COMPONENTS where tc_txnid=" + txnid + " and tc_operation_type IN(" +
-          quoteChar(OpertaionType.UPDATE.sqlConst) + "," + quoteChar(OpertaionType.DELETE.sqlConst) + ")";
-        rs = stmt.executeQuery(sqlGenerator.addLimitClause(1, "tc_operation_type " + conflictSQLSuffix));
-        if (rs.next()) {
-          close(rs);
-          //if here it means currently committing txn performed update/delete and we should check WW conflict
-          /**
-           * This S4U will mutex with other commitTxn() and openTxns(). 
-           * -1 below makes txn intervals look like [3,3] [4,4] if all txns are serial
-           * Note: it's possible to have several txns have the same commit id.  Suppose 3 txns start
-           * at the same time and no new txns start until all 3 commit.
-           * We could've incremented the sequence for commitId is well but it doesn't add anything functionally.
-           */
-          commitIdRs = stmt.executeQuery(sqlGenerator.addForUpdateClause("select ntxn_next - 1 from NEXT_TXN_ID"));
-          if (!commitIdRs.next()) {
-            throw new IllegalStateException("No rows found in NEXT_TXN_ID");
-          }
-          long commitId = commitIdRs.getLong(1);
-          Savepoint undoWriteSetForCurrentTxn = dbConn.setSavepoint();
-          /**
-           * "select distinct" is used below because
-           * 1. once we get to multi-statement txns, we only care to record that something was updated once
-           * 2. if {@link #addDynamicPartitions(AddDynamicPartitions)} is retried by caller it my create
-           *  duplicate entries in TXN_COMPONENTS
-           * but we want to add a PK on WRITE_SET which won't have unique rows w/o this distinct
-           * even if it includes all of it's columns
-           */
-          int numCompsWritten = stmt.executeUpdate(
-            "insert into WRITE_SET (ws_database, ws_table, ws_partition, ws_txnid, ws_commit_id, ws_operation_type)" +
-            " select distinct tc_database, tc_table, tc_partition, tc_txnid, " + commitId + ", tc_operation_type " + conflictSQLSuffix);
-          /**
-           * see if there are any overlapping txns wrote the same element, i.e. have a conflict
-           * Since entire commit operation is mutexed wrt other start/commit ops,
-           * committed.ws_commit_id <= current.ws_commit_id for all txns
-           * thus if committed.ws_commit_id < current.ws_txnid, transactions do NOT overlap
-           * For example, [17,20] is committed, [6,80] is being committed right now - these overlap
-           * [17,20] committed and [21,21] committing now - these do not overlap.
-           * [17,18] committed and [18,19] committing now - these overlap  (here 18 started while 17 was still running)
-           */
-          rs = stmt.executeQuery
-            (sqlGenerator.addLimitClause(1, "committed.ws_txnid, committed.ws_commit_id, committed.ws_database," +
-              "committed.ws_table, committed.ws_partition, cur.ws_commit_id cur_ws_commit_id, " +
-              "cur.ws_operation_type cur_op, committed.ws_operation_type committed_op " +
-              "from WRITE_SET committed INNER JOIN WRITE_SET cur " +
-              "ON committed.ws_database=cur.ws_database and committed.ws_table=cur.ws_table " +
-              //For partitioned table we always track writes at partition level (never at table)
-              //and for non partitioned - always at table level, thus the same table should never
-              //have entries with partition key and w/o
-              "and (committed.ws_partition=cur.ws_partition or (committed.ws_partition is null and cur.ws_partition is null)) " +
-              "where cur.ws_txnid <= committed.ws_commit_id" + //txns overlap; could replace ws_txnid
-              // with txnid, though any decent DB should infer this
-              " and cur.ws_txnid=" + txnid + //make sure RHS of join only has rows we just inserted as
-              // part of this commitTxn() op
-              " and committed.ws_txnid <> " + txnid + //and LHS only has committed txns
-              //U+U and U+D is a conflict but D+D is not and we don't currently track I in WRITE_SET at all
-              " and (committed.ws_operation_type=" + quoteChar(OpertaionType.UPDATE.sqlConst) +
-              " OR cur.ws_operation_type=" + quoteChar(OpertaionType.UPDATE.sqlConst) + ")"));
-          if (rs.next()) {
-            //found a conflict
-            String committedTxn = "[" + JavaUtils.txnIdToString(rs.getLong(1)) + "," + rs.getLong(2) + "]";
-            StringBuilder resource = new StringBuilder(rs.getString(3)).append("/").append(rs.getString(4));
-            String partitionName = rs.getString(5);
-            if (partitionName != null) {
-              resource.append('/').append(partitionName);
-            }
-            String msg = "Aborting [" + JavaUtils.txnIdToString(txnid) + "," + rs.getLong(6) + "]" + " due to a write conflict on " + resource +
-              " committed by " + committedTxn + " " + rs.getString(7) + "/" + rs.getString(8);
-            close(rs);
-            //remove WRITE_SET info for current txn since it's about to abort
-            dbConn.rollback(undoWriteSetForCurrentTxn);
-            LOG.info(msg);
-            //todo: should make abortTxns() write something into TXNS.TXN_META_INFO about this
-            if (abortTxns(dbConn, Collections.singletonList(txnid), true) != 1) {
-              throw new IllegalStateException(msg + " FAILED!");
-            }
-            dbConn.commit();
-            close(null, stmt, dbConn);
-            throw new TxnAbortedException(msg);
-          } else {
-            //no conflicting operations, proceed with the rest of commit sequence
-          }
-        }
-        else {
-          /**
-           * current txn didn't update/delete anything (may have inserted), so just proceed with commit
-           *
-           * We only care about commit id for write txns, so for RO (when supported) txns we don't
-           * have to mutex on NEXT_TXN_ID.
-           * Consider: if RO txn is after a W txn, then RO's openTxns() will be mutexed with W's
-           * commitTxn() because both do S4U on NEXT_TXN_ID and thus RO will see result of W txn.
-           * If RO < W, then there is no reads-from relationship.
-           */
-        }
-        // Move the record from txn_components into completed_txn_components so that the compactor
-        // knows where to look to compact.
-        String s = "insert into COMPLETED_TXN_COMPONENTS select tc_txnid, tc_database, tc_table, " +
-          "tc_partition from TXN_COMPONENTS where tc_txnid = " + txnid;
-        LOG.debug("Going to execute insert <" + s + ">");
-        int modCount = 0;
-        if ((modCount = stmt.executeUpdate(s)) < 1) {
-          //this can be reasonable for an empty txn START/COMMIT or read-only txn
-          //also an IUD with DP that didn't match any rows.
-          LOG.info("Expected to move at least one record from txn_components to " +
-            "completed_txn_components when committing txn! " + JavaUtils.txnIdToString(txnid));
-        }
-        s = "delete from TXN_COMPONENTS where tc_txnid = " + txnid;
-        LOG.debug("Going to execute update <" + s + ">");
-        modCount = stmt.executeUpdate(s);
-        s = "delete from HIVE_LOCKS where hl_txnid = " + txnid;
-        LOG.debug("Going to execute update <" + s + ">");
-        modCount = stmt.executeUpdate(s);
-        s = "delete from TXNS where txn_id = " + txnid;
-        LOG.debug("Going to execute update <" + s + ">");
-        modCount = stmt.executeUpdate(s);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "commitTxn(" + rqst + ")");
-        throw new MetaException("Unable to update transaction database "
-          + StringUtils.stringifyException(e));
-      } finally {
-        close(commitIdRs);
-        close(lockHandle, stmt, dbConn);
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      commitTxn(rqst);
-    }
-  }
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void performWriteSetGC() {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      stmt = dbConn.createStatement();
-      rs = stmt.executeQuery("select ntxn_next - 1 from NEXT_TXN_ID");
-      if(!rs.next()) {
-        throw new IllegalStateException("NEXT_TXN_ID is empty: DB is corrupted");
-      }
-      long highestAllocatedTxnId = rs.getLong(1);
-      close(rs);
-      rs = stmt.executeQuery("select min(txn_id) from TXNS where txn_state=" + quoteChar(TXN_OPEN));
-      if(!rs.next()) {
-        throw new IllegalStateException("Scalar query returned no rows?!?!!");
-      }
-      long commitHighWaterMark;//all currently open txns (if any) have txnid >= than commitHighWaterMark
-      long lowestOpenTxnId = rs.getLong(1);
-      if(rs.wasNull()) {
-        //if here then there are no Open txns and  highestAllocatedTxnId must be
-        //resolved (i.e. committed or aborted), either way
-        //there are no open txns with id <= highestAllocatedTxnId
-        //the +1 is there because "delete ..." below has < (which is correct for the case when
-        //there is an open txn
-        //Concurrency: even if new txn starts (or starts + commits) it is still true that
-        //there are no currently open txns that overlap with any committed txn with 
-        //commitId <= commitHighWaterMark (as set on next line).  So plain READ_COMMITTED is enough.
-        commitHighWaterMark = highestAllocatedTxnId + 1;
-      }
-      else {
-        commitHighWaterMark = lowestOpenTxnId;
-      }
-      int delCnt = stmt.executeUpdate("delete from WRITE_SET where ws_commit_id < " + commitHighWaterMark);
-      LOG.info("Deleted " + delCnt + " obsolete rows from WRTIE_SET");
-      dbConn.commit();
-    } catch (SQLException ex) {
-      LOG.warn("WriteSet GC failed due to " + getMessage(ex), ex);
-    }
-    finally {
-      close(rs, stmt, dbConn);
-    }
-  }
-  /**
-   * As much as possible (i.e. in absence of retries) we want both operations to be done on the same
-   * connection (but separate transactions).  This avoid some flakiness in BONECP where if you
-   * perform an operation on 1 connection and immediately get another from the pool, the 2nd one
-   * doesn't see results of the first.
-   * 
-   * Retry-by-caller note: If the call to lock is from a transaction, then in the worst case
-   * there will be a duplicate set of locks but both sets will belong to the same txn so they 
-   * will not conflict with each other.  For locks w/o txn context (i.e. read-only query), this
-   * may lead to deadlock (at least a long wait).  (e.g. 1st call creates locks in {@code LOCK_WAITING}
-   * mode and response gets lost.  Then {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient}
-   * retries, and enqueues another set of locks in LOCK_WAITING.  The 2nd LockResponse is delivered
-   * to the DbLockManager, which will keep dong {@link #checkLock(CheckLockRequest)} until the 1st
-   * set of locks times out.
-   */
-  @RetrySemantics.CannotRetry
-  public LockResponse lock(LockRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
-    ConnectionLockIdPair connAndLockId = enqueueLockWithRetry(rqst);
-    try {
-      return checkLockWithRetry(connAndLockId.dbConn, connAndLockId.extLockId, rqst.getTxnid());
-    }
-    catch(NoSuchLockException e) {
-      // This should never happen, as we just added the lock id
-      throw new MetaException("Couldn't find a lock we just created! " + e.getMessage());
-    }
-  }
-  private static final class ConnectionLockIdPair {
-    private final Connection dbConn;
-    private final long extLockId;
-    private ConnectionLockIdPair(Connection dbConn, long extLockId) {
-      this.dbConn = dbConn;
-      this.extLockId = extLockId;
-    }
-  }
-
-  /**
-   * Note that by definition select for update is divorced from update, i.e. you executeQuery() to read
-   * and then executeUpdate().  One other alternative would be to actually update the row in TXNS but
-   * to the same value as before thus forcing db to acquire write lock for duration of the transaction.
-   *
-   * There is no real reason to return the ResultSet here other than to make sure the reference to it
-   * is retained for duration of intended lock scope and is not GC'd thus (unlikely) causing lock
-   * to be released.
-   * @param txnState the state this txn is expected to be in.  may be null
-   * @return null if no row was found
-   * @throws SQLException
-   * @throws MetaException
-   */
-  private ResultSet lockTransactionRecord(Statement stmt, long txnId, Character txnState) throws SQLException, MetaException {
-    String query = "select TXN_STATE from TXNS where TXN_ID = " + txnId + (txnState != null ? " AND TXN_STATE=" + quoteChar(txnState) : "");
-    ResultSet rs = stmt.executeQuery(sqlGenerator.addForUpdateClause(query));
-    if(rs.next()) {
-      return rs;
-    }
-    close(rs);
-    return null;
-  }
-
-  /**
-   * This enters locks into the queue in {@link #LOCK_WAITING} mode.
-   *
-   * Isolation Level Notes:
-   * 1. We use S4U (withe read_committed) to generate the next (ext) lock id.  This serializes
-   * any 2 {@code enqueueLockWithRetry()} calls.
-   * 2. We use S4U on the relevant TXNS row to block any concurrent abort/commit/etc operations
-   * @see #checkLockWithRetry(Connection, long, long)
-   */
-  private ConnectionLockIdPair enqueueLockWithRetry(LockRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
-    boolean success = false;
-    Connection dbConn = null;
-    try {
-      Statement stmt = null;
-      ResultSet rs = null;
-      ResultSet lockHandle = null;
-      try {
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        long txnid = rqst.getTxnid();
-        stmt = dbConn.createStatement();
-        if (isValidTxn(txnid)) {
-          //this also ensures that txn is still there in expected state
-          lockHandle = lockTransactionRecord(stmt, txnid, TXN_OPEN);
-          if(lockHandle == null) {
-            ensureValidTxn(dbConn, txnid, stmt);
-            shouldNeverHappen(txnid);
-          }
-        }
-        /** Get the next lock id.
-         * This has to be atomic with adding entries to HIVE_LOCK entries (1st add in W state) to prevent a race.
-         * Suppose ID gen is a separate txn and 2 concurrent lock() methods are running.  1st one generates nl_next=7,
-         * 2nd nl_next=8.  Then 8 goes first to insert into HIVE_LOCKS and acquires the locks.  Then 7 unblocks,
-         * and add it's W locks but it won't see locks from 8 since to be 'fair' {@link #checkLock(java.sql.Connection, long)}
-         * doesn't block on locks acquired later than one it's checking*/
-        String s = sqlGenerator.addForUpdateClause("select nl_next from NEXT_LOCK_ID");
-        LOG.debug("Going to execute query <" + s + ">");
-        rs = stmt.executeQuery(s);
-        if (!rs.next()) {
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-          throw new MetaException("Transaction tables not properly " +
-            "initialized, no record found in next_lock_id");
-        }
-        long extLockId = rs.getLong(1);
-        s = "update NEXT_LOCK_ID set nl_next = " + (extLockId + 1);
-        LOG.debug("Going to execute update <" + s + ">");
-        stmt.executeUpdate(s);
-
-        if (txnid > 0) {
-          List<String> rows = new ArrayList<>();
-          // For each component in this lock request,
-          // add an entry to the txn_components table
-          for (LockComponent lc : rqst.getComponent()) {
-            if(lc.isSetIsAcid() && !lc.isIsAcid()) {
-              //we don't prevent using non-acid resources in a txn but we do lock them
-              continue;
-            }
-            boolean updateTxnComponents;
-            if(!lc.isSetOperationType()) {
-              //request came from old version of the client
-              updateTxnComponents = true;//this matches old behavior
-            }
-            else {
-              switch (lc.getOperationType()) {
-                case INSERT:
-                case UPDATE:
-                case DELETE:
-                  if(!lc.isSetIsDynamicPartitionWrite()) {
-                    //must be old client talking, i.e. we don't know if it's DP so be conservative
-                    updateTxnComponents = true;
-                  }
-                  else {
-                    /**
-                     * we know this is part of DP operation and so we'll get
-                     * {@link #addDynamicPartitions(AddDynamicPartitions)} call with the list
-                     * of partitions actually chaged.
-                     */
-                    updateTxnComponents = !lc.isIsDynamicPartitionWrite();
-                  }
-                  break;
-                case SELECT:
-                  updateTxnComponents = false;
-                  break;
-                case NO_TXN:
-                  /*this constant is a bit of a misnomer since we now always have a txn context.  It
-                   just means the operation is such that we don't care what tables/partitions it
-                   affected as it doesn't trigger a compaction or conflict detection.  A better name
-                   would be NON_TRANSACTIONAL.*/
-                  updateTxnComponents = false;
-                  break;
-                default:
-                  //since we have an open transaction, only 4 values above are expected 
-                  throw new IllegalStateException("Unexpected DataOperationType: " + lc.getOperationType()
-                    + " agentInfo=" + rqst.getAgentInfo() + " " + JavaUtils.txnIdToString(txnid));
-              }
-            }
-            if(!updateTxnComponents) {
-              continue;
-            }
-            String dbName = lc.getDbname();
-            String tblName = lc.getTablename();
-            String partName = lc.getPartitionname();
-            rows.add(txnid + ", '" + dbName + "', " +
-              (tblName == null ? "null" : "'" + tblName + "'") + ", " +
-              (partName == null ? "null" : "'" + partName + "'")+ "," +
-              quoteString(OpertaionType.fromDataOperationType(lc.getOperationType()).toString()));
-          }
-          List<String> queries = sqlGenerator.createInsertValuesStmt(
-            "TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition, tc_operation_type)", rows);
-          for(String query : queries) {
-            LOG.debug("Going to execute update <" + query + ">");
-            int modCount = stmt.executeUpdate(query);
-          }
-        }
-
-        List<String> rows = new ArrayList<>();
-        long intLockId = 0;
-        for (LockComponent lc : rqst.getComponent()) {
-          if(lc.isSetOperationType() && lc.getOperationType() == DataOperationType.UNSET &&
-            (conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEST) || conf.getBoolVar(HiveConf.ConfVars.HIVE_IN_TEZ_TEST))) {
-            //old version of thrift client should have (lc.isSetOperationType() == false) but they do not
-            //If you add a default value to a variable, isSet() for that variable is true regardless of the where the
-            //message was created (for object variables.  It works correctly for boolean vars, e.g. LockComponent.isAcid).
-            //in test mode, upgrades are not tested, so client version and server version of thrift always matches so
-            //we see UNSET here it means something didn't set the appropriate value.
-            throw new IllegalStateException("Bug: operationType=" + lc.getOperationType() + " for component "
-              + lc + " agentInfo=" + rqst.getAgentInfo());
-          }
-          intLockId++;
-          String dbName = lc.getDbname();
-          String tblName = lc.getTablename();
-          String partName = lc.getPartitionname();
-          LockType lockType = lc.getType();
-          char lockChar = 'z';
-          switch (lockType) {
-            case EXCLUSIVE:
-              lockChar = LOCK_EXCLUSIVE;
-              break;
-            case SHARED_READ:
-              lockChar = LOCK_SHARED;
-              break;
-            case SHARED_WRITE:
-              lockChar = LOCK_SEMI_SHARED;
-              break;
-          }
-          long now = getDbTime(dbConn);
-            rows.add(extLockId + ", " + intLockId + "," + txnid + ", " +
-            quoteString(dbName) + ", " +
-            valueOrNullLiteral(tblName) + ", " +
-            valueOrNullLiteral(partName) + ", " +
-            quoteChar(LOCK_WAITING) + ", " + quoteChar(lockChar) + ", " +
-            //for locks associated with a txn, we always heartbeat txn and timeout based on that
-            (isValidTxn(txnid) ? 0 : now) + ", " +
-            valueOrNullLiteral(rqst.getUser()) + ", " +
-            valueOrNullLiteral(rqst.getHostname()) + ", " +
-            valueOrNullLiteral(rqst.getAgentInfo()));// + ")";
-        }
-        List<String> queries = sqlGenerator.createInsertValuesStmt(
-          "HIVE_LOCKS (hl_lock_ext_id, hl_lock_int_id, hl_txnid, hl_db, " +
-            "hl_table, hl_partition,hl_lock_state, hl_lock_type, " +
-            "hl_last_heartbeat, hl_user, hl_host, hl_agent_info)", rows);
-        for(String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          int modCount = stmt.executeUpdate(query);
-        }
-        dbConn.commit();
-        success = true;
-        return new ConnectionLockIdPair(dbConn, extLockId);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "enqueueLockWithRetry(" + rqst + ")");
-        throw new MetaException("Unable to update transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(lockHandle);
-        close(rs, stmt, null);
-        if (!success) {
-          /* This needs to return a "live" connection to be used by operation that follows it.
-          Thus it only closes Connection on failure/retry. */
-          closeDbConn(dbConn);
-        }
-        unlockInternal();
-      }
-    }
-    catch(RetryException e) {
-      return enqueueLockWithRetry(rqst);
-    }
-  }
-  private LockResponse checkLockWithRetry(Connection dbConn, long extLockId, long txnId)
-    throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException {
-    try {
-      try {
-        lockInternal();
-        if(dbConn.isClosed()) {
-          //should only get here if retrying this op
-          dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        }
-        return checkLock(dbConn, extLockId);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "checkLockWithRetry(" + extLockId + "," + txnId + ")");
-        throw new MetaException("Unable to update transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        unlockInternal();
-        closeDbConn(dbConn);
-      }
-    }
-    catch(RetryException e) {
-      return checkLockWithRetry(dbConn, extLockId, txnId);
-    }
-  }
-  /**
-   * Why doesn't this get a txnid as parameter?  The caller should either know the txnid or know there isn't one.
-   * Either way getTxnIdFromLockId() will not be needed.  This would be a Thrift change.
-   *
-   * Also, when lock acquisition returns WAITING, it's retried every 15 seconds (best case, see DbLockManager.backoff(),
-   * in practice more often)
-   * which means this is heartbeating way more often than hive.txn.timeout and creating extra load on DB.
-   *
-   * The clients that operate in blocking mode, can't heartbeat a lock until the lock is acquired.
-   * We should make CheckLockRequest include timestamp or last request to skip unnecessary heartbeats. Thrift change.
-   *
-   * {@link #checkLock(java.sql.Connection, long)}  must run at SERIALIZABLE (make sure some lock we are checking
-   * against doesn't move from W to A in another txn) but this method can heartbeat in
-   * separate txn at READ_COMMITTED.
-   * 
-   * Retry-by-caller note:
-   * Retryable because {@link #checkLock(Connection, long)} is
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public LockResponse checkLock(CheckLockRequest rqst)
-    throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException {
-    try {
-      Connection dbConn = null;
-      long extLockId = rqst.getLockid();
-      try {
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        // Heartbeat on the lockid first, to assure that our lock is still valid.
-        // Then look up the lock info (hopefully in the cache).  If these locks
-        // are associated with a transaction then heartbeat on that as well.
-        LockInfo info = getTxnIdFromLockId(dbConn, extLockId);
-        if(info == null) {
-          throw new NoSuchLockException("No such lock " + JavaUtils.lockIdToString(extLockId));
-        }
-        if (info.txnId > 0) {
-          heartbeatTxn(dbConn, info.txnId);
-        }
-        else {
-          heartbeatLock(dbConn, extLockId);
-        }
-        //todo: strictly speaking there is a bug here.  heartbeat*() commits but both heartbeat and
-        //checkLock() are in the same retry block, so if checkLock() throws, heartbeat is also retired
-        //extra heartbeat is logically harmless, but ...
-        return checkLock(dbConn, extLockId);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "checkLock(" + rqst + " )");
-        throw new MetaException("Unable to update transaction database " +
-          JavaUtils.lockIdToString(extLockId) + " " + StringUtils.stringifyException(e));
-      } finally {
-        closeDbConn(dbConn);
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      return checkLock(rqst);
-    }
-
-  }
-
-  /**
-   * This would have been made simpler if all locks were associated with a txn.  Then only txn needs to
-   * be heartbeated, committed, etc.  no need for client to track individual locks.
-   * When removing locks not associated with txn this potentially conflicts with
-   * heartbeat/performTimeout which are update/delete of HIVE_LOCKS thus will be locked as needed by db.
-   * since this only removes from HIVE_LOCKS at worst some lock acquire is delayed
-   */
-  @RetrySemantics.Idempotent
-  public void unlock(UnlockRequest rqst)
-    throws NoSuchLockException, TxnOpenException, MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      long extLockId = rqst.getLockid();
-      try {
-        /**
-         * This method is logically like commit for read-only auto commit queries.
-         * READ_COMMITTED since this only has 1 delete statement and no new entries with the
-         * same hl_lock_ext_id can be added, i.e. all rows with a given hl_lock_ext_id are
-         * created in a single atomic operation.
-         * Theoretically, this competes with {@link #lock(org.apache.hadoop.hive.metastore.api.LockRequest)}
-         * but hl_lock_ext_id is not known until that method returns.
-         * Also competes with {@link #checkLock(org.apache.hadoop.hive.metastore.api.CheckLockRequest)}
-         * but using SERIALIZABLE doesn't materially change the interaction.
-         * If "delete" stmt misses, additional logic is best effort to produce meaningful error msg.
-         */
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        //hl_txnid <> 0 means it's associated with a transaction
-        String s = "delete from HIVE_LOCKS where hl_lock_ext_id = " + extLockId + " AND (hl_txnid = 0 OR" +
-          " (hl_txnid <> 0 AND hl_lock_state = '" + LOCK_WAITING + "'))";
-        //(hl_txnid <> 0 AND hl_lock_state = '" + LOCK_WAITING + "') is for multi-statement txns where
-        //some query attempted to lock (thus LOCK_WAITING state) but is giving up due to timeout for example
-        LOG.debug("Going to execute update <" + s + ">");
-        int rc = stmt.executeUpdate(s);
-        if (rc < 1) {
-          LOG.debug("Going to rollback");
-          dbConn.rollback();
-          LockInfo info = getTxnIdFromLockId(dbConn, extLockId);
-          if(info == null) {
-            //didn't find any lock with extLockId but at ReadCommitted there is a possibility that
-            //it existed when above delete ran but it didn't have the expected state.
-            LOG.info("No lock in " + LOCK_WAITING + " mode found for unlock(" +
-              JavaUtils.lockIdToString(rqst.getLockid()) + ")");
-            //bail here to make the operation idempotent
-            return;
-          }
-          if(info.txnId != 0) {
-            String msg = "Unlocking locks associated with transaction not permitted.  " + info;
-            //if a lock is associated with a txn we can only "unlock" if if it's in WAITING state
-            // which really means that the caller wants to give up waiting for the lock
-            LOG.error(msg);
-            throw new TxnOpenException(msg);
-          }
-          if(info.txnId == 0) {
-            //we didn't see this lock when running DELETE stmt above but now it showed up
-            //so should "should never happen" happened...
-            String msg = "Found lock in unexpected state " + info;
-            LOG.error(msg);
-            throw new MetaException(msg);
-          }
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "unlock(" + rqst + ")");
-        throw new MetaException("Unable to update transaction database " +
-          JavaUtils.lockIdToString(extLockId) + " " + StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      unlock(rqst);
-    }
-  }
-
-  /**
-   * used to sort entries in {@link org.apache.hadoop.hive.metastore.api.ShowLocksResponse}
-   */
-  private static class LockInfoExt extends LockInfo {
-    private final ShowLocksResponseElement e;
-    LockInfoExt(ShowLocksResponseElement e) {
-      super(e);
-      this.e = e;
-    }
-  }
-  @RetrySemantics.ReadOnly
-  public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException {
-    try {
-      Connection dbConn = null;
-      ShowLocksResponse rsp = new ShowLocksResponse();
-      List<ShowLocksResponseElement> elems = new ArrayList<ShowLocksResponseElement>();
-      List<LockInfoExt> sortedList = new ArrayList<LockInfoExt>();
-      Statement stmt = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-
-        String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " +
-          "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id," +
-          "hl_blockedby_ext_id, hl_blockedby_int_id, hl_agent_info from HIVE_LOCKS";
-
-        // Some filters may have been specified in the SHOW LOCKS statement. Add them to the query.
-        String dbName = rqst.getDbname();
-        String tableName = rqst.getTablename();
-        String partName = rqst.getPartname();
-
-        StringBuilder filter = new StringBuilder();
-        if (dbName != null && !dbName.isEmpty()) {
-          filter.append("hl_db=").append(quoteString(dbName));
-        }
-        if (tableName != null && !tableName.isEmpty()) {
-          if (filter.length() > 0) {
-            filter.append(" and ");
-          }
-          filter.append("hl_table=").append(quoteString(tableName));
-        }
-        if (partName != null && !partName.isEmpty()) {
-          if (filter.length() > 0) {
-            filter.append(" and ");
-          }
-          filter.append("hl_partition=").append(quoteString(partName));
-        }
-        String whereClause = filter.toString();
-
-        if (!whereClause.isEmpty()) {
-          s = s + " where " + whereClause;
-        }
-
-        LOG.debug("Doing to execute query <" + s + ">");
-        ResultSet rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          ShowLocksResponseElement e = new ShowLocksResponseElement();
-          e.setLockid(rs.getLong(1));
-          long txnid = rs.getLong(2);
-          if (!rs.wasNull()) e.setTxnid(txnid);
-          e.setDbname(rs.getString(3));
-          e.setTablename(rs.getString(4));
-          String partition = rs.getString(5);
-          if (partition != null) e.setPartname(partition);
-          switch (rs.getString(6).charAt(0)) {
-            case LOCK_ACQUIRED: e.setState(LockState.ACQUIRED); break;
-            case LOCK_WAITING: e.setState(LockState.WAITING); break;
-            default: throw new MetaException("Unknown lock state " + rs.getString(6).charAt(0));
-          }
-          switch (rs.getString(7).charAt(0)) {
-            case LOCK_SEMI_SHARED: e.setType(LockType.SHARED_WRITE); break;
-            case LOCK_EXCLUSIVE: e.setType(LockType.EXCLUSIVE); break;
-            case LOCK_SHARED: e.setType(LockType.SHARED_READ); break;
-            default: throw new MetaException("Unknown lock type " + rs.getString(6).charAt(0));
-          }
-          e.setLastheartbeat(rs.getLong(8));
-          long acquiredAt = rs.getLong(9);
-          if (!rs.wasNull()) e.setAcquiredat(acquiredAt);
-          e.setUser(rs.getString(10));
-          e.setHostname(rs.getString(11));
-          e.setLockIdInternal(rs.getLong(12));
-          long id = rs.getLong(13);
-          if(!rs.wasNull()) {
-            e.setBlockedByExtId(id);
-          }
-          id = rs.getLong(14);
-          if(!rs.wasNull()) {
-            e.setBlockedByIntId(id);
-          }
-          e.setAgentInfo(rs.getString(15));
-          sortedList.add(new LockInfoExt(e));
-        }
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } catch (SQLException e) {
-        checkRetryable(dbConn, e, "showLocks(" + rqst + ")");
-        throw new MetaException("Unable to select from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-      //this ensures that "SHOW LOCKS" prints the locks in the same order as they are examined
-      //by checkLock() - makes diagnostics easier.
-      Collections.sort(sortedList, new LockInfoComparator());
-      for(LockInfoExt lockInfoExt : sortedList) {
-        elems.add(lockInfoExt.e);
-      }
-      rsp.setLocks(elems);
-      return rsp;
-    } catch (RetryException e) {
-      return showLocks(rqst);
-    }
-  }
-
-  /**
-   * {@code ids} should only have txnid or lockid but not both, ideally.
-   * Currently DBTxnManager.heartbeat() enforces this.
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void heartbeat(HeartbeatRequest ids)
-    throws NoSuchTxnException,  NoSuchLockException, TxnAbortedException, MetaException {
-    try {
-      Connection dbConn = null;
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        heartbeatLock(dbConn, ids.getLockid());
-        heartbeatTxn(dbConn, ids.getTxnid());
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "heartbeat(" + ids + ")");
-        throw new MetaException("Unable to select from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      heartbeat(ids);
-    }
-  }
-  @Override
-  @RetrySemantics.SafeToRetry
-  public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst)
-    throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      HeartbeatTxnRangeResponse rsp = new HeartbeatTxnRangeResponse();
-      Set<Long> nosuch = new HashSet<Long>();
-      Set<Long> aborted = new HashSet<Long>();
-      rsp.setNosuch(nosuch);
-      rsp.setAborted(aborted);
-      try {
-        /**
-         * READ_COMMITTED is sufficient since {@link #heartbeatTxn(java.sql.Connection, long)}
-         * only has 1 update statement in it and
-         * we only update existing txns, i.e. nothing can add additional txns that this operation
-         * would care about (which would have required SERIALIZABLE)
-         */
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        /*do fast path first (in 1 statement) if doesn't work, rollback and do the long version*/
-        stmt = dbConn.createStatement();
-        List<String> queries = new ArrayList<>();
-        int numTxnsToHeartbeat = (int) (rqst.getMax() - rqst.getMin() + 1);
-        List<Long> txnIds = new ArrayList<>(numTxnsToHeartbeat);
-        for (long txn = rqst.getMin(); txn <= rqst.getMax(); txn++) {
-          txnIds.add(txn);
-        }
-        TxnUtils.buildQueryWithINClause(conf, queries,
-          new StringBuilder("update TXNS set txn_last_heartbeat = " + getDbTime(dbConn) +
-            " where txn_state = " + quoteChar(TXN_OPEN) + " and "),
-          new StringBuilder(""), txnIds, "txn_id", true, false);
-        int updateCnt = 0;
-        for (String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          updateCnt += stmt.executeUpdate(query);
-        }
-        if (updateCnt == numTxnsToHeartbeat) {
-          //fast pass worked, i.e. all txns we were asked to heartbeat were Open as expected
-          dbConn.commit();
-          return rsp;
-        }
-        //if here, do the slow path so that we can return info txns which were not in expected state
-        dbConn.rollback();
-        for (long txn = rqst.getMin(); txn <= rqst.getMax(); txn++) {
-          try {
-            heartbeatTxn(dbConn, txn);
-          } catch (NoSuchTxnException e) {
-            nosuch.add(txn);
-          } catch (TxnAbortedException e) {
-            aborted.add(txn);
-          }
-        }
-        return rsp;
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "heartbeatTxnRange(" + rqst + ")");
-        throw new MetaException("Unable to select from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(null, stmt, dbConn);
-      }
-    } catch (RetryException e) {
-      return heartbeatTxnRange(rqst);
-    }
-  }
-
-  long generateCompactionQueueId(Statement stmt) throws SQLException, MetaException {
-    // Get the id for the next entry in the queue
-    String s = sqlGenerator.addForUpdateClause("select ncq_next from NEXT_COMPACTION_QUEUE_ID");
-    LOG.debug("going to execute query <" + s + ">");
-    try (ResultSet rs = stmt.executeQuery(s)) {
-      if (!rs.next()) {
-        throw new IllegalStateException("Transaction tables not properly initiated, "
-            + "no record found in next_compaction_queue_id");
-      }
-      long id = rs.getLong(1);
-      s = "update NEXT_COMPACTION_QUEUE_ID set ncq_next = " + (id + 1);
-      LOG.debug("Going to execute update <" + s + ">");
-      stmt.executeUpdate(s);
-      return id;
-    }
-  }
-  @Override
-  @RetrySemantics.Idempotent
-  public CompactionResponse compact(CompactionRequest rqst) throws MetaException {
-    // Put a compaction request in the queue.
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-      TxnStore.MutexAPI.LockHandle handle = null;
-      try {
-        lockInternal();
-        /**
-         * MUTEX_KEY.CompactionScheduler lock ensures that there is only 1 entry in
-         * Initiated/Working state for any resource.  This ensures that we don't run concurrent
-         * compactions for any resource.
-         */
-        handle = getMutexAPI().acquireLock(MUTEX_KEY.CompactionScheduler.name());
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-
-        long id = generateCompactionQueueId(stmt);
-
-        StringBuilder sb = new StringBuilder("select cq_id, cq_state from COMPACTION_QUEUE where").
-          append(" cq_state IN(").append(quoteChar(INITIATED_STATE)).
-            append(",").append(quoteChar(WORKING_STATE)).
-          append(") AND cq_database=").append(quoteString(rqst.getDbname())).
-          append(" AND cq_table=").append(quoteString(rqst.getTablename())).append(" AND ");
-        if(rqst.getPartitionname() == null) {
-          sb.append("cq_partition is null");
-        }
-        else {
-          sb.append("cq_partition=").append(quoteString(rqst.getPartitionname()));
-        }
-
-        LOG.debug("Going to execute query <" + sb.toString() + ">");
-        ResultSet rs = stmt.executeQuery(sb.toString());
-        if(rs.next()) {
-          long enqueuedId = rs.getLong(1);
-          String state = compactorStateToResponse(rs.getString(2).charAt(0));
-          LOG.info("Ignoring request to compact " + rqst.getDbname() + "/" + rqst.getTablename() +
-            "/" + rqst.getPartitionname() + " since it is already " + quoteString(state) +
-            " with id=" + enqueuedId);
-          return new CompactionResponse(enqueuedId, state, false);
-        }
-        close(rs);
-        StringBuilder buf = new StringBuilder("insert into COMPACTION_QUEUE (cq_id, cq_database, " +
-          "cq_table, ");
-        String partName = rqst.getPartitionname();
-        if (partName != null) buf.append("cq_partition, ");
-        buf.append("cq_state, cq_type");
-        if (rqst.getProperties() != null) {
-          buf.append(", cq_tblproperties");
-        }
-        if (rqst.getRunas() != null) buf.append(", cq_run_as");
-        buf.append(") values (");
-        buf.append(id);
-        buf.append(", '");
-        buf.append(rqst.getDbname());
-        buf.append("', '");
-        buf.append(rqst.getTablename());
-        buf.append("', '");
-        if (partName != null) {
-          buf.append(partName);
-          buf.append("', '");
-        }
-        buf.append(INITIATED_STATE);
-        buf.append("', '");
-        switch (rqst.getType()) {
-          case MAJOR:
-            buf.append(MAJOR_TYPE);
-            break;
-
-          case MINOR:
-            buf.append(MINOR_TYPE);
-            break;
-
-          default:
-            LOG.debug("Going to rollback");
-            dbConn.rollback();
-            throw new MetaException("Unexpected compaction type " + rqst.getType().toString());
-        }
-        if (rqst.getProperties() != null) {
-          buf.append("', '");
-          buf.append(new StringableMap(rqst.getProperties()).toString());
-        }
-        if (rqst.getRunas() != null) {
-          buf.append("', '");
-          buf.append(rqst.getRunas());
-        }
-        buf.append("')");
-        String s = buf.toString();
-        LOG.debug("Going to execute update <" + s + ">");
-        stmt.executeUpdate(s);
-        LOG.debug("Going to commit");
-        dbConn.commit();
-        return new CompactionResponse(id, INITIATED_RESPONSE, true);
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "compact(" + rqst + ")");
-        throw new MetaException("Unable to select from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-        if(handle != null) {
-          handle.releaseLocks();
-        }
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      return compact(rqst);
-    }
-  }
-
-  private static String compactorStateToResponse(char s) {
-    switch (s) {
-      case INITIATED_STATE: return INITIATED_RESPONSE;
-      case WORKING_STATE: return WORKING_RESPONSE;
-      case READY_FOR_CLEANING: return CLEANING_RESPONSE;
-      case FAILED_STATE: return FAILED_RESPONSE;
-      case SUCCEEDED_STATE: return SUCCEEDED_RESPONSE;
-      case ATTEMPTED_STATE: return ATTEMPTED_RESPONSE;
-      default:
-        return Character.toString(s);
-    }
-  }
-  @RetrySemantics.ReadOnly
-  public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException {
-    ShowCompactResponse response = new ShowCompactResponse(new ArrayList<ShowCompactResponseElement>());
-    Connection dbConn = null;
-    Statement stmt = null;
-    try {
-      try {
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        String s = "select cq_database, cq_table, cq_partition, cq_state, cq_type, cq_worker_id, " +
-          //-1 because 'null' literal doesn't work for all DBs...
-          "cq_start, -1 cc_end, cq_run_as, cq_hadoop_job_id, cq_id from COMPACTION_QUEUE union all " +
-          "select cc_database, cc_table, cc_partition, cc_state, cc_type, cc_worker_id, " +
-          "cc_start, cc_end, cc_run_as, cc_hadoop_job_id, cc_id from COMPLETED_COMPACTIONS";
-        //what I want is order by cc_end desc, cc_start asc (but derby has a bug https://issues.apache.org/jira/browse/DERBY-6013)
-        //to sort so that currently running jobs are at the end of the list (bottom of screen)
-        //and currently running ones are in sorted by start time
-        //w/o order by likely currently running compactions will be first (LHS of Union)
-        LOG.debug("Going to execute query <" + s + ">");
-        ResultSet rs = stmt.executeQuery(s);
-        while (rs.next()) {
-          ShowCompactResponseElement e = new ShowCompactResponseElement();
-          e.setDbname(rs.getString(1));
-          e.setTablename(rs.getString(2));
-          e.setPartitionname(rs.getString(3));
-          e.setState(compactorStateToResponse(rs.getString(4).charAt(0)));
-          switch (rs.getString(5).charAt(0)) {
-            case MAJOR_TYPE: e.setType(CompactionType.MAJOR); break;
-            case MINOR_TYPE: e.setType(CompactionType.MINOR); break;
-            default:
-              //do nothing to handle RU/D if we add another status
-          }
-          e.setWorkerid(rs.getString(6));
-          long start = rs.getLong(7);
-          if(!rs.wasNull()) {
-            e.setStart(start);
-          }
-          long endTime = rs.getLong(8);
-          if(endTime != -1) {
-            e.setEndTime(endTime);
-          }
-          e.setRunAs(rs.getString(9));
-          e.setHadoopJobId(rs.getString(10));
-          e.setId(rs.getLong(11));
-          response.addToCompacts(e);
-        }
-        LOG.debug("Going to rollback");
-        dbConn.rollback();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "showCompact(" + rqst + ")");
-        throw new MetaException("Unable to select from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-      return response;
-    } catch (RetryException e) {
-      return showCompact(rqst);
-    }
-  }
-
-  private static void shouldNeverHappen(long txnid) {
-    throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid));
-  }
-  private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) {
-    throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid) + " "
-      + JavaUtils.lockIdToString(extLockId) + " " + intLockId);
-  }
-
-  /**
-   * Retry-by-caller note:
-   * This may be retried after dbConn.commit.  At worst, it will create duplicate entries in
-   * TXN_COMPONENTS which won't affect anything.  See more comments in {@link #commitTxn(CommitTxnRequest)}
-   */
-  @Override
-  @RetrySemantics.SafeToRetry
-  public void addDynamicPartitions(AddDynamicPartitions rqst)
-      throws NoSuchTxnException,  TxnAbortedException, MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet lockHandle = null;
-    ResultSet rs = null;
-    try {
-      try {
-        lockInternal();
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        lockHandle = lockTransactionRecord(stmt, rqst.getTxnid(), TXN_OPEN);
-        if(lockHandle == null) {
-          //ensures txn is still there and in expected state
-          ensureValidTxn(dbConn, rqst.getTxnid(), stmt);
-          shouldNeverHappen(rqst.getTxnid());
-        }
-        //for RU this may be null so we should default it to 'u' which is most restrictive
-        OpertaionType ot = OpertaionType.UPDATE;
-        if(rqst.isSetOperationType()) {
-          ot = OpertaionType.fromDataOperationType(rqst.getOperationType());
-        }
-        List<String> rows = new ArrayList<>();
-        for (String partName : rqst.getPartitionnames()) {
-          rows.add(rqst.getTxnid() + "," + quoteString(rqst.getDbname()) + "," + quoteString(rqst.getTablename()) +
-            "," + quoteString(partName) + "," + quoteChar(ot.sqlConst));
-        }
-        int modCount = 0;
-        //record partitions that were written to
-        List<String> queries = sqlGenerator.createInsertValuesStmt(
-          "TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition, tc_operation_type)", rows);
-        for(String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          modCount = stmt.executeUpdate(query);
-        }
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "addDynamicPartitions(" + rqst + ")");
-        throw new MetaException("Unable to insert into from transaction database " +
-          StringUtils.stringifyException(e));
-      } finally {
-        close(lockHandle, stmt, dbConn);
-        unlockInternal();
-      }
-    } catch (RetryException e) {
-      addDynamicPartitions(rqst);
-    }
-  }
-
-  /**
-   * Clean up corresponding records in metastore tables when corresponding object is dropped,
-   * specifically: TXN_COMPONENTS, COMPLETED_TXN_COMPONENTS, COMPACTION_QUEUE, COMPLETED_COMPACTIONS
-   * Retry-by-caller note: this is only idempotent assuming it's only called by dropTable/Db/etc
-   * operations.
-   */
-  @Override
-  @RetrySemantics.Idempotent
-  public void cleanupRecords(HiveObjectType type, Database db, Table table,
-                             Iterator<Partition> partitionIterator) throws MetaException {
-    try {
-      Connection dbConn = null;
-      Statement stmt = null;
-
-      try {
-        String dbName;
-        String tblName;
-        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-        stmt = dbConn.createStatement();
-        List<String> queries = new ArrayList<String>();
-        StringBuilder buff = new StringBuilder();
-
-        switch (type) {
-          case DATABASE:
-            dbName = db.getName();
-
-            buff.append("delete from TXN_COMPONENTS where tc_database='");
-            buff.append(dbName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
-            buff.append(dbName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPACTION_QUEUE where cq_database='");
-            buff.append(dbName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
-            buff.append(dbName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            break;
-          case TABLE:
-            dbName = table.getDbName();
-            tblName = table.getTableName();
-
-            buff.append("delete from TXN_COMPONENTS where tc_database='");
-            buff.append(dbName);
-            buff.append("' and tc_table='");
-            buff.append(tblName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
-            buff.append(dbName);
-            buff.append("' and ctc_table='");
-            buff.append(tblName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPACTION_QUEUE where cq_database='");
-            buff.append(dbName);
-            buff.append("' and cq_table='");
-            buff.append(tblName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            buff.setLength(0);
-            buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
-            buff.append(dbName);
-            buff.append("' and cc_table='");
-            buff.append(tblName);
-            buff.append("'");
-            queries.add(buff.toString());
-
-            break;
-          case PARTITION:
-            dbName = table.getDbName();
-            tblName = table.getTableName();
-            List<FieldSchema> partCols = table.getPartitionKeys();  // partition columns
-            List<String> partVals;                                  // partition values
-            String partName;
-
-            while (partitionIterator.hasNext()) {
-              Partition p = partitionIterator.next();
-              partVals = p.getValues();
-              partName = Warehouse.makePartName(partCols, partVals);
-
-              buff.append("delete from TXN_COMPONENTS where tc_database='");
-              buff.append(dbName);
-              buff.append("' and tc_table='");
-              buff.append(tblName);
-              buff.append("' and tc_partition='");
-              buff.append(partName);
-              buff.append("'");
-              queries.add(buff.toString());
-
-              buff.setLength(0);
-              buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
-              buff.append(dbName);
-              buff.append("' and ctc_table='");
-              buff.append(tblName);
-              buff.append("' and ctc_partition='");
-              buff.append(partName);
-              buff.append("'");
-              queries.add(buff.toString());
-
-              buff.setLength(0);
-              buff.append("delete from COMPACTION_QUEUE where cq_database='");
-              buff.append(dbName);
-              buff.append("' and cq_table='");
-              buff.append(tblName);
-              buff.append("' and cq_partition='");
-              buff.append(partName);
-              buff.append("'");
-              queries.add(buff.toString());
-
-              buff.setLength(0);
-              buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
-              buff.append(dbName);
-              buff.append("' and cc_table='");
-              buff.append(tblName);
-              buff.append("' and cc_partition='");
-              buff.append(partName);
-              buff.append("'");
-              queries.add(buff.toString());
-            }
-
-            break;
-          default:
-            throw new MetaException("Invalid object type for cleanup: " + type);
-        }
-
-        for (String query : queries) {
-          LOG.debug("Going to execute update <" + query + ">");
-          stmt.executeUpdate(query);
-        }
-
-        LOG.debug("Going to commit");
-        dbConn.commit();
-      } catch (SQLException e) {
-        LOG.debug("Going to rollback");
-        rollbackDBConn(dbConn);
-        checkRetryable(dbConn, e, "cleanupRecords");
-        if (e.getMessage().contains("does not exist")) {
-          LOG.warn("Cannot perform cleanup since metastore table does not exist");
-        } else {
-          throw new MetaException("Unable to clean up " + StringUtils.stringifyException(e));
-        }
-      } finally {
-        closeStmt(stmt);
-        closeDbConn(dbConn);
-      }
-    } catch (RetryException e) {
-      cleanupRecords(type, db, table, partitionIterator);
-    }
-  }
-
-  /**
-   * For testing only, do not use.
-   */
-  @VisibleForTesting
-  public int numLocksInLockTable() throws SQLException, MetaException {
-    Connection dbConn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-    try {
-      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
-      stmt = dbConn.createStatement();
-      String s = "select count(*) from HIVE_LOCKS";
-      LOG.debug("Going to execute query <" + s + ">");
-      rs = stmt.executeQuery(s);
-      rs.next();
-      int rc = rs.getInt(1);
-      // Necessary to clean up the transaction in the db.
-      dbConn.rollback

<TRUNCATED>

[04/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
new file mode 100644
index 0000000..7087b03
--- /dev/null
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -0,0 +1,3643 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.commons.dbcp.ConnectionFactory;
+import org.apache.commons.dbcp.DriverManagerConnectionFactory;
+import org.apache.commons.dbcp.PoolableConnectionFactory;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.RetrySemantics;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.RunnableConfigurable;
+import org.apache.hadoop.hive.metastore.ThreadPool;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.datasource.BoneCPDataSourceProvider;
+import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider;
+import org.apache.hadoop.hive.metastore.datasource.HikariCPDataSourceProvider;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.hadoop.hive.metastore.metrics.Metrics;
+import org.apache.hadoop.hive.metastore.metrics.MetricsConstants;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.utils.JavaUtils;
+import org.apache.hadoop.hive.metastore.utils.StringableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.dbcp.PoolingDataSource;
+
+import org.apache.commons.pool.impl.GenericObjectPool;
+import org.apache.hadoop.hive.metastore.api.*;
+import org.apache.hadoop.util.StringUtils;
+
+import javax.sql.DataSource;
+
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.sql.*;
+import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+
+/**
+ * A handler to answer transaction related calls that come into the metastore
+ * server.
+ *
+ * Note on log messages:  Please include txnid:X and lockid info using
+ * {@link JavaUtils#txnIdToString(long)}
+ * and {@link JavaUtils#lockIdToString(long)} in all messages.
+ * The txnid:X and lockid:Y matches how Thrift object toString() methods are generated,
+ * so keeping the format consistent makes grep'ing the logs much easier.
+ *
+ * Note on HIVE_LOCKS.hl_last_heartbeat.
+ * For locks that are part of transaction, we set this 0 (would rather set it to NULL but
+ * Currently the DB schema has this NOT NULL) and only update/read heartbeat from corresponding
+ * transaction in TXNS.
+ *
+ * In general there can be multiple metastores where this logic can execute, thus the DB is
+ * used to ensure proper mutexing of operations.
+ * Select ... For Update (or equivalent: either MsSql with(updlock) or actual Update stmt) is
+ * used to properly sequence operations.  Most notably:
+ * 1. various sequence IDs are generated with aid of this mutex
+ * 2. ensuring that each (Hive) Transaction state is transitioned atomically.  Transaction state
+ *  includes its actual state (Open, Aborted) as well as it's lock list/component list.  Thus all
+ *  per transaction ops, either start by update/delete of the relevant TXNS row or do S4U on that row.
+ *  This allows almost all operations to run at READ_COMMITTED and minimizes DB deadlocks.
+ * 3. checkLock() - this is mutexted entirely since we must ensure that while we check if some lock
+ *  can be granted, no other (strictly speaking "earlier") lock can change state.
+ *
+ * The exception to his is Derby which doesn't support proper S4U.  Derby is always running embedded
+ * (this is the only supported configuration for Derby)
+ * in the same JVM as HiveMetaStoreHandler thus we use JVM wide lock to properly sequnce the operations.
+ *
+ * {@link #derbyLock}
+
+ * If we ever decide to run remote Derby server, according to
+ * https://db.apache.org/derby/docs/10.0/manuals/develop/develop78.html all transactions will be
+ * seriazlied, so that would also work though has not been tested.
+ *
+ * General design note:
+ * It's imperative that any operation on a txn (e.g. commit), ensure (atomically) that this txn is
+ * still valid and active.  In the code this is usually achieved at the same time the txn record
+ * is locked for some operation.
+ * 
+ * Note on retry logic:
+ * Metastore has retry logic in both {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient}
+ * and {@link org.apache.hadoop.hive.metastore.RetryingHMSHandler}.  The retry logic there is very
+ * generic and is not aware whether the operations are idempotent or not.  (This is separate from
+ * retry logic here in TxnHander which can/does retry DB errors intelligently).  The worst case is
+ * when an op here issues a successful commit against the RDBMS but the calling stack doesn't
+ * receive the ack and retries.  (If an op fails before commit, it's trivially idempotent)
+ * Thus the ops here need to be made idempotent as much as possible or
+ * the metstore call stack should have logic not to retry.  There are {@link RetrySemantics}
+ * annotations to document the behavior.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
+
+  static final protected char INITIATED_STATE = 'i';
+  static final protected char WORKING_STATE = 'w';
+  static final protected char READY_FOR_CLEANING = 'r';
+  static final char FAILED_STATE = 'f';
+  static final char SUCCEEDED_STATE = 's';
+  static final char ATTEMPTED_STATE = 'a';
+
+  // Compactor types
+  static final protected char MAJOR_TYPE = 'a';
+  static final protected char MINOR_TYPE = 'i';
+
+  // Transaction states
+  static final protected char TXN_ABORTED = 'a';
+  static final protected char TXN_OPEN = 'o';
+  //todo: make these like OperationType and remove above char constatns
+  enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN}
+
+  // Lock states
+  static final protected char LOCK_ACQUIRED = 'a';
+  static final protected char LOCK_WAITING = 'w';
+
+  // Lock types
+  static final protected char LOCK_EXCLUSIVE = 'e';
+  static final protected char LOCK_SHARED = 'r';
+  static final protected char LOCK_SEMI_SHARED = 'w';
+
+  static final private int ALLOWED_REPEATED_DEADLOCKS = 10;
+  static final private Logger LOG = LoggerFactory.getLogger(TxnHandler.class.getName());
+
+  static private DataSource connPool;
+  private static DataSource connPoolMutex;
+  static private boolean doRetryOnConnPool = false;
+  
+  private enum OpertaionType {
+    SELECT('s'), INSERT('i'), UPDATE('u'), DELETE('d');
+    private final char sqlConst;
+    OpertaionType(char sqlConst) {
+      this.sqlConst = sqlConst;
+    }
+    public String toString() {
+      return Character.toString(sqlConst);
+    }
+    public static OpertaionType fromString(char sqlConst) {
+      switch (sqlConst) {
+        case 's':
+          return SELECT;
+        case 'i':
+          return INSERT;
+        case 'u':
+          return UPDATE;
+        case 'd':
+          return DELETE;
+        default:
+          throw new IllegalArgumentException(quoteChar(sqlConst));
+      }
+    }
+    public static OpertaionType fromDataOperationType(DataOperationType dop) {
+      switch (dop) {
+        case SELECT:
+          return OpertaionType.SELECT;
+        case INSERT:
+          return OpertaionType.INSERT;
+        case UPDATE:
+          return OpertaionType.UPDATE;
+        case DELETE:
+          return OpertaionType.DELETE;
+        default:
+          throw new IllegalArgumentException("Unexpected value: " + dop);
+      }
+    }
+  }
+
+  // Maximum number of open transactions that's allowed
+  private static volatile int maxOpenTxns = 0;
+  // Whether number of open transactions reaches the threshold
+  private static volatile boolean tooManyOpenTxns = false;
+
+  /**
+   * Number of consecutive deadlocks we have seen
+   */
+  private int deadlockCnt;
+  private long deadlockRetryInterval;
+  protected Configuration conf;
+  private static DatabaseProduct dbProduct;
+  private static SQLGenerator sqlGenerator;
+
+  // (End user) Transaction timeout, in milliseconds.
+  private long timeout;
+
+  private String identifierQuoteString; // quotes to use for quoting tables, where necessary
+  private long retryInterval;
+  private int retryLimit;
+  private int retryNum;
+  // Current number of open txns
+  private AtomicInteger numOpenTxns;
+
+  /**
+   * Derby specific concurrency control
+   */
+  private static final ReentrantLock derbyLock = new ReentrantLock(true);
+  /**
+   * must be static since even in UT there may be > 1 instance of TxnHandler
+   * (e.g. via Compactor services)
+   */
+  private final static ConcurrentHashMap<String, Semaphore> derbyKey2Lock = new ConcurrentHashMap<>();
+  private static final String hostname = JavaUtils.hostname();
+
+  // Private methods should never catch SQLException and then throw MetaException.  The public
+  // methods depend on SQLException coming back so they can detect and handle deadlocks.  Private
+  // methods should only throw MetaException when they explicitly know there's a logic error and
+  // they want to throw past the public methods.
+  //
+  // All public methods that write to the database have to check for deadlocks when a SQLException
+  // comes back and handle it if they see one.  This has to be done with the connection pooling
+  // in mind.  To do this they should call checkRetryable() AFTER rolling back the db transaction,
+  // and then they should catch RetryException and call themselves recursively. See commitTxn for an example.
+
+  public TxnHandler() {
+  }
+
+  /**
+   * This is logically part of c'tor and must be called prior to any other method.
+   * Not physically part of c'tor due to use of reflection
+   */
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+
+    checkQFileTestHack();
+
+    synchronized (TxnHandler.class) {
+      if (connPool == null) {
+        Connection dbConn = null;
+        // Set up the JDBC connection pool
+        try {
+          int maxPoolSize = MetastoreConf.getIntVar(conf, ConfVars.CONNECTION_POOLING_MAX_CONNECTIONS);
+          long getConnectionTimeoutMs = 30000;
+          connPool = setupJdbcConnectionPool(conf, maxPoolSize, getConnectionTimeoutMs);
+          /*the mutex pools should ideally be somewhat larger since some operations require 1
+           connection from each pool and we want to avoid taking a connection from primary pool
+           and then blocking because mutex pool is empty.  There is only 1 thread in any HMS trying
+           to mutex on each MUTEX_KEY except MUTEX_KEY.CheckLock.  The CheckLock operation gets a
+           connection from connPool first, then connPoolMutex.  All others, go in the opposite
+           order (not very elegant...).  So number of connection requests for connPoolMutex cannot
+           exceed (size of connPool + MUTEX_KEY.values().length - 1).*/
+          connPoolMutex = setupJdbcConnectionPool(conf, maxPoolSize + MUTEX_KEY.values().length, getConnectionTimeoutMs);
+          dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+          determineDatabaseProduct(dbConn);
+          sqlGenerator = new SQLGenerator(dbProduct, conf);
+        } catch (SQLException e) {
+          String msg = "Unable to instantiate JDBC connection pooling, " + e.getMessage();
+          LOG.error(msg);
+          throw new RuntimeException(e);
+        } finally {
+          closeDbConn(dbConn);
+        }
+      }
+    }
+
+    numOpenTxns = Metrics.getOrCreateGauge(MetricsConstants.NUM_OPEN_TXNS);
+
+    timeout = MetastoreConf.getTimeVar(conf, ConfVars.TXN_TIMEOUT, TimeUnit.MILLISECONDS);
+    buildJumpTable();
+    retryInterval = MetastoreConf.getTimeVar(conf, ConfVars.HMSHANDLERINTERVAL,
+        TimeUnit.MILLISECONDS);
+    retryLimit = MetastoreConf.getIntVar(conf, ConfVars.HMSHANDLERATTEMPTS);
+    deadlockRetryInterval = retryInterval / 10;
+    maxOpenTxns = MetastoreConf.getIntVar(conf, ConfVars.MAX_OPEN_TXNS);
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  @RetrySemantics.ReadOnly
+  public GetOpenTxnsInfoResponse getOpenTxnsInfo() throws MetaException {
+    try {
+      // We need to figure out the current transaction number and the list of
+      // open transactions.  To avoid needing a transaction on the underlying
+      // database we'll look at the current transaction number first.  If it
+      // subsequently shows up in the open list that's ok.
+      Connection dbConn = null;
+      Statement stmt = null;
+      ResultSet rs = null;
+      try {
+        /**
+         * This method can run at READ_COMMITTED as long as long as
+         * {@link #openTxns(org.apache.hadoop.hive.metastore.api.OpenTxnRequest)} is atomic.
+         * More specifically, as long as advancing TransactionID in NEXT_TXN_ID is atomic with
+         * adding corresponding entries into TXNS.  The reason is that any txnid below HWM
+         * is either in TXNS and thus considered open (Open/Aborted) or it's considered Committed.
+         */
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select ntxn_next - 1 from NEXT_TXN_ID";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        if (!rs.next()) {
+          throw new MetaException("Transaction tables not properly " +
+            "initialized, no record found in next_txn_id");
+        }
+        long hwm = rs.getLong(1);
+        if (rs.wasNull()) {
+          throw new MetaException("Transaction tables not properly " +
+            "initialized, null record found in next_txn_id");
+        }
+        close(rs);
+        List<TxnInfo> txnInfos = new ArrayList<>();
+        //need the WHERE clause below to ensure consistent results with READ_COMMITTED
+        s = "select txn_id, txn_state, txn_user, txn_host, txn_started, txn_last_heartbeat from " +
+            "TXNS where txn_id <= " + hwm;
+        LOG.debug("Going to execute query<" + s + ">");
+        rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          char c = rs.getString(2).charAt(0);
+          TxnState state;
+          switch (c) {
+            case TXN_ABORTED:
+              state = TxnState.ABORTED;
+              break;
+
+            case TXN_OPEN:
+              state = TxnState.OPEN;
+              break;
+
+            default:
+              throw new MetaException("Unexpected transaction state " + c +
+                " found in txns table");
+          }
+          TxnInfo txnInfo = new TxnInfo(rs.getLong(1), state, rs.getString(3), rs.getString(4));
+          txnInfo.setStartedTime(rs.getLong(5));
+          txnInfo.setLastHeartbeatTime(rs.getLong(6));
+          txnInfos.add(txnInfo);
+        }
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+        return new GetOpenTxnsInfoResponse(hwm, txnInfos);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "getOpenTxnsInfo");
+        throw new MetaException("Unable to select from transaction database: " + getMessage(e)
+          + StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return getOpenTxnsInfo();
+    }
+  }
+  @Override
+  @RetrySemantics.ReadOnly
+  public GetOpenTxnsResponse getOpenTxns() throws MetaException {
+    try {
+      // We need to figure out the current transaction number and the list of
+      // open transactions.  To avoid needing a transaction on the underlying
+      // database we'll look at the current transaction number first.  If it
+      // subsequently shows up in the open list that's ok.
+      Connection dbConn = null;
+      Statement stmt = null;
+      ResultSet rs = null;
+      try {
+        /**
+         * This runs at READ_COMMITTED for exactly the same reason as {@link #getOpenTxnsInfo()}
+         */
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select ntxn_next - 1 from NEXT_TXN_ID";
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        if (!rs.next()) {
+          throw new MetaException("Transaction tables not properly " +
+            "initialized, no record found in next_txn_id");
+        }
+        long hwm = rs.getLong(1);
+        if (rs.wasNull()) {
+          throw new MetaException("Transaction tables not properly " +
+            "initialized, null record found in next_txn_id");
+        }
+        close(rs);
+        List<Long> openList = new ArrayList<>();
+        //need the WHERE clause below to ensure consistent results with READ_COMMITTED
+        s = "select txn_id, txn_state from TXNS where txn_id <= " + hwm + " order by txn_id";
+        LOG.debug("Going to execute query<" + s + ">");
+        rs = stmt.executeQuery(s);
+        long minOpenTxn = Long.MAX_VALUE;
+        BitSet abortedBits = new BitSet();
+        while (rs.next()) {
+          long txnId = rs.getLong(1);
+          openList.add(txnId);
+          char c = rs.getString(2).charAt(0);
+          if(c == TXN_OPEN) {
+            minOpenTxn = Math.min(minOpenTxn, txnId);
+          } else if (c == TXN_ABORTED) {
+            abortedBits.set(openList.size() - 1);
+          }
+        }
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+        ByteBuffer byteBuffer = ByteBuffer.wrap(abortedBits.toByteArray());
+        GetOpenTxnsResponse otr = new GetOpenTxnsResponse(hwm, openList, byteBuffer);
+        if(minOpenTxn < Long.MAX_VALUE) {
+          otr.setMin_open_txn(minOpenTxn);
+        }
+        return otr;
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "getOpenTxns");
+        throw new MetaException("Unable to select from transaction database, "
+          + StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return getOpenTxns();
+    }
+  }
+
+  /**
+   * Retry-by-caller note:
+   * Worst case, it will leave an open txn which will timeout.
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public OpenTxnsResponse openTxns(OpenTxnRequest rqst) throws MetaException {
+    if (!tooManyOpenTxns && numOpenTxns.get() >= maxOpenTxns) {
+      tooManyOpenTxns = true;
+    }
+    if (tooManyOpenTxns) {
+      if (numOpenTxns.get() < maxOpenTxns * 0.9) {
+        tooManyOpenTxns = false;
+      } else {
+        LOG.warn("Maximum allowed number of open transactions (" + maxOpenTxns + ") has been " +
+            "reached. Current number of open transactions: " + numOpenTxns);
+        throw new MetaException("Maximum allowed number of open transactions has been reached. " +
+            "See hive.max.open.txns.");
+      }
+    }
+
+    int numTxns = rqst.getNum_txns();
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      ResultSet rs = null;
+      try {
+        lockInternal();
+        /**
+         * To make {@link #getOpenTxns()}/{@link #getOpenTxnsInfo()} work correctly, this operation must ensure
+         * that advancing the counter in NEXT_TXN_ID and adding appropriate entries to TXNS is atomic.
+         * Also, advancing the counter must work when multiple metastores are running.
+         * SELECT ... FOR UPDATE is used to prevent
+         * concurrent DB transactions being rolled back due to Write-Write conflict on NEXT_TXN_ID.
+         *
+         * In the current design, there can be several metastore instances running in a given Warehouse.
+         * This makes ideas like reserving a range of IDs to save trips to DB impossible.  For example,
+         * a client may go to MS1 and start a transaction with ID 500 to update a particular row.
+         * Now the same client will start another transaction, except it ends up on MS2 and may get
+         * transaction ID 400 and update the same row.  Now the merge that happens to materialize the snapshot
+         * on read will thing the version of the row from transaction ID 500 is the latest one.
+         *
+         * Longer term we can consider running Active-Passive MS (at least wrt to ACID operations).  This
+         * set could support a write-through cache for added performance.
+         */
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        // Make sure the user has not requested an insane amount of txns.
+        int maxTxns = MetastoreConf.getIntVar(conf, ConfVars.TXN_MAX_OPEN_BATCH);
+        if (numTxns > maxTxns) numTxns = maxTxns;
+
+        stmt = dbConn.createStatement();
+        String s = sqlGenerator.addForUpdateClause("select ntxn_next from NEXT_TXN_ID");
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        if (!rs.next()) {
+          throw new MetaException("Transaction database not properly " +
+            "configured, can't find next transaction id.");
+        }
+        long first = rs.getLong(1);
+        s = "update NEXT_TXN_ID set ntxn_next = " + (first + numTxns);
+        LOG.debug("Going to execute update <" + s + ">");
+        stmt.executeUpdate(s);
+
+        long now = getDbTime(dbConn);
+        List<Long> txnIds = new ArrayList<>(numTxns);
+
+        List<String> rows = new ArrayList<>();
+        for (long i = first; i < first + numTxns; i++) {
+          txnIds.add(i);
+          rows.add(i + "," + quoteChar(TXN_OPEN) + "," + now + "," + now + "," + quoteString(rqst.getUser()) + "," + quoteString(rqst.getHostname()));
+        }
+        List<String> queries = sqlGenerator.createInsertValuesStmt(
+          "TXNS (txn_id, txn_state, txn_started, txn_last_heartbeat, txn_user, txn_host)", rows);
+        for (String q : queries) {
+          LOG.debug("Going to execute update <" + q + ">");
+          stmt.execute(q);
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+        return new OpenTxnsResponse(txnIds);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "openTxns(" + rqst + ")");
+        throw new MetaException("Unable to select from transaction database "
+          + StringUtils.stringifyException(e));
+      } finally {
+        close(rs, stmt, dbConn);
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      return openTxns(rqst);
+    }
+  }
+  @Override
+  @RetrySemantics.Idempotent
+  public void abortTxn(AbortTxnRequest rqst) throws NoSuchTxnException, MetaException, TxnAbortedException {
+    long txnid = rqst.getTxnid();
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      try {
+        lockInternal();
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        if (abortTxns(dbConn, Collections.singletonList(txnid), true) != 1) {
+          stmt = dbConn.createStatement();
+          TxnStatus status = findTxnState(txnid,stmt);
+          if(status == TxnStatus.ABORTED) {
+            LOG.info("abortTxn(" + JavaUtils.txnIdToString(txnid) +
+              ") requested by it is already " + TxnStatus.ABORTED);
+            return;
+          }
+          raiseTxnUnexpectedState(status, txnid);
+        }
+
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "abortTxn(" + rqst + ")");
+        throw new MetaException("Unable to update transaction database "
+          + StringUtils.stringifyException(e));
+      } finally {
+        close(null, stmt, dbConn);
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      abortTxn(rqst);
+    }
+  }
+  @Override
+  @RetrySemantics.Idempotent
+  public void abortTxns(AbortTxnsRequest rqst) throws NoSuchTxnException, MetaException {
+    List<Long> txnids = rqst.getTxn_ids();
+    try {
+      Connection dbConn = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        int numAborted = abortTxns(dbConn, txnids, false);
+        if (numAborted != txnids.size()) {
+          LOG.warn("Abort Transactions command only aborted " + numAborted + " out of " +
+              txnids.size() + " transactions. It's possible that the other " +
+              (txnids.size() - numAborted) +
+              " transactions have been aborted or committed, or the transaction ids are invalid.");
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "abortTxns(" + rqst + ")");
+        throw new MetaException("Unable to update transaction database "
+            + StringUtils.stringifyException(e));
+      } finally {
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      abortTxns(rqst);
+    }
+  }
+
+  /**
+   * Concurrency/isolation notes:
+   * This is mutexed with {@link #openTxns(OpenTxnRequest)} and other {@link #commitTxn(CommitTxnRequest)}
+   * operations using select4update on NEXT_TXN_ID.  Also, mutexes on TXNX table for specific txnid:X
+   * see more notes below.
+   * In order to prevent lost updates, we need to determine if any 2 transactions overlap.  Each txn
+   * is viewed as an interval [M,N]. M is the txnid and N is taken from the same NEXT_TXN_ID sequence
+   * so that we can compare commit time of txn T with start time of txn S.  This sequence can be thought of
+   * as a logical time counter.  If S.commitTime < T.startTime, T and S do NOT overlap.
+   *
+   * Motivating example:
+   * Suppose we have multi-statment transactions T and S both of which are attempting x = x + 1
+   * In order to prevent lost update problem, the the non-overlapping txns must lock in the snapshot
+   * that they read appropriately.  In particular, if txns do not overlap, then one follows the other
+   * (assumig they write the same entity), and thus the 2nd must see changes of the 1st.  We ensure
+   * this by locking in snapshot after 
+   * {@link #openTxns(OpenTxnRequest)} call is made (see org.apache.hadoop.hive.ql.Driver.acquireLocksAndOpenTxn)
+   * and mutexing openTxn() with commit().  In other words, once a S.commit() starts we must ensure
+   * that txn T which will be considered a later txn, locks in a snapshot that includes the result
+   * of S's commit (assuming no other txns).
+   * As a counter example, suppose we have S[3,3] and T[4,4] (commitId=txnid means no other transactions
+   * were running in parallel).  If T and S both locked in the same snapshot (for example commit of
+   * txnid:2, which is possible if commitTxn() and openTxnx() is not mutexed)
+   * 'x' would be updated to the same value by both, i.e. lost update. 
+   */
+  @Override
+  @RetrySemantics.Idempotent("No-op if already committed")
+  public void commitTxn(CommitTxnRequest rqst)
+    throws NoSuchTxnException, TxnAbortedException,  MetaException {
+    long txnid = rqst.getTxnid();
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      ResultSet lockHandle = null;
+      ResultSet commitIdRs = null, rs;
+      try {
+        lockInternal();
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        /**
+         * Runs at READ_COMMITTED with S4U on TXNS row for "txnid".  S4U ensures that no other
+         * operation can change this txn (such acquiring locks). While lock() and commitTxn()
+         * should not normally run concurrently (for same txn) but could due to bugs in the client
+         * which could then corrupt internal transaction manager state.  Also competes with abortTxn().
+         */
+        lockHandle = lockTransactionRecord(stmt, txnid, TXN_OPEN);
+        if (lockHandle == null) {
+          //if here, txn was not found (in expected state)
+          TxnStatus actualTxnStatus = findTxnState(txnid, stmt);
+          if(actualTxnStatus == TxnStatus.COMMITTED) {
+            /**
+             * This makes the operation idempotent
+             * (assume that this is most likely due to retry logic)
+             */
+            LOG.info("Nth commitTxn(" + JavaUtils.txnIdToString(txnid) + ") msg");
+            return;
+          }
+          raiseTxnUnexpectedState(actualTxnStatus, txnid);
+          shouldNeverHappen(txnid);
+          //dbConn is rolled back in finally{}
+        }
+        String conflictSQLSuffix = "from TXN_COMPONENTS where tc_txnid=" + txnid + " and tc_operation_type IN(" +
+          quoteChar(OpertaionType.UPDATE.sqlConst) + "," + quoteChar(OpertaionType.DELETE.sqlConst) + ")";
+        rs = stmt.executeQuery(sqlGenerator.addLimitClause(1, "tc_operation_type " + conflictSQLSuffix));
+        if (rs.next()) {
+          close(rs);
+          //if here it means currently committing txn performed update/delete and we should check WW conflict
+          /**
+           * This S4U will mutex with other commitTxn() and openTxns(). 
+           * -1 below makes txn intervals look like [3,3] [4,4] if all txns are serial
+           * Note: it's possible to have several txns have the same commit id.  Suppose 3 txns start
+           * at the same time and no new txns start until all 3 commit.
+           * We could've incremented the sequence for commitId is well but it doesn't add anything functionally.
+           */
+          commitIdRs = stmt.executeQuery(sqlGenerator.addForUpdateClause("select ntxn_next - 1 from NEXT_TXN_ID"));
+          if (!commitIdRs.next()) {
+            throw new IllegalStateException("No rows found in NEXT_TXN_ID");
+          }
+          long commitId = commitIdRs.getLong(1);
+          Savepoint undoWriteSetForCurrentTxn = dbConn.setSavepoint();
+          /**
+           * "select distinct" is used below because
+           * 1. once we get to multi-statement txns, we only care to record that something was updated once
+           * 2. if {@link #addDynamicPartitions(AddDynamicPartitions)} is retried by caller it my create
+           *  duplicate entries in TXN_COMPONENTS
+           * but we want to add a PK on WRITE_SET which won't have unique rows w/o this distinct
+           * even if it includes all of it's columns
+           */
+          int numCompsWritten = stmt.executeUpdate(
+            "insert into WRITE_SET (ws_database, ws_table, ws_partition, ws_txnid, ws_commit_id, ws_operation_type)" +
+            " select distinct tc_database, tc_table, tc_partition, tc_txnid, " + commitId + ", tc_operation_type " + conflictSQLSuffix);
+          /**
+           * see if there are any overlapping txns wrote the same element, i.e. have a conflict
+           * Since entire commit operation is mutexed wrt other start/commit ops,
+           * committed.ws_commit_id <= current.ws_commit_id for all txns
+           * thus if committed.ws_commit_id < current.ws_txnid, transactions do NOT overlap
+           * For example, [17,20] is committed, [6,80] is being committed right now - these overlap
+           * [17,20] committed and [21,21] committing now - these do not overlap.
+           * [17,18] committed and [18,19] committing now - these overlap  (here 18 started while 17 was still running)
+           */
+          rs = stmt.executeQuery
+            (sqlGenerator.addLimitClause(1, "committed.ws_txnid, committed.ws_commit_id, committed.ws_database," +
+              "committed.ws_table, committed.ws_partition, cur.ws_commit_id cur_ws_commit_id, " +
+              "cur.ws_operation_type cur_op, committed.ws_operation_type committed_op " +
+              "from WRITE_SET committed INNER JOIN WRITE_SET cur " +
+              "ON committed.ws_database=cur.ws_database and committed.ws_table=cur.ws_table " +
+              //For partitioned table we always track writes at partition level (never at table)
+              //and for non partitioned - always at table level, thus the same table should never
+              //have entries with partition key and w/o
+              "and (committed.ws_partition=cur.ws_partition or (committed.ws_partition is null and cur.ws_partition is null)) " +
+              "where cur.ws_txnid <= committed.ws_commit_id" + //txns overlap; could replace ws_txnid
+              // with txnid, though any decent DB should infer this
+              " and cur.ws_txnid=" + txnid + //make sure RHS of join only has rows we just inserted as
+              // part of this commitTxn() op
+              " and committed.ws_txnid <> " + txnid + //and LHS only has committed txns
+              //U+U and U+D is a conflict but D+D is not and we don't currently track I in WRITE_SET at all
+              " and (committed.ws_operation_type=" + quoteChar(OpertaionType.UPDATE.sqlConst) +
+              " OR cur.ws_operation_type=" + quoteChar(OpertaionType.UPDATE.sqlConst) + ")"));
+          if (rs.next()) {
+            //found a conflict
+            String committedTxn = "[" + JavaUtils.txnIdToString(rs.getLong(1)) + "," + rs.getLong(2) + "]";
+            StringBuilder resource = new StringBuilder(rs.getString(3)).append("/").append(rs.getString(4));
+            String partitionName = rs.getString(5);
+            if (partitionName != null) {
+              resource.append('/').append(partitionName);
+            }
+            String msg = "Aborting [" + JavaUtils.txnIdToString(txnid) + "," + rs.getLong(6) + "]" + " due to a write conflict on " + resource +
+              " committed by " + committedTxn + " " + rs.getString(7) + "/" + rs.getString(8);
+            close(rs);
+            //remove WRITE_SET info for current txn since it's about to abort
+            dbConn.rollback(undoWriteSetForCurrentTxn);
+            LOG.info(msg);
+            //todo: should make abortTxns() write something into TXNS.TXN_META_INFO about this
+            if (abortTxns(dbConn, Collections.singletonList(txnid), true) != 1) {
+              throw new IllegalStateException(msg + " FAILED!");
+            }
+            dbConn.commit();
+            close(null, stmt, dbConn);
+            throw new TxnAbortedException(msg);
+          } else {
+            //no conflicting operations, proceed with the rest of commit sequence
+          }
+        }
+        else {
+          /**
+           * current txn didn't update/delete anything (may have inserted), so just proceed with commit
+           *
+           * We only care about commit id for write txns, so for RO (when supported) txns we don't
+           * have to mutex on NEXT_TXN_ID.
+           * Consider: if RO txn is after a W txn, then RO's openTxns() will be mutexed with W's
+           * commitTxn() because both do S4U on NEXT_TXN_ID and thus RO will see result of W txn.
+           * If RO < W, then there is no reads-from relationship.
+           */
+        }
+        // Move the record from txn_components into completed_txn_components so that the compactor
+        // knows where to look to compact.
+        String s = "insert into COMPLETED_TXN_COMPONENTS select tc_txnid, tc_database, tc_table, " +
+          "tc_partition from TXN_COMPONENTS where tc_txnid = " + txnid;
+        LOG.debug("Going to execute insert <" + s + ">");
+        int modCount = 0;
+        if ((modCount = stmt.executeUpdate(s)) < 1) {
+          //this can be reasonable for an empty txn START/COMMIT or read-only txn
+          //also an IUD with DP that didn't match any rows.
+          LOG.info("Expected to move at least one record from txn_components to " +
+            "completed_txn_components when committing txn! " + JavaUtils.txnIdToString(txnid));
+        }
+        s = "delete from TXN_COMPONENTS where tc_txnid = " + txnid;
+        LOG.debug("Going to execute update <" + s + ">");
+        modCount = stmt.executeUpdate(s);
+        s = "delete from HIVE_LOCKS where hl_txnid = " + txnid;
+        LOG.debug("Going to execute update <" + s + ">");
+        modCount = stmt.executeUpdate(s);
+        s = "delete from TXNS where txn_id = " + txnid;
+        LOG.debug("Going to execute update <" + s + ">");
+        modCount = stmt.executeUpdate(s);
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "commitTxn(" + rqst + ")");
+        throw new MetaException("Unable to update transaction database "
+          + StringUtils.stringifyException(e));
+      } finally {
+        close(commitIdRs);
+        close(lockHandle, stmt, dbConn);
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      commitTxn(rqst);
+    }
+  }
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void performWriteSetGC() {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+      stmt = dbConn.createStatement();
+      rs = stmt.executeQuery("select ntxn_next - 1 from NEXT_TXN_ID");
+      if(!rs.next()) {
+        throw new IllegalStateException("NEXT_TXN_ID is empty: DB is corrupted");
+      }
+      long highestAllocatedTxnId = rs.getLong(1);
+      close(rs);
+      rs = stmt.executeQuery("select min(txn_id) from TXNS where txn_state=" + quoteChar(TXN_OPEN));
+      if(!rs.next()) {
+        throw new IllegalStateException("Scalar query returned no rows?!?!!");
+      }
+      long commitHighWaterMark;//all currently open txns (if any) have txnid >= than commitHighWaterMark
+      long lowestOpenTxnId = rs.getLong(1);
+      if(rs.wasNull()) {
+        //if here then there are no Open txns and  highestAllocatedTxnId must be
+        //resolved (i.e. committed or aborted), either way
+        //there are no open txns with id <= highestAllocatedTxnId
+        //the +1 is there because "delete ..." below has < (which is correct for the case when
+        //there is an open txn
+        //Concurrency: even if new txn starts (or starts + commits) it is still true that
+        //there are no currently open txns that overlap with any committed txn with 
+        //commitId <= commitHighWaterMark (as set on next line).  So plain READ_COMMITTED is enough.
+        commitHighWaterMark = highestAllocatedTxnId + 1;
+      }
+      else {
+        commitHighWaterMark = lowestOpenTxnId;
+      }
+      int delCnt = stmt.executeUpdate("delete from WRITE_SET where ws_commit_id < " + commitHighWaterMark);
+      LOG.info("Deleted " + delCnt + " obsolete rows from WRTIE_SET");
+      dbConn.commit();
+    } catch (SQLException ex) {
+      LOG.warn("WriteSet GC failed due to " + getMessage(ex), ex);
+    }
+    finally {
+      close(rs, stmt, dbConn);
+    }
+  }
+  /**
+   * As much as possible (i.e. in absence of retries) we want both operations to be done on the same
+   * connection (but separate transactions).  This avoid some flakiness in BONECP where if you
+   * perform an operation on 1 connection and immediately get another from the pool, the 2nd one
+   * doesn't see results of the first.
+   * 
+   * Retry-by-caller note: If the call to lock is from a transaction, then in the worst case
+   * there will be a duplicate set of locks but both sets will belong to the same txn so they 
+   * will not conflict with each other.  For locks w/o txn context (i.e. read-only query), this
+   * may lead to deadlock (at least a long wait).  (e.g. 1st call creates locks in {@code LOCK_WAITING}
+   * mode and response gets lost.  Then {@link org.apache.hadoop.hive.metastore.RetryingMetaStoreClient}
+   * retries, and enqueues another set of locks in LOCK_WAITING.  The 2nd LockResponse is delivered
+   * to the DbLockManager, which will keep dong {@link #checkLock(CheckLockRequest)} until the 1st
+   * set of locks times out.
+   */
+  @RetrySemantics.CannotRetry
+  public LockResponse lock(LockRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
+    ConnectionLockIdPair connAndLockId = enqueueLockWithRetry(rqst);
+    try {
+      return checkLockWithRetry(connAndLockId.dbConn, connAndLockId.extLockId, rqst.getTxnid());
+    }
+    catch(NoSuchLockException e) {
+      // This should never happen, as we just added the lock id
+      throw new MetaException("Couldn't find a lock we just created! " + e.getMessage());
+    }
+  }
+  private static final class ConnectionLockIdPair {
+    private final Connection dbConn;
+    private final long extLockId;
+    private ConnectionLockIdPair(Connection dbConn, long extLockId) {
+      this.dbConn = dbConn;
+      this.extLockId = extLockId;
+    }
+  }
+
+  /**
+   * Note that by definition select for update is divorced from update, i.e. you executeQuery() to read
+   * and then executeUpdate().  One other alternative would be to actually update the row in TXNS but
+   * to the same value as before thus forcing db to acquire write lock for duration of the transaction.
+   *
+   * There is no real reason to return the ResultSet here other than to make sure the reference to it
+   * is retained for duration of intended lock scope and is not GC'd thus (unlikely) causing lock
+   * to be released.
+   * @param txnState the state this txn is expected to be in.  may be null
+   * @return null if no row was found
+   * @throws SQLException
+   * @throws MetaException
+   */
+  private ResultSet lockTransactionRecord(Statement stmt, long txnId, Character txnState) throws SQLException, MetaException {
+    String query = "select TXN_STATE from TXNS where TXN_ID = " + txnId + (txnState != null ? " AND TXN_STATE=" + quoteChar(txnState) : "");
+    ResultSet rs = stmt.executeQuery(sqlGenerator.addForUpdateClause(query));
+    if(rs.next()) {
+      return rs;
+    }
+    close(rs);
+    return null;
+  }
+
+  /**
+   * This enters locks into the queue in {@link #LOCK_WAITING} mode.
+   *
+   * Isolation Level Notes:
+   * 1. We use S4U (withe read_committed) to generate the next (ext) lock id.  This serializes
+   * any 2 {@code enqueueLockWithRetry()} calls.
+   * 2. We use S4U on the relevant TXNS row to block any concurrent abort/commit/etc operations
+   * @see #checkLockWithRetry(Connection, long, long)
+   */
+  private ConnectionLockIdPair enqueueLockWithRetry(LockRequest rqst) throws NoSuchTxnException, TxnAbortedException, MetaException {
+    boolean success = false;
+    Connection dbConn = null;
+    try {
+      Statement stmt = null;
+      ResultSet rs = null;
+      ResultSet lockHandle = null;
+      try {
+        lockInternal();
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        long txnid = rqst.getTxnid();
+        stmt = dbConn.createStatement();
+        if (isValidTxn(txnid)) {
+          //this also ensures that txn is still there in expected state
+          lockHandle = lockTransactionRecord(stmt, txnid, TXN_OPEN);
+          if(lockHandle == null) {
+            ensureValidTxn(dbConn, txnid, stmt);
+            shouldNeverHappen(txnid);
+          }
+        }
+        /** Get the next lock id.
+         * This has to be atomic with adding entries to HIVE_LOCK entries (1st add in W state) to prevent a race.
+         * Suppose ID gen is a separate txn and 2 concurrent lock() methods are running.  1st one generates nl_next=7,
+         * 2nd nl_next=8.  Then 8 goes first to insert into HIVE_LOCKS and acquires the locks.  Then 7 unblocks,
+         * and add it's W locks but it won't see locks from 8 since to be 'fair' {@link #checkLock(java.sql.Connection, long)}
+         * doesn't block on locks acquired later than one it's checking*/
+        String s = sqlGenerator.addForUpdateClause("select nl_next from NEXT_LOCK_ID");
+        LOG.debug("Going to execute query <" + s + ">");
+        rs = stmt.executeQuery(s);
+        if (!rs.next()) {
+          LOG.debug("Going to rollback");
+          dbConn.rollback();
+          throw new MetaException("Transaction tables not properly " +
+            "initialized, no record found in next_lock_id");
+        }
+        long extLockId = rs.getLong(1);
+        s = "update NEXT_LOCK_ID set nl_next = " + (extLockId + 1);
+        LOG.debug("Going to execute update <" + s + ">");
+        stmt.executeUpdate(s);
+
+        if (txnid > 0) {
+          List<String> rows = new ArrayList<>();
+          // For each component in this lock request,
+          // add an entry to the txn_components table
+          for (LockComponent lc : rqst.getComponent()) {
+            if(lc.isSetIsAcid() && !lc.isIsAcid()) {
+              //we don't prevent using non-acid resources in a txn but we do lock them
+              continue;
+            }
+            boolean updateTxnComponents;
+            if(!lc.isSetOperationType()) {
+              //request came from old version of the client
+              updateTxnComponents = true;//this matches old behavior
+            }
+            else {
+              switch (lc.getOperationType()) {
+                case INSERT:
+                case UPDATE:
+                case DELETE:
+                  if(!lc.isSetIsDynamicPartitionWrite()) {
+                    //must be old client talking, i.e. we don't know if it's DP so be conservative
+                    updateTxnComponents = true;
+                  }
+                  else {
+                    /**
+                     * we know this is part of DP operation and so we'll get
+                     * {@link #addDynamicPartitions(AddDynamicPartitions)} call with the list
+                     * of partitions actually chaged.
+                     */
+                    updateTxnComponents = !lc.isIsDynamicPartitionWrite();
+                  }
+                  break;
+                case SELECT:
+                  updateTxnComponents = false;
+                  break;
+                case NO_TXN:
+                  /*this constant is a bit of a misnomer since we now always have a txn context.  It
+                   just means the operation is such that we don't care what tables/partitions it
+                   affected as it doesn't trigger a compaction or conflict detection.  A better name
+                   would be NON_TRANSACTIONAL.*/
+                  updateTxnComponents = false;
+                  break;
+                default:
+                  //since we have an open transaction, only 4 values above are expected 
+                  throw new IllegalStateException("Unexpected DataOperationType: " + lc.getOperationType()
+                    + " agentInfo=" + rqst.getAgentInfo() + " " + JavaUtils.txnIdToString(txnid));
+              }
+            }
+            if(!updateTxnComponents) {
+              continue;
+            }
+            String dbName = lc.getDbname();
+            String tblName = lc.getTablename();
+            String partName = lc.getPartitionname();
+            rows.add(txnid + ", '" + dbName + "', " +
+              (tblName == null ? "null" : "'" + tblName + "'") + ", " +
+              (partName == null ? "null" : "'" + partName + "'")+ "," +
+              quoteString(OpertaionType.fromDataOperationType(lc.getOperationType()).toString()));
+          }
+          List<String> queries = sqlGenerator.createInsertValuesStmt(
+            "TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition, tc_operation_type)", rows);
+          for(String query : queries) {
+            LOG.debug("Going to execute update <" + query + ">");
+            int modCount = stmt.executeUpdate(query);
+          }
+        }
+
+        List<String> rows = new ArrayList<>();
+        long intLockId = 0;
+        for (LockComponent lc : rqst.getComponent()) {
+          if(lc.isSetOperationType() && lc.getOperationType() == DataOperationType.UNSET &&
+            (MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEST) || MetastoreConf.getBoolVar(conf, ConfVars.HIVE_IN_TEZ_TEST))) {
+            //old version of thrift client should have (lc.isSetOperationType() == false) but they do not
+            //If you add a default value to a variable, isSet() for that variable is true regardless of the where the
+            //message was created (for object variables.  It works correctly for boolean vars, e.g. LockComponent.isAcid).
+            //in test mode, upgrades are not tested, so client version and server version of thrift always matches so
+            //we see UNSET here it means something didn't set the appropriate value.
+            throw new IllegalStateException("Bug: operationType=" + lc.getOperationType() + " for component "
+              + lc + " agentInfo=" + rqst.getAgentInfo());
+          }
+          intLockId++;
+          String dbName = lc.getDbname();
+          String tblName = lc.getTablename();
+          String partName = lc.getPartitionname();
+          LockType lockType = lc.getType();
+          char lockChar = 'z';
+          switch (lockType) {
+            case EXCLUSIVE:
+              lockChar = LOCK_EXCLUSIVE;
+              break;
+            case SHARED_READ:
+              lockChar = LOCK_SHARED;
+              break;
+            case SHARED_WRITE:
+              lockChar = LOCK_SEMI_SHARED;
+              break;
+          }
+          long now = getDbTime(dbConn);
+            rows.add(extLockId + ", " + intLockId + "," + txnid + ", " +
+            quoteString(dbName) + ", " +
+            valueOrNullLiteral(tblName) + ", " +
+            valueOrNullLiteral(partName) + ", " +
+            quoteChar(LOCK_WAITING) + ", " + quoteChar(lockChar) + ", " +
+            //for locks associated with a txn, we always heartbeat txn and timeout based on that
+            (isValidTxn(txnid) ? 0 : now) + ", " +
+            valueOrNullLiteral(rqst.getUser()) + ", " +
+            valueOrNullLiteral(rqst.getHostname()) + ", " +
+            valueOrNullLiteral(rqst.getAgentInfo()));// + ")";
+        }
+        List<String> queries = sqlGenerator.createInsertValuesStmt(
+          "HIVE_LOCKS (hl_lock_ext_id, hl_lock_int_id, hl_txnid, hl_db, " +
+            "hl_table, hl_partition,hl_lock_state, hl_lock_type, " +
+            "hl_last_heartbeat, hl_user, hl_host, hl_agent_info)", rows);
+        for(String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          int modCount = stmt.executeUpdate(query);
+        }
+        dbConn.commit();
+        success = true;
+        return new ConnectionLockIdPair(dbConn, extLockId);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "enqueueLockWithRetry(" + rqst + ")");
+        throw new MetaException("Unable to update transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(lockHandle);
+        close(rs, stmt, null);
+        if (!success) {
+          /* This needs to return a "live" connection to be used by operation that follows it.
+          Thus it only closes Connection on failure/retry. */
+          closeDbConn(dbConn);
+        }
+        unlockInternal();
+      }
+    }
+    catch(RetryException e) {
+      return enqueueLockWithRetry(rqst);
+    }
+  }
+  private LockResponse checkLockWithRetry(Connection dbConn, long extLockId, long txnId)
+    throws NoSuchLockException, NoSuchTxnException, TxnAbortedException, MetaException {
+    try {
+      try {
+        lockInternal();
+        if(dbConn.isClosed()) {
+          //should only get here if retrying this op
+          dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        }
+        return checkLock(dbConn, extLockId);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "checkLockWithRetry(" + extLockId + "," + txnId + ")");
+        throw new MetaException("Unable to update transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        unlockInternal();
+        closeDbConn(dbConn);
+      }
+    }
+    catch(RetryException e) {
+      return checkLockWithRetry(dbConn, extLockId, txnId);
+    }
+  }
+  /**
+   * Why doesn't this get a txnid as parameter?  The caller should either know the txnid or know there isn't one.
+   * Either way getTxnIdFromLockId() will not be needed.  This would be a Thrift change.
+   *
+   * Also, when lock acquisition returns WAITING, it's retried every 15 seconds (best case, see DbLockManager.backoff(),
+   * in practice more often)
+   * which means this is heartbeating way more often than hive.txn.timeout and creating extra load on DB.
+   *
+   * The clients that operate in blocking mode, can't heartbeat a lock until the lock is acquired.
+   * We should make CheckLockRequest include timestamp or last request to skip unnecessary heartbeats. Thrift change.
+   *
+   * {@link #checkLock(java.sql.Connection, long)}  must run at SERIALIZABLE (make sure some lock we are checking
+   * against doesn't move from W to A in another txn) but this method can heartbeat in
+   * separate txn at READ_COMMITTED.
+   * 
+   * Retry-by-caller note:
+   * Retryable because {@link #checkLock(Connection, long)} is
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public LockResponse checkLock(CheckLockRequest rqst)
+    throws NoSuchTxnException, NoSuchLockException, TxnAbortedException, MetaException {
+    try {
+      Connection dbConn = null;
+      long extLockId = rqst.getLockid();
+      try {
+        lockInternal();
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        // Heartbeat on the lockid first, to assure that our lock is still valid.
+        // Then look up the lock info (hopefully in the cache).  If these locks
+        // are associated with a transaction then heartbeat on that as well.
+        LockInfo info = getTxnIdFromLockId(dbConn, extLockId);
+        if(info == null) {
+          throw new NoSuchLockException("No such lock " + JavaUtils.lockIdToString(extLockId));
+        }
+        if (info.txnId > 0) {
+          heartbeatTxn(dbConn, info.txnId);
+        }
+        else {
+          heartbeatLock(dbConn, extLockId);
+        }
+        //todo: strictly speaking there is a bug here.  heartbeat*() commits but both heartbeat and
+        //checkLock() are in the same retry block, so if checkLock() throws, heartbeat is also retired
+        //extra heartbeat is logically harmless, but ...
+        return checkLock(dbConn, extLockId);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "checkLock(" + rqst + " )");
+        throw new MetaException("Unable to update transaction database " +
+          JavaUtils.lockIdToString(extLockId) + " " + StringUtils.stringifyException(e));
+      } finally {
+        closeDbConn(dbConn);
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      return checkLock(rqst);
+    }
+
+  }
+
+  /**
+   * This would have been made simpler if all locks were associated with a txn.  Then only txn needs to
+   * be heartbeated, committed, etc.  no need for client to track individual locks.
+   * When removing locks not associated with txn this potentially conflicts with
+   * heartbeat/performTimeout which are update/delete of HIVE_LOCKS thus will be locked as needed by db.
+   * since this only removes from HIVE_LOCKS at worst some lock acquire is delayed
+   */
+  @RetrySemantics.Idempotent
+  public void unlock(UnlockRequest rqst)
+    throws NoSuchLockException, TxnOpenException, MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      long extLockId = rqst.getLockid();
+      try {
+        /**
+         * This method is logically like commit for read-only auto commit queries.
+         * READ_COMMITTED since this only has 1 delete statement and no new entries with the
+         * same hl_lock_ext_id can be added, i.e. all rows with a given hl_lock_ext_id are
+         * created in a single atomic operation.
+         * Theoretically, this competes with {@link #lock(org.apache.hadoop.hive.metastore.api.LockRequest)}
+         * but hl_lock_ext_id is not known until that method returns.
+         * Also competes with {@link #checkLock(org.apache.hadoop.hive.metastore.api.CheckLockRequest)}
+         * but using SERIALIZABLE doesn't materially change the interaction.
+         * If "delete" stmt misses, additional logic is best effort to produce meaningful error msg.
+         */
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        //hl_txnid <> 0 means it's associated with a transaction
+        String s = "delete from HIVE_LOCKS where hl_lock_ext_id = " + extLockId + " AND (hl_txnid = 0 OR" +
+          " (hl_txnid <> 0 AND hl_lock_state = '" + LOCK_WAITING + "'))";
+        //(hl_txnid <> 0 AND hl_lock_state = '" + LOCK_WAITING + "') is for multi-statement txns where
+        //some query attempted to lock (thus LOCK_WAITING state) but is giving up due to timeout for example
+        LOG.debug("Going to execute update <" + s + ">");
+        int rc = stmt.executeUpdate(s);
+        if (rc < 1) {
+          LOG.debug("Going to rollback");
+          dbConn.rollback();
+          LockInfo info = getTxnIdFromLockId(dbConn, extLockId);
+          if(info == null) {
+            //didn't find any lock with extLockId but at ReadCommitted there is a possibility that
+            //it existed when above delete ran but it didn't have the expected state.
+            LOG.info("No lock in " + LOCK_WAITING + " mode found for unlock(" +
+              JavaUtils.lockIdToString(rqst.getLockid()) + ")");
+            //bail here to make the operation idempotent
+            return;
+          }
+          if(info.txnId != 0) {
+            String msg = "Unlocking locks associated with transaction not permitted.  " + info;
+            //if a lock is associated with a txn we can only "unlock" if if it's in WAITING state
+            // which really means that the caller wants to give up waiting for the lock
+            LOG.error(msg);
+            throw new TxnOpenException(msg);
+          }
+          if(info.txnId == 0) {
+            //we didn't see this lock when running DELETE stmt above but now it showed up
+            //so should "should never happen" happened...
+            String msg = "Found lock in unexpected state " + info;
+            LOG.error(msg);
+            throw new MetaException(msg);
+          }
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "unlock(" + rqst + ")");
+        throw new MetaException("Unable to update transaction database " +
+          JavaUtils.lockIdToString(extLockId) + " " + StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      unlock(rqst);
+    }
+  }
+
+  /**
+   * used to sort entries in {@link org.apache.hadoop.hive.metastore.api.ShowLocksResponse}
+   */
+  private static class LockInfoExt extends LockInfo {
+    private final ShowLocksResponseElement e;
+    LockInfoExt(ShowLocksResponseElement e) {
+      super(e);
+      this.e = e;
+    }
+  }
+  @RetrySemantics.ReadOnly
+  public ShowLocksResponse showLocks(ShowLocksRequest rqst) throws MetaException {
+    try {
+      Connection dbConn = null;
+      ShowLocksResponse rsp = new ShowLocksResponse();
+      List<ShowLocksResponseElement> elems = new ArrayList<>();
+      List<LockInfoExt> sortedList = new ArrayList<>();
+      Statement stmt = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+
+        String s = "select hl_lock_ext_id, hl_txnid, hl_db, hl_table, hl_partition, hl_lock_state, " +
+          "hl_lock_type, hl_last_heartbeat, hl_acquired_at, hl_user, hl_host, hl_lock_int_id," +
+          "hl_blockedby_ext_id, hl_blockedby_int_id, hl_agent_info from HIVE_LOCKS";
+
+        // Some filters may have been specified in the SHOW LOCKS statement. Add them to the query.
+        String dbName = rqst.getDbname();
+        String tableName = rqst.getTablename();
+        String partName = rqst.getPartname();
+
+        StringBuilder filter = new StringBuilder();
+        if (dbName != null && !dbName.isEmpty()) {
+          filter.append("hl_db=").append(quoteString(dbName));
+        }
+        if (tableName != null && !tableName.isEmpty()) {
+          if (filter.length() > 0) {
+            filter.append(" and ");
+          }
+          filter.append("hl_table=").append(quoteString(tableName));
+        }
+        if (partName != null && !partName.isEmpty()) {
+          if (filter.length() > 0) {
+            filter.append(" and ");
+          }
+          filter.append("hl_partition=").append(quoteString(partName));
+        }
+        String whereClause = filter.toString();
+
+        if (!whereClause.isEmpty()) {
+          s = s + " where " + whereClause;
+        }
+
+        LOG.debug("Doing to execute query <" + s + ">");
+        ResultSet rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          ShowLocksResponseElement e = new ShowLocksResponseElement();
+          e.setLockid(rs.getLong(1));
+          long txnid = rs.getLong(2);
+          if (!rs.wasNull()) e.setTxnid(txnid);
+          e.setDbname(rs.getString(3));
+          e.setTablename(rs.getString(4));
+          String partition = rs.getString(5);
+          if (partition != null) e.setPartname(partition);
+          switch (rs.getString(6).charAt(0)) {
+            case LOCK_ACQUIRED: e.setState(LockState.ACQUIRED); break;
+            case LOCK_WAITING: e.setState(LockState.WAITING); break;
+            default: throw new MetaException("Unknown lock state " + rs.getString(6).charAt(0));
+          }
+          switch (rs.getString(7).charAt(0)) {
+            case LOCK_SEMI_SHARED: e.setType(LockType.SHARED_WRITE); break;
+            case LOCK_EXCLUSIVE: e.setType(LockType.EXCLUSIVE); break;
+            case LOCK_SHARED: e.setType(LockType.SHARED_READ); break;
+            default: throw new MetaException("Unknown lock type " + rs.getString(6).charAt(0));
+          }
+          e.setLastheartbeat(rs.getLong(8));
+          long acquiredAt = rs.getLong(9);
+          if (!rs.wasNull()) e.setAcquiredat(acquiredAt);
+          e.setUser(rs.getString(10));
+          e.setHostname(rs.getString(11));
+          e.setLockIdInternal(rs.getLong(12));
+          long id = rs.getLong(13);
+          if(!rs.wasNull()) {
+            e.setBlockedByExtId(id);
+          }
+          id = rs.getLong(14);
+          if(!rs.wasNull()) {
+            e.setBlockedByIntId(id);
+          }
+          e.setAgentInfo(rs.getString(15));
+          sortedList.add(new LockInfoExt(e));
+        }
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+      } catch (SQLException e) {
+        checkRetryable(dbConn, e, "showLocks(" + rqst + ")");
+        throw new MetaException("Unable to select from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+      //this ensures that "SHOW LOCKS" prints the locks in the same order as they are examined
+      //by checkLock() - makes diagnostics easier.
+      Collections.sort(sortedList, new LockInfoComparator());
+      for(LockInfoExt lockInfoExt : sortedList) {
+        elems.add(lockInfoExt.e);
+      }
+      rsp.setLocks(elems);
+      return rsp;
+    } catch (RetryException e) {
+      return showLocks(rqst);
+    }
+  }
+
+  /**
+   * {@code ids} should only have txnid or lockid but not both, ideally.
+   * Currently DBTxnManager.heartbeat() enforces this.
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void heartbeat(HeartbeatRequest ids)
+    throws NoSuchTxnException,  NoSuchLockException, TxnAbortedException, MetaException {
+    try {
+      Connection dbConn = null;
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        heartbeatLock(dbConn, ids.getLockid());
+        heartbeatTxn(dbConn, ids.getTxnid());
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "heartbeat(" + ids + ")");
+        throw new MetaException("Unable to select from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      heartbeat(ids);
+    }
+  }
+  @Override
+  @RetrySemantics.SafeToRetry
+  public HeartbeatTxnRangeResponse heartbeatTxnRange(HeartbeatTxnRangeRequest rqst)
+    throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      HeartbeatTxnRangeResponse rsp = new HeartbeatTxnRangeResponse();
+      Set<Long> nosuch = new HashSet<>();
+      Set<Long> aborted = new HashSet<>();
+      rsp.setNosuch(nosuch);
+      rsp.setAborted(aborted);
+      try {
+        /**
+         * READ_COMMITTED is sufficient since {@link #heartbeatTxn(java.sql.Connection, long)}
+         * only has 1 update statement in it and
+         * we only update existing txns, i.e. nothing can add additional txns that this operation
+         * would care about (which would have required SERIALIZABLE)
+         */
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        /*do fast path first (in 1 statement) if doesn't work, rollback and do the long version*/
+        stmt = dbConn.createStatement();
+        List<String> queries = new ArrayList<>();
+        int numTxnsToHeartbeat = (int) (rqst.getMax() - rqst.getMin() + 1);
+        List<Long> txnIds = new ArrayList<>(numTxnsToHeartbeat);
+        for (long txn = rqst.getMin(); txn <= rqst.getMax(); txn++) {
+          txnIds.add(txn);
+        }
+        TxnUtils.buildQueryWithINClause(conf, queries,
+          new StringBuilder("update TXNS set txn_last_heartbeat = " + getDbTime(dbConn) +
+            " where txn_state = " + quoteChar(TXN_OPEN) + " and "),
+          new StringBuilder(""), txnIds, "txn_id", true, false);
+        int updateCnt = 0;
+        for (String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          updateCnt += stmt.executeUpdate(query);
+        }
+        if (updateCnt == numTxnsToHeartbeat) {
+          //fast pass worked, i.e. all txns we were asked to heartbeat were Open as expected
+          dbConn.commit();
+          return rsp;
+        }
+        //if here, do the slow path so that we can return info txns which were not in expected state
+        dbConn.rollback();
+        for (long txn = rqst.getMin(); txn <= rqst.getMax(); txn++) {
+          try {
+            heartbeatTxn(dbConn, txn);
+          } catch (NoSuchTxnException e) {
+            nosuch.add(txn);
+          } catch (TxnAbortedException e) {
+            aborted.add(txn);
+          }
+        }
+        return rsp;
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "heartbeatTxnRange(" + rqst + ")");
+        throw new MetaException("Unable to select from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(null, stmt, dbConn);
+      }
+    } catch (RetryException e) {
+      return heartbeatTxnRange(rqst);
+    }
+  }
+
+  long generateCompactionQueueId(Statement stmt) throws SQLException, MetaException {
+    // Get the id for the next entry in the queue
+    String s = sqlGenerator.addForUpdateClause("select ncq_next from NEXT_COMPACTION_QUEUE_ID");
+    LOG.debug("going to execute query <" + s + ">");
+    try (ResultSet rs = stmt.executeQuery(s)) {
+      if (!rs.next()) {
+        throw new IllegalStateException("Transaction tables not properly initiated, "
+            + "no record found in next_compaction_queue_id");
+      }
+      long id = rs.getLong(1);
+      s = "update NEXT_COMPACTION_QUEUE_ID set ncq_next = " + (id + 1);
+      LOG.debug("Going to execute update <" + s + ">");
+      stmt.executeUpdate(s);
+      return id;
+    }
+  }
+  @Override
+  @RetrySemantics.Idempotent
+  public CompactionResponse compact(CompactionRequest rqst) throws MetaException {
+    // Put a compaction request in the queue.
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+      TxnStore.MutexAPI.LockHandle handle = null;
+      try {
+        lockInternal();
+        /**
+         * MUTEX_KEY.CompactionScheduler lock ensures that there is only 1 entry in
+         * Initiated/Working state for any resource.  This ensures that we don't run concurrent
+         * compactions for any resource.
+         */
+        handle = getMutexAPI().acquireLock(MUTEX_KEY.CompactionScheduler.name());
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+
+        long id = generateCompactionQueueId(stmt);
+
+        StringBuilder sb = new StringBuilder("select cq_id, cq_state from COMPACTION_QUEUE where").
+          append(" cq_state IN(").append(quoteChar(INITIATED_STATE)).
+            append(",").append(quoteChar(WORKING_STATE)).
+          append(") AND cq_database=").append(quoteString(rqst.getDbname())).
+          append(" AND cq_table=").append(quoteString(rqst.getTablename())).append(" AND ");
+        if(rqst.getPartitionname() == null) {
+          sb.append("cq_partition is null");
+        }
+        else {
+          sb.append("cq_partition=").append(quoteString(rqst.getPartitionname()));
+        }
+
+        LOG.debug("Going to execute query <" + sb.toString() + ">");
+        ResultSet rs = stmt.executeQuery(sb.toString());
+        if(rs.next()) {
+          long enqueuedId = rs.getLong(1);
+          String state = compactorStateToResponse(rs.getString(2).charAt(0));
+          LOG.info("Ignoring request to compact " + rqst.getDbname() + "/" + rqst.getTablename() +
+            "/" + rqst.getPartitionname() + " since it is already " + quoteString(state) +
+            " with id=" + enqueuedId);
+          return new CompactionResponse(enqueuedId, state, false);
+        }
+        close(rs);
+        StringBuilder buf = new StringBuilder("insert into COMPACTION_QUEUE (cq_id, cq_database, " +
+          "cq_table, ");
+        String partName = rqst.getPartitionname();
+        if (partName != null) buf.append("cq_partition, ");
+        buf.append("cq_state, cq_type");
+        if (rqst.getProperties() != null) {
+          buf.append(", cq_tblproperties");
+        }
+        if (rqst.getRunas() != null) buf.append(", cq_run_as");
+        buf.append(") values (");
+        buf.append(id);
+        buf.append(", '");
+        buf.append(rqst.getDbname());
+        buf.append("', '");
+        buf.append(rqst.getTablename());
+        buf.append("', '");
+        if (partName != null) {
+          buf.append(partName);
+          buf.append("', '");
+        }
+        buf.append(INITIATED_STATE);
+        buf.append("', '");
+        switch (rqst.getType()) {
+          case MAJOR:
+            buf.append(MAJOR_TYPE);
+            break;
+
+          case MINOR:
+            buf.append(MINOR_TYPE);
+            break;
+
+          default:
+            LOG.debug("Going to rollback");
+            dbConn.rollback();
+            throw new MetaException("Unexpected compaction type " + rqst.getType().toString());
+        }
+        if (rqst.getProperties() != null) {
+          buf.append("', '");
+          buf.append(new StringableMap(rqst.getProperties()).toString());
+        }
+        if (rqst.getRunas() != null) {
+          buf.append("', '");
+          buf.append(rqst.getRunas());
+        }
+        buf.append("')");
+        String s = buf.toString();
+        LOG.debug("Going to execute update <" + s + ">");
+        stmt.executeUpdate(s);
+        LOG.debug("Going to commit");
+        dbConn.commit();
+        return new CompactionResponse(id, INITIATED_RESPONSE, true);
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "compact(" + rqst + ")");
+        throw new MetaException("Unable to select from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+        if(handle != null) {
+          handle.releaseLocks();
+        }
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      return compact(rqst);
+    }
+  }
+
+  private static String compactorStateToResponse(char s) {
+    switch (s) {
+      case INITIATED_STATE: return INITIATED_RESPONSE;
+      case WORKING_STATE: return WORKING_RESPONSE;
+      case READY_FOR_CLEANING: return CLEANING_RESPONSE;
+      case FAILED_STATE: return FAILED_RESPONSE;
+      case SUCCEEDED_STATE: return SUCCEEDED_RESPONSE;
+      case ATTEMPTED_STATE: return ATTEMPTED_RESPONSE;
+      default:
+        return Character.toString(s);
+    }
+  }
+  @RetrySemantics.ReadOnly
+  public ShowCompactResponse showCompact(ShowCompactRequest rqst) throws MetaException {
+    ShowCompactResponse response = new ShowCompactResponse(new ArrayList<>());
+    Connection dbConn = null;
+    Statement stmt = null;
+    try {
+      try {
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        String s = "select cq_database, cq_table, cq_partition, cq_state, cq_type, cq_worker_id, " +
+          //-1 because 'null' literal doesn't work for all DBs...
+          "cq_start, -1 cc_end, cq_run_as, cq_hadoop_job_id, cq_id from COMPACTION_QUEUE union all " +
+          "select cc_database, cc_table, cc_partition, cc_state, cc_type, cc_worker_id, " +
+          "cc_start, cc_end, cc_run_as, cc_hadoop_job_id, cc_id from COMPLETED_COMPACTIONS";
+        //what I want is order by cc_end desc, cc_start asc (but derby has a bug https://issues.apache.org/jira/browse/DERBY-6013)
+        //to sort so that currently running jobs are at the end of the list (bottom of screen)
+        //and currently running ones are in sorted by start time
+        //w/o order by likely currently running compactions will be first (LHS of Union)
+        LOG.debug("Going to execute query <" + s + ">");
+        ResultSet rs = stmt.executeQuery(s);
+        while (rs.next()) {
+          ShowCompactResponseElement e = new ShowCompactResponseElement();
+          e.setDbname(rs.getString(1));
+          e.setTablename(rs.getString(2));
+          e.setPartitionname(rs.getString(3));
+          e.setState(compactorStateToResponse(rs.getString(4).charAt(0)));
+          switch (rs.getString(5).charAt(0)) {
+            case MAJOR_TYPE: e.setType(CompactionType.MAJOR); break;
+            case MINOR_TYPE: e.setType(CompactionType.MINOR); break;
+            default:
+              //do nothing to handle RU/D if we add another status
+          }
+          e.setWorkerid(rs.getString(6));
+          long start = rs.getLong(7);
+          if(!rs.wasNull()) {
+            e.setStart(start);
+          }
+          long endTime = rs.getLong(8);
+          if(endTime != -1) {
+            e.setEndTime(endTime);
+          }
+          e.setRunAs(rs.getString(9));
+          e.setHadoopJobId(rs.getString(10));
+          e.setId(rs.getLong(11));
+          response.addToCompacts(e);
+        }
+        LOG.debug("Going to rollback");
+        dbConn.rollback();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "showCompact(" + rqst + ")");
+        throw new MetaException("Unable to select from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+      return response;
+    } catch (RetryException e) {
+      return showCompact(rqst);
+    }
+  }
+
+  private static void shouldNeverHappen(long txnid) {
+    throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid));
+  }
+  private static void shouldNeverHappen(long txnid, long extLockId, long intLockId) {
+    throw new RuntimeException("This should never happen: " + JavaUtils.txnIdToString(txnid) + " "
+      + JavaUtils.lockIdToString(extLockId) + " " + intLockId);
+  }
+
+  /**
+   * Retry-by-caller note:
+   * This may be retried after dbConn.commit.  At worst, it will create duplicate entries in
+   * TXN_COMPONENTS which won't affect anything.  See more comments in {@link #commitTxn(CommitTxnRequest)}
+   */
+  @Override
+  @RetrySemantics.SafeToRetry
+  public void addDynamicPartitions(AddDynamicPartitions rqst)
+      throws NoSuchTxnException,  TxnAbortedException, MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet lockHandle = null;
+    ResultSet rs = null;
+    try {
+      try {
+        lockInternal();
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        lockHandle = lockTransactionRecord(stmt, rqst.getTxnid(), TXN_OPEN);
+        if(lockHandle == null) {
+          //ensures txn is still there and in expected state
+          ensureValidTxn(dbConn, rqst.getTxnid(), stmt);
+          shouldNeverHappen(rqst.getTxnid());
+        }
+        //for RU this may be null so we should default it to 'u' which is most restrictive
+        OpertaionType ot = OpertaionType.UPDATE;
+        if(rqst.isSetOperationType()) {
+          ot = OpertaionType.fromDataOperationType(rqst.getOperationType());
+        }
+        List<String> rows = new ArrayList<>();
+        for (String partName : rqst.getPartitionnames()) {
+          rows.add(rqst.getTxnid() + "," + quoteString(rqst.getDbname()) + "," + quoteString(rqst.getTablename()) +
+            "," + quoteString(partName) + "," + quoteChar(ot.sqlConst));
+        }
+        int modCount = 0;
+        //record partitions that were written to
+        List<String> queries = sqlGenerator.createInsertValuesStmt(
+          "TXN_COMPONENTS (tc_txnid, tc_database, tc_table, tc_partition, tc_operation_type)", rows);
+        for(String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          modCount = stmt.executeUpdate(query);
+        }
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "addDynamicPartitions(" + rqst + ")");
+        throw new MetaException("Unable to insert into from transaction database " +
+          StringUtils.stringifyException(e));
+      } finally {
+        close(lockHandle, stmt, dbConn);
+        unlockInternal();
+      }
+    } catch (RetryException e) {
+      addDynamicPartitions(rqst);
+    }
+  }
+
+  /**
+   * Clean up corresponding records in metastore tables when corresponding object is dropped,
+   * specifically: TXN_COMPONENTS, COMPLETED_TXN_COMPONENTS, COMPACTION_QUEUE, COMPLETED_COMPACTIONS
+   * Retry-by-caller note: this is only idempotent assuming it's only called by dropTable/Db/etc
+   * operations.
+   */
+  @Override
+  @RetrySemantics.Idempotent
+  public void cleanupRecords(HiveObjectType type, Database db, Table table,
+                             Iterator<Partition> partitionIterator) throws MetaException {
+    try {
+      Connection dbConn = null;
+      Statement stmt = null;
+
+      try {
+        String dbName;
+        String tblName;
+        dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+        stmt = dbConn.createStatement();
+        List<String> queries = new ArrayList<>();
+        StringBuilder buff = new StringBuilder();
+
+        switch (type) {
+          case DATABASE:
+            dbName = db.getName();
+
+            buff.append("delete from TXN_COMPONENTS where tc_database='");
+            buff.append(dbName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
+            buff.append(dbName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPACTION_QUEUE where cq_database='");
+            buff.append(dbName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
+            buff.append(dbName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            break;
+          case TABLE:
+            dbName = table.getDbName();
+            tblName = table.getTableName();
+
+            buff.append("delete from TXN_COMPONENTS where tc_database='");
+            buff.append(dbName);
+            buff.append("' and tc_table='");
+            buff.append(tblName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
+            buff.append(dbName);
+            buff.append("' and ctc_table='");
+            buff.append(tblName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPACTION_QUEUE where cq_database='");
+            buff.append(dbName);
+            buff.append("' and cq_table='");
+            buff.append(tblName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            buff.setLength(0);
+            buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
+            buff.append(dbName);
+            buff.append("' and cc_table='");
+            buff.append(tblName);
+            buff.append("'");
+            queries.add(buff.toString());
+
+            break;
+          case PARTITION:
+            dbName = table.getDbName();
+            tblName = table.getTableName();
+            List<FieldSchema> partCols = table.getPartitionKeys();  // partition columns
+            List<String> partVals;                                  // partition values
+            String partName;
+
+            while (partitionIterator.hasNext()) {
+              Partition p = partitionIterator.next();
+              partVals = p.getValues();
+              partName = Warehouse.makePartName(partCols, partVals);
+
+              buff.append("delete from TXN_COMPONENTS where tc_database='");
+              buff.append(dbName);
+              buff.append("' and tc_table='");
+              buff.append(tblName);
+              buff.append("' and tc_partition='");
+              buff.append(partName);
+              buff.append("'");
+              queries.add(buff.toString());
+
+              buff.setLength(0);
+              buff.append("delete from COMPLETED_TXN_COMPONENTS where ctc_database='");
+              buff.append(dbName);
+              buff.append("' and ctc_table='");
+              buff.append(tblName);
+              buff.append("' and ctc_partition='");
+              buff.append(partName);
+              buff.append("'");
+              queries.add(buff.toString());
+
+              buff.setLength(0);
+              buff.append("delete from COMPACTION_QUEUE where cq_database='");
+              buff.append(dbName);
+              buff.append("' and cq_table='");
+              buff.append(tblName);
+              buff.append("' and cq_partition='");
+              buff.append(partName);
+              buff.append("'");
+              queries.add(buff.toString());
+
+              buff.setLength(0);
+              buff.append("delete from COMPLETED_COMPACTIONS where cc_database='");
+              buff.append(dbName);
+              buff.append("' and cc_table='");
+              buff.append(tblName);
+              buff.append("' and cc_partition='");
+              buff.append(partName);
+              buff.append("'");
+              queries.add(buff.toString());
+            }
+
+            break;
+          default:
+            throw new MetaException("Invalid object type for cleanup: " + type);
+        }
+
+        for (String query : queries) {
+          LOG.debug("Going to execute update <" + query + ">");
+          stmt.executeUpdate(query);
+        }
+
+        LOG.debug("Going to commit");
+        dbConn.commit();
+      } catch (SQLException e) {
+        LOG.debug("Going to rollback");
+        rollbackDBConn(dbConn);
+        checkRetryable(dbConn, e, "cleanupRecords");
+        if (e.getMessage().contains("does not exist")) {
+          LOG.warn("Cannot perform cleanup since metastore table does not exist");
+        } else {
+          throw new MetaException("Unable to clean up " + StringUtils.stringifyException(e));
+        }
+      } finally {
+        closeStmt(stmt);
+        closeDbConn(dbConn);
+      }
+    } catch (RetryException e) {
+      cleanupRecords(type, db, table, partitionIterator);
+    }
+  }
+
+  /**
+   * For testing only, do not use.
+   */
+  @VisibleForTesting
+  public int numLocksInLockTable() throws SQLException, MetaException {
+    Connection dbConn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+    try {
+      dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
+      stmt = dbConn.createStatement();
+      String s = "select count(*) from HIVE_LOCKS";
+      LOG.debug("Going to execute query <" + s + ">");
+      rs = stmt.executeQuery(s);
+      rs.next();
+      int rc = rs.getInt(1);
+      // Necessary to clean up the transaction in the db.
+      dbConn.rollback();
+      return rc;
+    } finally {
+      close(rs, stmt, dbConn);
+    }
+  }
+
+  /**
+   * For testing only, do not use.
+   */
+  public long setTimeout(long milliseconds) {
+    long previous_timeout = timeout;
+    timeout = milliseconds;
+    return previous_timeout;
+  }
+
+  protected class RetryException extends Exception {
+
+  }
+
+  Connection getDbConn(int isolationLevel) throws SQLException {
+    return getDbConn(isolationLevel, connPool);
+  }
+  private Connection getDbConn(int isolationLevel, DataSource connPool) throws SQLException {
+    int rc = doRetryOnConnPool ? 10 : 1;
+    Connection dbConn = null;
+    while (true) {
+      try {
+        dbConn = connPool.getConnection();
+        dbConn.setAutoCommit(false);
+        dbConn.setTransactionIsolation(isolationLevel);
+        return dbConn;
+      } catch (SQLException e){
+        closeDbConn(dbConn);
+        if ((--rc) <= 0) throw e;
+        LOG.error("There is a problem with a connection from th

<TRUNCATED>

[02/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
new file mode 100644
index 0000000..674084d
--- /dev/null
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.txn;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.DatabaseProduct;
+import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests for TxnUtils
+ */
+public class TestTxnUtils {
+  private Configuration conf;
+
+  public TestTxnUtils() throws Exception {
+  }
+
+  @Test
+  public void testBuildQueryWithINClause() throws Exception {
+    List<String> queries = new ArrayList<>();
+
+    StringBuilder prefix = new StringBuilder();
+    StringBuilder suffix = new StringBuilder();
+
+    // Note, this is a "real" query that depends on one of the metastore tables
+    prefix.append("select count(*) from TXNS where ");
+    suffix.append(" and TXN_STATE = 'o'");
+
+    // Case 1 - Max in list members: 10; Max query string length: 1KB
+    //          The first query happens to have 2 full batches.
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 1);
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 10);
+    List<Long> inList = new ArrayList<>();
+    for (long i = 1; i <= 200; i++) {
+      inList.add(i);
+    }
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 2 - Max in list members: 10; Max query string length: 1KB
+    //          The first query has 2 full batches, and the second query only has 1 batch which only contains 1 member
+    queries.clear();
+    inList.add((long)201);
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(2, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 3.1 - Max in list members: 1000, Max query string length: 1KB, and exact 1000 members in a single IN clause
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 1);
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
+    queries.clear();
+    for (long i = 202; i <= 1000; i++) {
+      inList.add(i);
+    }
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 3.2 - Max in list members: 1000, Max query string length: 10KB, and exact 1000 members in a single IN clause
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 10);
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 3.3 - Now with 2000 entries, try the above settings
+    for (long i = 1001; i <= 2000; i++) {
+      inList.add(i);
+    }
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 1);
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(2, queries.size());
+    runAgainstDerby(queries);
+    MetastoreConf.setLongVar(conf, ConfVars.DIRECT_SQL_MAX_QUERY_LENGTH, 10);
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(1, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 4 - Max in list members: 1000; Max query string length: 10KB
+    queries.clear();
+    for (long i = 2001; i <= 4321; i++) {
+      inList.add(i);
+    }
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
+    Assert.assertEquals(3, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 5 - NOT IN list
+    queries.clear();
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, true);
+    Assert.assertEquals(3, queries.size());
+    runAgainstDerby(queries);
+
+    // Case 6 - No parenthesis
+    queries.clear();
+    suffix.setLength(0);
+    suffix.append("");
+    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", false, false);
+    Assert.assertEquals(3, queries.size());
+    runAgainstDerby(queries);
+  }
+
+  /** Verify queries can run against Derby DB.
+   *  As long as Derby doesn't complain, we assume the query is syntactically/semantically correct.
+   */
+  private void runAgainstDerby(List<String> queries) throws Exception {
+    Connection conn = null;
+    Statement stmt = null;
+    ResultSet rs = null;
+
+    try {
+      conn = TxnDbUtil.getConnection(conf);
+      stmt = conn.createStatement();
+      for (String query : queries) {
+        rs = stmt.executeQuery(query);
+        Assert.assertTrue("The query is not valid", rs.next());
+      }
+    } finally {
+      TxnDbUtil.closeResources(conn, stmt, rs);
+    }
+  }
+  @Test
+  public void testSQLGenerator() throws Exception {
+    //teseted on Oracle Database 11g Express Edition Release 11.2.0.2.0 - 64bit Production
+    SQLGenerator sqlGenerator =
+      new SQLGenerator(DatabaseProduct.ORACLE, conf);
+    List<String> rows = new ArrayList<>();
+    rows.add("'yellow', 1");
+    List<String> sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 1, sql.size());
+    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1)", sql.get(0));
+    rows.add("'red', 2");
+    rows.add("'orange', 3");
+    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 1, sql.size());
+    
+    Assert.assertEquals("Wrong stmt", 
+      "insert all into colors(name, category) values('yellow', 1) into colors(name, category) values('red', 2) into colors(name, category) values('orange', 3) select * from dual", sql.get(0));
+    for(int i = 0; i < MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE); i++) {
+      rows.add("\'G\'," + i);
+    }
+    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 2, sql.size());
+    Assert.assertEquals("Wrong stmt", "insert all into colors(name, category) values('yellow', 1) into colors(name, category) values('red', 2) into colors(name, category) values('orange', 3) into colors(name, category) values('G',0) into colors(name, category) values('G',1) into colors(name, category) values('G',2) into colors(name, category) values('G',3) into colors(name, category) values('G',4) into colors(name, category) values('G',5) into colors(name, category) values('G',6) into colors(name, category) values('G',7) into colors(name, category) values('G',8) into colors(name, category) values('G',9) into colors(name, category) values('G',10) into colors(name, category) values('G',11) into colors(name, category) values('G',12) into colors(name, category) values('G',13) into colors(name, category) values('G',14) into colors(name, category) values('G',15) into colors(name, category) values('G',16) into colors(name, category) values('G',17) into colors(name, category) values('G',18)
  into colors(name, category) values('G',19) into colors(name, category) values('G',20) into colors(name, category) values('G',21) into colors(name, category) values('G',22) into colors(name, category) values('G',23) into colors(name, category) values('G',24) into colors(name, category) values('G',25) into colors(name, category) values('G',26) into colors(name, category) values('G',27) into colors(name, category) values('G',28) into colors(name, category) values('G',29) into colors(name, category) values('G',30) into colors(name, category) values('G',31) into colors(name, category) values('G',32) into colors(name, category) values('G',33) into colors(name, category) values('G',34) into colors(name, category) values('G',35) into colors(name, category) values('G',36) into colors(name, category) values('G',37) into colors(name, category) values('G',38) into colors(name, category) values('G',39) into colors(name, category) values('G',40) into colors(name, category) values('G',41) into co
 lors(name, category) values('G',42) into colors(name, category) values('G',43) into colors(name, category) values('G',44) into colors(name, category) values('G',45) into colors(name, category) values('G',46) into colors(name, category) values('G',47) into colors(name, category) values('G',48) into colors(name, category) values('G',49) into colors(name, category) values('G',50) into colors(name, category) values('G',51) into colors(name, category) values('G',52) into colors(name, category) values('G',53) into colors(name, category) values('G',54) into colors(name, category) values('G',55) into colors(name, category) values('G',56) into colors(name, category) values('G',57) into colors(name, category) values('G',58) into colors(name, category) values('G',59) into colors(name, category) values('G',60) into colors(name, category) values('G',61) into colors(name, category) values('G',62) into colors(name, category) values('G',63) into colors(name, category) values('G',64) into colors(nam
 e, category) values('G',65) into colors(name, category) values('G',66) into colors(name, category) values('G',67) into colors(name, category) values('G',68) into colors(name, category) values('G',69) into colors(name, category) values('G',70) into colors(name, category) values('G',71) into colors(name, category) values('G',72) into colors(name, category) values('G',73) into colors(name, category) values('G',74) into colors(name, category) values('G',75) into colors(name, category) values('G',76) into colors(name, category) values('G',77) into colors(name, category) values('G',78) into colors(name, category) values('G',79) into colors(name, category) values('G',80) into colors(name, category) values('G',81) into colors(name, category) values('G',82) into colors(name, category) values('G',83) into colors(name, category) values('G',84) into colors(name, category) values('G',85) into colors(name, category) values('G',86) into colors(name, category) values('G',87) into colors(name, categ
 ory) values('G',88) into colors(name, category) values('G',89) into colors(name, category) values('G',90) into colors(name, category) values('G',91) into colors(name, category) values('G',92) into colors(name, category) values('G',93) into colors(name, category) values('G',94) into colors(name, category) values('G',95) into colors(name, category) values('G',96) into colors(name, category) values('G',97) into colors(name, category) values('G',98) into colors(name, category) values('G',99) into colors(name, category) values('G',100) into colors(name, category) values('G',101) into colors(name, category) values('G',102) into colors(name, category) values('G',103) into colors(name, category) values('G',104) into colors(name, category) values('G',105) into colors(name, category) values('G',106) into colors(name, category) values('G',107) into colors(name, category) values('G',108) into colors(name, category) values('G',109) into colors(name, category) values('G',110) into colors(name, ca
 tegory) values('G',111) into colors(name, category) values('G',112) into colors(name, category) values('G',113) into colors(name, category) values('G',114) into colors(name, category) values('G',115) into colors(name, category) values('G',116) into colors(name, category) values('G',117) into colors(name, category) values('G',118) into colors(name, category) values('G',119) into colors(name, category) values('G',120) into colors(name, category) values('G',121) into colors(name, category) values('G',122) into colors(name, category) values('G',123) into colors(name, category) values('G',124) into colors(name, category) values('G',125) into colors(name, category) values('G',126) into colors(name, category) values('G',127) into colors(name, category) values('G',128) into colors(name, category) values('G',129) into colors(name, category) values('G',130) into colors(name, category) values('G',131) into colors(name, category) values('G',132) into colors(name, category) values('G',133) into 
 colors(name, category) values('G',134) into colors(name, category) values('G',135) into colors(name, category) values('G',136) into colors(name, category) values('G',137) into colors(name, category) values('G',138) into colors(name, category) values('G',139) into colors(name, category) values('G',140) into colors(name, category) values('G',141) into colors(name, category) values('G',142) into colors(name, category) values('G',143) into colors(name, category) values('G',144) into colors(name, category) values('G',145) into colors(name, category) values('G',146) into colors(name, category) values('G',147) into colors(name, category) values('G',148) into colors(name, category) values('G',149) into colors(name, category) values('G',150) into colors(name, category) values('G',151) into colors(name, category) values('G',152) into colors(name, category) values('G',153) into colors(name, category) values('G',154) into colors(name, category) values('G',155) into colors(name, category) values
 ('G',156) into colors(name, category) values('G',157) into colors(name, category) values('G',158) into colors(name, category) values('G',159) into colors(name, category) values('G',160) into colors(name, category) values('G',161) into colors(name, category) values('G',162) into colors(name, category) values('G',163) into colors(name, category) values('G',164) into colors(name, category) values('G',165) into colors(name, category) values('G',166) into colors(name, category) values('G',167) into colors(name, category) values('G',168) into colors(name, category) values('G',169) into colors(name, category) values('G',170) into colors(name, category) values('G',171) into colors(name, category) values('G',172) into colors(name, category) values('G',173) into colors(name, category) values('G',174) into colors(name, category) values('G',175) into colors(name, category) values('G',176) into colors(name, category) values('G',177) into colors(name, category) values('G',178) into colors(name, c
 ategory) values('G',179) into colors(name, category) values('G',180) into colors(name, category) values('G',181) into colors(name, category) values('G',182) into colors(name, category) values('G',183) into colors(name, category) values('G',184) into colors(name, category) values('G',185) into colors(name, category) values('G',186) into colors(name, category) values('G',187) into colors(name, category) values('G',188) into colors(name, category) values('G',189) into colors(name, category) values('G',190) into colors(name, category) values('G',191) into colors(name, category) values('G',192) into colors(name, category) values('G',193) into colors(name, category) values('G',194) into colors(name, category) values('G',195) into colors(name, category) values('G',196) into colors(name, category) values('G',197) into colors(name, category) values('G',198) into colors(name, category) values('G',199) into colors(name, category) values('G',200) into colors(name, category) values('G',201) into
  colors(name, category) values('G',202) into colors(name, category) values('G',203) into colors(name, category) values('G',204) into colors(name, category) values('G',205) into colors(name, category) values('G',206) into colors(name, category) values('G',207) into colors(name, category) values('G',208) into colors(name, category) values('G',209) into colors(name, category) values('G',210) into colors(name, category) values('G',211) into colors(name, category) values('G',212) into colors(name, category) values('G',213) into colors(name, category) values('G',214) into colors(name, category) values('G',215) into colors(name, category) values('G',216) into colors(name, category) values('G',217) into colors(name, category) values('G',218) into colors(name, category) values('G',219) into colors(name, category) values('G',220) into colors(name, category) values('G',221) into colors(name, category) values('G',222) into colors(name, category) values('G',223) into colors(name, category) value
 s('G',224) into colors(name, category) values('G',225) into colors(name, category) values('G',226) into colors(name, category) values('G',227) into colors(name, category) values('G',228) into colors(name, category) values('G',229) into colors(name, category) values('G',230) into colors(name, category) values('G',231) into colors(name, category) values('G',232) into colors(name, category) values('G',233) into colors(name, category) values('G',234) into colors(name, category) values('G',235) into colors(name, category) values('G',236) into colors(name, category) values('G',237) into colors(name, category) values('G',238) into colors(name, category) values('G',239) into colors(name, category) values('G',240) into colors(name, category) values('G',241) into colors(name, category) values('G',242) into colors(name, category) values('G',243) into colors(name, category) values('G',244) into colors(name, category) values('G',245) into colors(name, category) values('G',246) into colors(name, 
 category) values('G',247) into colors(name, category) values('G',248) into colors(name, category) values('G',249) into colors(name, category) values('G',250) into colors(name, category) values('G',251) into colors(name, category) values('G',252) into colors(name, category) values('G',253) into colors(name, category) values('G',254) into colors(name, category) values('G',255) into colors(name, category) values('G',256) into colors(name, category) values('G',257) into colors(name, category) values('G',258) into colors(name, category) values('G',259) into colors(name, category) values('G',260) into colors(name, category) values('G',261) into colors(name, category) values('G',262) into colors(name, category) values('G',263) into colors(name, category) values('G',264) into colors(name, category) values('G',265) into colors(name, category) values('G',266) into colors(name, category) values('G',267) into colors(name, category) values('G',268) into colors(name, category) values('G',269) int
 o colors(name, category) values('G',270) into colors(name, category) values('G',271) into colors(name, category) values('G',272) into colors(name, category) values('G',273) into colors(name, category) values('G',274) into colors(name, category) values('G',275) into colors(name, category) values('G',276) into colors(name, category) values('G',277) into colors(name, category) values('G',278) into colors(name, category) values('G',279) into colors(name, category) values('G',280) into colors(name, category) values('G',281) into colors(name, category) values('G',282) into colors(name, category) values('G',283) into colors(name, category) values('G',284) into colors(name, category) values('G',285) into colors(name, category) values('G',286) into colors(name, category) values('G',287) into colors(name, category) values('G',288) into colors(name, category) values('G',289) into colors(name, category) values('G',290) into colors(name, category) values('G',291) into colors(name, category) valu
 es('G',292) into colors(name, category) values('G',293) into colors(name, category) values('G',294) into colors(name, category) values('G',295) into colors(name, category) values('G',296) into colors(name, category) values('G',297) into colors(name, category) values('G',298) into colors(name, category) values('G',299) into colors(name, category) values('G',300) into colors(name, category) values('G',301) into colors(name, category) values('G',302) into colors(name, category) values('G',303) into colors(name, category) values('G',304) into colors(name, category) values('G',305) into colors(name, category) values('G',306) into colors(name, category) values('G',307) into colors(name, category) values('G',308) into colors(name, category) values('G',309) into colors(name, category) values('G',310) into colors(name, category) values('G',311) into colors(name, category) values('G',312) into colors(name, category) values('G',313) into colors(name, category) values('G',314) into colors(name,
  category) values('G',315) into colors(name, category) values('G',316) into colors(name, category) values('G',317) into colors(name, category) values('G',318) into colors(name, category) values('G',319) into colors(name, category) values('G',320) into colors(name, category) values('G',321) into colors(name, category) values('G',322) into colors(name, category) values('G',323) into colors(name, category) values('G',324) into colors(name, category) values('G',325) into colors(name, category) values('G',326) into colors(name, category) values('G',327) into colors(name, category) values('G',328) into colors(name, category) values('G',329) into colors(name, category) values('G',330) into colors(name, category) values('G',331) into colors(name, category) values('G',332) into colors(name, category) values('G',333) into colors(name, category) values('G',334) into colors(name, category) values('G',335) into colors(name, category) values('G',336) into colors(name, category) values('G',337) in
 to colors(name, category) values('G',338) into colors(name, category) values('G',339) into colors(name, category) values('G',340) into colors(name, category) values('G',341) into colors(name, category) values('G',342) into colors(name, category) values('G',343) into colors(name, category) values('G',344) into colors(name, category) values('G',345) into colors(name, category) values('G',346) into colors(name, category) values('G',347) into colors(name, category) values('G',348) into colors(name, category) values('G',349) into colors(name, category) values('G',350) into colors(name, category) values('G',351) into colors(name, category) values('G',352) into colors(name, category) values('G',353) into colors(name, category) values('G',354) into colors(name, category) values('G',355) into colors(name, category) values('G',356) into colors(name, category) values('G',357) into colors(name, category) values('G',358) into colors(name, category) values('G',359) into colors(name, category) val
 ues('G',360) into colors(name, category) values('G',361) into colors(name, category) values('G',362) into colors(name, category) values('G',363) into colors(name, category) values('G',364) into colors(name, category) values('G',365) into colors(name, category) values('G',366) into colors(name, category) values('G',367) into colors(name, category) values('G',368) into colors(name, category) values('G',369) into colors(name, category) values('G',370) into colors(name, category) values('G',371) into colors(name, category) values('G',372) into colors(name, category) values('G',373) into colors(name, category) values('G',374) into colors(name, category) values('G',375) into colors(name, category) values('G',376) into colors(name, category) values('G',377) into colors(name, category) values('G',378) into colors(name, category) values('G',379) into colors(name, category) values('G',380) into colors(name, category) values('G',381) into colors(name, category) values('G',382) into colors(name
 , category) values('G',383) into colors(name, category) values('G',384) into colors(name, category) values('G',385) into colors(name, category) values('G',386) into colors(name, category) values('G',387) into colors(name, category) values('G',388) into colors(name, category) values('G',389) into colors(name, category) values('G',390) into colors(name, category) values('G',391) into colors(name, category) values('G',392) into colors(name, category) values('G',393) into colors(name, category) values('G',394) into colors(name, category) values('G',395) into colors(name, category) values('G',396) into colors(name, category) values('G',397) into colors(name, category) values('G',398) into colors(name, category) values('G',399) into colors(name, category) values('G',400) into colors(name, category) values('G',401) into colors(name, category) values('G',402) into colors(name, category) values('G',403) into colors(name, category) values('G',404) into colors(name, category) values('G',405) i
 nto colors(name, category) values('G',406) into colors(name, category) values('G',407) into colors(name, category) values('G',408) into colors(name, category) values('G',409) into colors(name, category) values('G',410) into colors(name, category) values('G',411) into colors(name, category) values('G',412) into colors(name, category) values('G',413) into colors(name, category) values('G',414) into colors(name, category) values('G',415) into colors(name, category) values('G',416) into colors(name, category) values('G',417) into colors(name, category) values('G',418) into colors(name, category) values('G',419) into colors(name, category) values('G',420) into colors(name, category) values('G',421) into colors(name, category) values('G',422) into colors(name, category) values('G',423) into colors(name, category) values('G',424) into colors(name, category) values('G',425) into colors(name, category) values('G',426) into colors(name, category) values('G',427) into colors(name, category) va
 lues('G',428) into colors(name, category) values('G',429) into colors(name, category) values('G',430) into colors(name, category) values('G',431) into colors(name, category) values('G',432) into colors(name, category) values('G',433) into colors(name, category) values('G',434) into colors(name, category) values('G',435) into colors(name, category) values('G',436) into colors(name, category) values('G',437) into colors(name, category) values('G',438) into colors(name, category) values('G',439) into colors(name, category) values('G',440) into colors(name, category) values('G',441) into colors(name, category) values('G',442) into colors(name, category) values('G',443) into colors(name, category) values('G',444) into colors(name, category) values('G',445) into colors(name, category) values('G',446) into colors(name, category) values('G',447) into colors(name, category) values('G',448) into colors(name, category) values('G',449) into colors(name, category) values('G',450) into colors(nam
 e, category) values('G',451) into colors(name, category) values('G',452) into colors(name, category) values('G',453) into colors(name, category) values('G',454) into colors(name, category) values('G',455) into colors(name, category) values('G',456) into colors(name, category) values('G',457) into colors(name, category) values('G',458) into colors(name, category) values('G',459) into colors(name, category) values('G',460) into colors(name, category) values('G',461) into colors(name, category) values('G',462) into colors(name, category) values('G',463) into colors(name, category) values('G',464) into colors(name, category) values('G',465) into colors(name, category) values('G',466) into colors(name, category) values('G',467) into colors(name, category) values('G',468) into colors(name, category) values('G',469) into colors(name, category) values('G',470) into colors(name, category) values('G',471) into colors(name, category) values('G',472) into colors(name, category) values('G',473) 
 into colors(name, category) values('G',474) into colors(name, category) values('G',475) into colors(name, category) values('G',476) into colors(name, category) values('G',477) into colors(name, category) values('G',478) into colors(name, category) values('G',479) into colors(name, category) values('G',480) into colors(name, category) values('G',481) into colors(name, category) values('G',482) into colors(name, category) values('G',483) into colors(name, category) values('G',484) into colors(name, category) values('G',485) into colors(name, category) values('G',486) into colors(name, category) values('G',487) into colors(name, category) values('G',488) into colors(name, category) values('G',489) into colors(name, category) values('G',490) into colors(name, category) values('G',491) into colors(name, category) values('G',492) into colors(name, category) values('G',493) into colors(name, category) values('G',494) into colors(name, category) values('G',495) into colors(name, category) v
 alues('G',496) into colors(name, category) values('G',497) into colors(name, category) values('G',498) into colors(name, category) values('G',499) into colors(name, category) values('G',500) into colors(name, category) values('G',501) into colors(name, category) values('G',502) into colors(name, category) values('G',503) into colors(name, category) values('G',504) into colors(name, category) values('G',505) into colors(name, category) values('G',506) into colors(name, category) values('G',507) into colors(name, category) values('G',508) into colors(name, category) values('G',509) into colors(name, category) values('G',510) into colors(name, category) values('G',511) into colors(name, category) values('G',512) into colors(name, category) values('G',513) into colors(name, category) values('G',514) into colors(name, category) values('G',515) into colors(name, category) values('G',516) into colors(name, category) values('G',517) into colors(name, category) values('G',518) into colors(na
 me, category) values('G',519) into colors(name, category) values('G',520) into colors(name, category) values('G',521) into colors(name, category) values('G',522) into colors(name, category) values('G',523) into colors(name, category) values('G',524) into colors(name, category) values('G',525) into colors(name, category) values('G',526) into colors(name, category) values('G',527) into colors(name, category) values('G',528) into colors(name, category) values('G',529) into colors(name, category) values('G',530) into colors(name, category) values('G',531) into colors(name, category) values('G',532) into colors(name, category) values('G',533) into colors(name, category) values('G',534) into colors(name, category) values('G',535) into colors(name, category) values('G',536) into colors(name, category) values('G',537) into colors(name, category) values('G',538) into colors(name, category) values('G',539) into colors(name, category) values('G',540) into colors(name, category) values('G',541)
  into colors(name, category) values('G',542) into colors(name, category) values('G',543) into colors(name, category) values('G',544) into colors(name, category) values('G',545) into colors(name, category) values('G',546) into colors(name, category) values('G',547) into colors(name, category) values('G',548) into colors(name, category) values('G',549) into colors(name, category) values('G',550) into colors(name, category) values('G',551) into colors(name, category) values('G',552) into colors(name, category) values('G',553) into colors(name, category) values('G',554) into colors(name, category) values('G',555) into colors(name, category) values('G',556) into colors(name, category) values('G',557) into colors(name, category) values('G',558) into colors(name, category) values('G',559) into colors(name, category) values('G',560) into colors(name, category) values('G',561) into colors(name, category) values('G',562) into colors(name, category) values('G',563) into colors(name, category) 
 values('G',564) into colors(name, category) values('G',565) into colors(name, category) values('G',566) into colors(name, category) values('G',567) into colors(name, category) values('G',568) into colors(name, category) values('G',569) into colors(name, category) values('G',570) into colors(name, category) values('G',571) into colors(name, category) values('G',572) into colors(name, category) values('G',573) into colors(name, category) values('G',574) into colors(name, category) values('G',575) into colors(name, category) values('G',576) into colors(name, category) values('G',577) into colors(name, category) values('G',578) into colors(name, category) values('G',579) into colors(name, category) values('G',580) into colors(name, category) values('G',581) into colors(name, category) values('G',582) into colors(name, category) values('G',583) into colors(name, category) values('G',584) into colors(name, category) values('G',585) into colors(name, category) values('G',586) into colors(n
 ame, category) values('G',587) into colors(name, category) values('G',588) into colors(name, category) values('G',589) into colors(name, category) values('G',590) into colors(name, category) values('G',591) into colors(name, category) values('G',592) into colors(name, category) values('G',593) into colors(name, category) values('G',594) into colors(name, category) values('G',595) into colors(name, category) values('G',596) into colors(name, category) values('G',597) into colors(name, category) values('G',598) into colors(name, category) values('G',599) into colors(name, category) values('G',600) into colors(name, category) values('G',601) into colors(name, category) values('G',602) into colors(name, category) values('G',603) into colors(name, category) values('G',604) into colors(name, category) values('G',605) into colors(name, category) values('G',606) into colors(name, category) values('G',607) into colors(name, category) values('G',608) into colors(name, category) values('G',609
 ) into colors(name, category) values('G',610) into colors(name, category) values('G',611) into colors(name, category) values('G',612) into colors(name, category) values('G',613) into colors(name, category) values('G',614) into colors(name, category) values('G',615) into colors(name, category) values('G',616) into colors(name, category) values('G',617) into colors(name, category) values('G',618) into colors(name, category) values('G',619) into colors(name, category) values('G',620) into colors(name, category) values('G',621) into colors(name, category) values('G',622) into colors(name, category) values('G',623) into colors(name, category) values('G',624) into colors(name, category) values('G',625) into colors(name, category) values('G',626) into colors(name, category) values('G',627) into colors(name, category) values('G',628) into colors(name, category) values('G',629) into colors(name, category) values('G',630) into colors(name, category) values('G',631) into colors(name, category)
  values('G',632) into colors(name, category) values('G',633) into colors(name, category) values('G',634) into colors(name, category) values('G',635) into colors(name, category) values('G',636) into colors(name, category) values('G',637) into colors(name, category) values('G',638) into colors(name, category) values('G',639) into colors(name, category) values('G',640) into colors(name, category) values('G',641) into colors(name, category) values('G',642) into colors(name, category) values('G',643) into colors(name, category) values('G',644) into colors(name, category) values('G',645) into colors(name, category) values('G',646) into colors(name, category) values('G',647) into colors(name, category) values('G',648) into colors(name, category) values('G',649) into colors(name, category) values('G',650) into colors(name, category) values('G',651) into colors(name, category) values('G',652) into colors(name, category) values('G',653) into colors(name, category) values('G',654) into colors(
 name, category) values('G',655) into colors(name, category) values('G',656) into colors(name, category) values('G',657) into colors(name, category) values('G',658) into colors(name, category) values('G',659) into colors(name, category) values('G',660) into colors(name, category) values('G',661) into colors(name, category) values('G',662) into colors(name, category) values('G',663) into colors(name, category) values('G',664) into colors(name, category) values('G',665) into colors(name, category) values('G',666) into colors(name, category) values('G',667) into colors(name, category) values('G',668) into colors(name, category) values('G',669) into colors(name, category) values('G',670) into colors(name, category) values('G',671) into colors(name, category) values('G',672) into colors(name, category) values('G',673) into colors(name, category) values('G',674) into colors(name, category) values('G',675) into colors(name, category) values('G',676) into colors(name, category) values('G',67
 7) into colors(name, category) values('G',678) into colors(name, category) values('G',679) into colors(name, category) values('G',680) into colors(name, category) values('G',681) into colors(name, category) values('G',682) into colors(name, category) values('G',683) into colors(name, category) values('G',684) into colors(name, category) values('G',685) into colors(name, category) values('G',686) into colors(name, category) values('G',687) into colors(name, category) values('G',688) into colors(name, category) values('G',689) into colors(name, category) values('G',690) into colors(name, category) values('G',691) into colors(name, category) values('G',692) into colors(name, category) values('G',693) into colors(name, category) values('G',694) into colors(name, category) values('G',695) into colors(name, category) values('G',696) into colors(name, category) values('G',697) into colors(name, category) values('G',698) into colors(name, category) values('G',699) into colors(name, category
 ) values('G',700) into colors(name, category) values('G',701) into colors(name, category) values('G',702) into colors(name, category) values('G',703) into colors(name, category) values('G',704) into colors(name, category) values('G',705) into colors(name, category) values('G',706) into colors(name, category) values('G',707) into colors(name, category) values('G',708) into colors(name, category) values('G',709) into colors(name, category) values('G',710) into colors(name, category) values('G',711) into colors(name, category) values('G',712) into colors(name, category) values('G',713) into colors(name, category) values('G',714) into colors(name, category) values('G',715) into colors(name, category) values('G',716) into colors(name, category) values('G',717) into colors(name, category) values('G',718) into colors(name, category) values('G',719) into colors(name, category) values('G',720) into colors(name, category) values('G',721) into colors(name, category) values('G',722) into colors
 (name, category) values('G',723) into colors(name, category) values('G',724) into colors(name, category) values('G',725) into colors(name, category) values('G',726) into colors(name, category) values('G',727) into colors(name, category) values('G',728) into colors(name, category) values('G',729) into colors(name, category) values('G',730) into colors(name, category) values('G',731) into colors(name, category) values('G',732) into colors(name, category) values('G',733) into colors(name, category) values('G',734) into colors(name, category) values('G',735) into colors(name, category) values('G',736) into colors(name, category) values('G',737) into colors(name, category) values('G',738) into colors(name, category) values('G',739) into colors(name, category) values('G',740) into colors(name, category) values('G',741) into colors(name, category) values('G',742) into colors(name, category) values('G',743) into colors(name, category) values('G',744) into colors(name, category) values('G',7
 45) into colors(name, category) values('G',746) into colors(name, category) values('G',747) into colors(name, category) values('G',748) into colors(name, category) values('G',749) into colors(name, category) values('G',750) into colors(name, category) values('G',751) into colors(name, category) values('G',752) into colors(name, category) values('G',753) into colors(name, category) values('G',754) into colors(name, category) values('G',755) into colors(name, category) values('G',756) into colors(name, category) values('G',757) into colors(name, category) values('G',758) into colors(name, category) values('G',759) into colors(name, category) values('G',760) into colors(name, category) values('G',761) into colors(name, category) values('G',762) into colors(name, category) values('G',763) into colors(name, category) values('G',764) into colors(name, category) values('G',765) into colors(name, category) values('G',766) into colors(name, category) values('G',767) into colors(name, categor
 y) values('G',768) into colors(name, category) values('G',769) into colors(name, category) values('G',770) into colors(name, category) values('G',771) into colors(name, category) values('G',772) into colors(name, category) values('G',773) into colors(name, category) values('G',774) into colors(name, category) values('G',775) into colors(name, category) values('G',776) into colors(name, category) values('G',777) into colors(name, category) values('G',778) into colors(name, category) values('G',779) into colors(name, category) values('G',780) into colors(name, category) values('G',781) into colors(name, category) values('G',782) into colors(name, category) values('G',783) into colors(name, category) values('G',784) into colors(name, category) values('G',785) into colors(name, category) values('G',786) into colors(name, category) values('G',787) into colors(name, category) values('G',788) into colors(name, category) values('G',789) into colors(name, category) values('G',790) into color
 s(name, category) values('G',791) into colors(name, category) values('G',792) into colors(name, category) values('G',793) into colors(name, category) values('G',794) into colors(name, category) values('G',795) into colors(name, category) values('G',796) into colors(name, category) values('G',797) into colors(name, category) values('G',798) into colors(name, category) values('G',799) into colors(name, category) values('G',800) into colors(name, category) values('G',801) into colors(name, category) values('G',802) into colors(name, category) values('G',803) into colors(name, category) values('G',804) into colors(name, category) values('G',805) into colors(name, category) values('G',806) into colors(name, category) values('G',807) into colors(name, category) values('G',808) into colors(name, category) values('G',809) into colors(name, category) values('G',810) into colors(name, category) values('G',811) into colors(name, category) values('G',812) into colors(name, category) values('G',
 813) into colors(name, category) values('G',814) into colors(name, category) values('G',815) into colors(name, category) values('G',816) into colors(name, category) values('G',817) into colors(name, category) values('G',818) into colors(name, category) values('G',819) into colors(name, category) values('G',820) into colors(name, category) values('G',821) into colors(name, category) values('G',822) into colors(name, category) values('G',823) into colors(name, category) values('G',824) into colors(name, category) values('G',825) into colors(name, category) values('G',826) into colors(name, category) values('G',827) into colors(name, category) values('G',828) into colors(name, category) values('G',829) into colors(name, category) values('G',830) into colors(name, category) values('G',831) into colors(name, category) values('G',832) into colors(name, category) values('G',833) into colors(name, category) values('G',834) into colors(name, category) values('G',835) into colors(name, catego
 ry) values('G',836) into colors(name, category) values('G',837) into colors(name, category) values('G',838) into colors(name, category) values('G',839) into colors(name, category) values('G',840) into colors(name, category) values('G',841) into colors(name, category) values('G',842) into colors(name, category) values('G',843) into colors(name, category) values('G',844) into colors(name, category) values('G',845) into colors(name, category) values('G',846) into colors(name, category) values('G',847) into colors(name, category) values('G',848) into colors(name, category) values('G',849) into colors(name, category) values('G',850) into colors(name, category) values('G',851) into colors(name, category) values('G',852) into colors(name, category) values('G',853) into colors(name, category) values('G',854) into colors(name, category) values('G',855) into colors(name, category) values('G',856) into colors(name, category) values('G',857) into colors(name, category) values('G',858) into colo
 rs(name, category) values('G',859) into colors(name, category) values('G',860) into colors(name, category) values('G',861) into colors(name, category) values('G',862) into colors(name, category) values('G',863) into colors(name, category) values('G',864) into colors(name, category) values('G',865) into colors(name, category) values('G',866) into colors(name, category) values('G',867) into colors(name, category) values('G',868) into colors(name, category) values('G',869) into colors(name, category) values('G',870) into colors(name, category) values('G',871) into colors(name, category) values('G',872) into colors(name, category) values('G',873) into colors(name, category) values('G',874) into colors(name, category) values('G',875) into colors(name, category) values('G',876) into colors(name, category) values('G',877) into colors(name, category) values('G',878) into colors(name, category) values('G',879) into colors(name, category) values('G',880) into colors(name, category) values('G'
 ,881) into colors(name, category) values('G',882) into colors(name, category) values('G',883) into colors(name, category) values('G',884) into colors(name, category) values('G',885) into colors(name, category) values('G',886) into colors(name, category) values('G',887) into colors(name, category) values('G',888) into colors(name, category) values('G',889) into colors(name, category) values('G',890) into colors(name, category) values('G',891) into colors(name, category) values('G',892) into colors(name, category) values('G',893) into colors(name, category) values('G',894) into colors(name, category) values('G',895) into colors(name, category) values('G',896) into colors(name, category) values('G',897) into colors(name, category) values('G',898) into colors(name, category) values('G',899) into colors(name, category) values('G',900) into colors(name, category) values('G',901) into colors(name, category) values('G',902) into colors(name, category) values('G',903) into colors(name, categ
 ory) values('G',904) into colors(name, category) values('G',905) into colors(name, category) values('G',906) into colors(name, category) values('G',907) into colors(name, category) values('G',908) into colors(name, category) values('G',909) into colors(name, category) values('G',910) into colors(name, category) values('G',911) into colors(name, category) values('G',912) into colors(name, category) values('G',913) into colors(name, category) values('G',914) into colors(name, category) values('G',915) into colors(name, category) values('G',916) into colors(name, category) values('G',917) into colors(name, category) values('G',918) into colors(name, category) values('G',919) into colors(name, category) values('G',920) into colors(name, category) values('G',921) into colors(name, category) values('G',922) into colors(name, category) values('G',923) into colors(name, category) values('G',924) into colors(name, category) values('G',925) into colors(name, category) values('G',926) into col
 ors(name, category) values('G',927) into colors(name, category) values('G',928) into colors(name, category) values('G',929) into colors(name, category) values('G',930) into colors(name, category) values('G',931) into colors(name, category) values('G',932) into colors(name, category) values('G',933) into colors(name, category) values('G',934) into colors(name, category) values('G',935) into colors(name, category) values('G',936) into colors(name, category) values('G',937) into colors(name, category) values('G',938) into colors(name, category) values('G',939) into colors(name, category) values('G',940) into colors(name, category) values('G',941) into colors(name, category) values('G',942) into colors(name, category) values('G',943) into colors(name, category) values('G',944) into colors(name, category) values('G',945) into colors(name, category) values('G',946) into colors(name, category) values('G',947) into colors(name, category) values('G',948) into colors(name, category) values('G
 ',949) into colors(name, category) values('G',950) into colors(name, category) values('G',951) into colors(name, category) values('G',952) into colors(name, category) values('G',953) into colors(name, category) values('G',954) into colors(name, category) values('G',955) into colors(name, category) values('G',956) into colors(name, category) values('G',957) into colors(name, category) values('G',958) into colors(name, category) values('G',959) into colors(name, category) values('G',960) into colors(name, category) values('G',961) into colors(name, category) values('G',962) into colors(name, category) values('G',963) into colors(name, category) values('G',964) into colors(name, category) values('G',965) into colors(name, category) values('G',966) into colors(name, category) values('G',967) into colors(name, category) values('G',968) into colors(name, category) values('G',969) into colors(name, category) values('G',970) into colors(name, category) values('G',971) into colors(name, cate
 gory) values('G',972) into colors(name, category) values('G',973) into colors(name, category) values('G',974) into colors(name, category) values('G',975) into colors(name, category) values('G',976) into colors(name, category) values('G',977) into colors(name, category) values('G',978) into colors(name, category) values('G',979) into colors(name, category) values('G',980) into colors(name, category) values('G',981) into colors(name, category) values('G',982) into colors(name, category) values('G',983) into colors(name, category) values('G',984) into colors(name, category) values('G',985) into colors(name, category) values('G',986) into colors(name, category) values('G',987) into colors(name, category) values('G',988) into colors(name, category) values('G',989) into colors(name, category) values('G',990) into colors(name, category) values('G',991) into colors(name, category) values('G',992) into colors(name, category) values('G',993) into colors(name, category) values('G',994) into co
 lors(name, category) values('G',995) into colors(name, category) values('G',996)  select * from dual", sql.get(0));
+    Assert.assertEquals("Wrong stmt", "insert all into colors(name, category) values('G',997) into colors(name, category) values('G',998) into colors(name, category) values('G',999) select * from dual", sql.get(1));
+    
+    sqlGenerator =
+      new SQLGenerator(DatabaseProduct.MYSQL, conf);
+    rows.clear();
+    rows.add("'yellow', 1");
+    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 1, sql.size());
+    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1)", sql.get(0));
+    rows.add("'red', 2");
+    rows.add("'orange', 3");
+    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 1, sql.size());
+    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1),('red', 2),('orange', 3)", sql.get(0));
+    for(int i = 0; i < MetastoreConf.getIntVar(conf, ConfVars.DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE); i++) {
+      rows.add("\'G\'," + i);
+    }
+    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
+    Assert.assertEquals("Number of stmts", 2, sql.size());
+    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1),('red', 2),('orange', 3),('G',0),('G',1),('G',2),('G',3),('G',4),('G',5),('G',6),('G',7),('G',8),('G',9),('G',10),('G',11),('G',12),('G',13),('G',14),('G',15),('G',16),('G',17),('G',18),('G',19),('G',20),('G',21),('G',22),('G',23),('G',24),('G',25),('G',26),('G',27),('G',28),('G',29),('G',30),('G',31),('G',32),('G',33),('G',34),('G',35),('G',36),('G',37),('G',38),('G',39),('G',40),('G',41),('G',42),('G',43),('G',44),('G',45),('G',46),('G',47),('G',48),('G',49),('G',50),('G',51),('G',52),('G',53),('G',54),('G',55),('G',56),('G',57),('G',58),('G',59),('G',60),('G',61),('G',62),('G',63),('G',64),('G',65),('G',66),('G',67),('G',68),('G',69),('G',70),('G',71),('G',72),('G',73),('G',74),('G',75),('G',76),('G',77),('G',78),('G',79),('G',80),('G',81),('G',82),('G',83),('G',84),('G',85),('G',86),('G',87),('G',88),('G',89),('G',90),('G',91),('G',92),('G',93),('G',94),('G',95),('G',96),('G',97),('G',9
 8),('G',99),('G',100),('G',101),('G',102),('G',103),('G',104),('G',105),('G',106),('G',107),('G',108),('G',109),('G',110),('G',111),('G',112),('G',113),('G',114),('G',115),('G',116),('G',117),('G',118),('G',119),('G',120),('G',121),('G',122),('G',123),('G',124),('G',125),('G',126),('G',127),('G',128),('G',129),('G',130),('G',131),('G',132),('G',133),('G',134),('G',135),('G',136),('G',137),('G',138),('G',139),('G',140),('G',141),('G',142),('G',143),('G',144),('G',145),('G',146),('G',147),('G',148),('G',149),('G',150),('G',151),('G',152),('G',153),('G',154),('G',155),('G',156),('G',157),('G',158),('G',159),('G',160),('G',161),('G',162),('G',163),('G',164),('G',165),('G',166),('G',167),('G',168),('G',169),('G',170),('G',171),('G',172),('G',173),('G',174),('G',175),('G',176),('G',177),('G',178),('G',179),('G',180),('G',181),('G',182),('G',183),('G',184),('G',185),('G',186),('G',187),('G',188),('G',189),('G',190),('G',191),('G',192),('G',193),('G',194),('G',195),('G',196),('G',197),('G',
 198),('G',199),('G',200),('G',201),('G',202),('G',203),('G',204),('G',205),('G',206),('G',207),('G',208),('G',209),('G',210),('G',211),('G',212),('G',213),('G',214),('G',215),('G',216),('G',217),('G',218),('G',219),('G',220),('G',221),('G',222),('G',223),('G',224),('G',225),('G',226),('G',227),('G',228),('G',229),('G',230),('G',231),('G',232),('G',233),('G',234),('G',235),('G',236),('G',237),('G',238),('G',239),('G',240),('G',241),('G',242),('G',243),('G',244),('G',245),('G',246),('G',247),('G',248),('G',249),('G',250),('G',251),('G',252),('G',253),('G',254),('G',255),('G',256),('G',257),('G',258),('G',259),('G',260),('G',261),('G',262),('G',263),('G',264),('G',265),('G',266),('G',267),('G',268),('G',269),('G',270),('G',271),('G',272),('G',273),('G',274),('G',275),('G',276),('G',277),('G',278),('G',279),('G',280),('G',281),('G',282),('G',283),('G',284),('G',285),('G',286),('G',287),('G',288),('G',289),('G',290),('G',291),('G',292),('G',293),('G',294),('G',295),('G',296),('G',297),('
 G',298),('G',299),('G',300),('G',301),('G',302),('G',303),('G',304),('G',305),('G',306),('G',307),('G',308),('G',309),('G',310),('G',311),('G',312),('G',313),('G',314),('G',315),('G',316),('G',317),('G',318),('G',319),('G',320),('G',321),('G',322),('G',323),('G',324),('G',325),('G',326),('G',327),('G',328),('G',329),('G',330),('G',331),('G',332),('G',333),('G',334),('G',335),('G',336),('G',337),('G',338),('G',339),('G',340),('G',341),('G',342),('G',343),('G',344),('G',345),('G',346),('G',347),('G',348),('G',349),('G',350),('G',351),('G',352),('G',353),('G',354),('G',355),('G',356),('G',357),('G',358),('G',359),('G',360),('G',361),('G',362),('G',363),('G',364),('G',365),('G',366),('G',367),('G',368),('G',369),('G',370),('G',371),('G',372),('G',373),('G',374),('G',375),('G',376),('G',377),('G',378),('G',379),('G',380),('G',381),('G',382),('G',383),('G',384),('G',385),('G',386),('G',387),('G',388),('G',389),('G',390),('G',391),('G',392),('G',393),('G',394),('G',395),('G',396),('G',397)
 ,('G',398),('G',399),('G',400),('G',401),('G',402),('G',403),('G',404),('G',405),('G',406),('G',407),('G',408),('G',409),('G',410),('G',411),('G',412),('G',413),('G',414),('G',415),('G',416),('G',417),('G',418),('G',419),('G',420),('G',421),('G',422),('G',423),('G',424),('G',425),('G',426),('G',427),('G',428),('G',429),('G',430),('G',431),('G',432),('G',433),('G',434),('G',435),('G',436),('G',437),('G',438),('G',439),('G',440),('G',441),('G',442),('G',443),('G',444),('G',445),('G',446),('G',447),('G',448),('G',449),('G',450),('G',451),('G',452),('G',453),('G',454),('G',455),('G',456),('G',457),('G',458),('G',459),('G',460),('G',461),('G',462),('G',463),('G',464),('G',465),('G',466),('G',467),('G',468),('G',469),('G',470),('G',471),('G',472),('G',473),('G',474),('G',475),('G',476),('G',477),('G',478),('G',479),('G',480),('G',481),('G',482),('G',483),('G',484),('G',485),('G',486),('G',487),('G',488),('G',489),('G',490),('G',491),('G',492),('G',493),('G',494),('G',495),('G',496),('G',4
 97),('G',498),('G',499),('G',500),('G',501),('G',502),('G',503),('G',504),('G',505),('G',506),('G',507),('G',508),('G',509),('G',510),('G',511),('G',512),('G',513),('G',514),('G',515),('G',516),('G',517),('G',518),('G',519),('G',520),('G',521),('G',522),('G',523),('G',524),('G',525),('G',526),('G',527),('G',528),('G',529),('G',530),('G',531),('G',532),('G',533),('G',534),('G',535),('G',536),('G',537),('G',538),('G',539),('G',540),('G',541),('G',542),('G',543),('G',544),('G',545),('G',546),('G',547),('G',548),('G',549),('G',550),('G',551),('G',552),('G',553),('G',554),('G',555),('G',556),('G',557),('G',558),('G',559),('G',560),('G',561),('G',562),('G',563),('G',564),('G',565),('G',566),('G',567),('G',568),('G',569),('G',570),('G',571),('G',572),('G',573),('G',574),('G',575),('G',576),('G',577),('G',578),('G',579),('G',580),('G',581),('G',582),('G',583),('G',584),('G',585),('G',586),('G',587),('G',588),('G',589),('G',590),('G',591),('G',592),('G',593),('G',594),('G',595),('G',596),('G
 ',597),('G',598),('G',599),('G',600),('G',601),('G',602),('G',603),('G',604),('G',605),('G',606),('G',607),('G',608),('G',609),('G',610),('G',611),('G',612),('G',613),('G',614),('G',615),('G',616),('G',617),('G',618),('G',619),('G',620),('G',621),('G',622),('G',623),('G',624),('G',625),('G',626),('G',627),('G',628),('G',629),('G',630),('G',631),('G',632),('G',633),('G',634),('G',635),('G',636),('G',637),('G',638),('G',639),('G',640),('G',641),('G',642),('G',643),('G',644),('G',645),('G',646),('G',647),('G',648),('G',649),('G',650),('G',651),('G',652),('G',653),('G',654),('G',655),('G',656),('G',657),('G',658),('G',659),('G',660),('G',661),('G',662),('G',663),('G',664),('G',665),('G',666),('G',667),('G',668),('G',669),('G',670),('G',671),('G',672),('G',673),('G',674),('G',675),('G',676),('G',677),('G',678),('G',679),('G',680),('G',681),('G',682),('G',683),('G',684),('G',685),('G',686),('G',687),('G',688),('G',689),('G',690),('G',691),('G',692),('G',693),('G',694),('G',695),('G',696),
 ('G',697),('G',698),('G',699),('G',700),('G',701),('G',702),('G',703),('G',704),('G',705),('G',706),('G',707),('G',708),('G',709),('G',710),('G',711),('G',712),('G',713),('G',714),('G',715),('G',716),('G',717),('G',718),('G',719),('G',720),('G',721),('G',722),('G',723),('G',724),('G',725),('G',726),('G',727),('G',728),('G',729),('G',730),('G',731),('G',732),('G',733),('G',734),('G',735),('G',736),('G',737),('G',738),('G',739),('G',740),('G',741),('G',742),('G',743),('G',744),('G',745),('G',746),('G',747),('G',748),('G',749),('G',750),('G',751),('G',752),('G',753),('G',754),('G',755),('G',756),('G',757),('G',758),('G',759),('G',760),('G',761),('G',762),('G',763),('G',764),('G',765),('G',766),('G',767),('G',768),('G',769),('G',770),('G',771),('G',772),('G',773),('G',774),('G',775),('G',776),('G',777),('G',778),('G',779),('G',780),('G',781),('G',782),('G',783),('G',784),('G',785),('G',786),('G',787),('G',788),('G',789),('G',790),('G',791),('G',792),('G',793),('G',794),('G',795),('G',79
 6),('G',797),('G',798),('G',799),('G',800),('G',801),('G',802),('G',803),('G',804),('G',805),('G',806),('G',807),('G',808),('G',809),('G',810),('G',811),('G',812),('G',813),('G',814),('G',815),('G',816),('G',817),('G',818),('G',819),('G',820),('G',821),('G',822),('G',823),('G',824),('G',825),('G',826),('G',827),('G',828),('G',829),('G',830),('G',831),('G',832),('G',833),('G',834),('G',835),('G',836),('G',837),('G',838),('G',839),('G',840),('G',841),('G',842),('G',843),('G',844),('G',845),('G',846),('G',847),('G',848),('G',849),('G',850),('G',851),('G',852),('G',853),('G',854),('G',855),('G',856),('G',857),('G',858),('G',859),('G',860),('G',861),('G',862),('G',863),('G',864),('G',865),('G',866),('G',867),('G',868),('G',869),('G',870),('G',871),('G',872),('G',873),('G',874),('G',875),('G',876),('G',877),('G',878),('G',879),('G',880),('G',881),('G',882),('G',883),('G',884),('G',885),('G',886),('G',887),('G',888),('G',889),('G',890),('G',891),('G',892),('G',893),('G',894),('G',895),('G'
 ,896),('G',897),('G',898),('G',899),('G',900),('G',901),('G',902),('G',903),('G',904),('G',905),('G',906),('G',907),('G',908),('G',909),('G',910),('G',911),('G',912),('G',913),('G',914),('G',915),('G',916),('G',917),('G',918),('G',919),('G',920),('G',921),('G',922),('G',923),('G',924),('G',925),('G',926),('G',927),('G',928),('G',929),('G',930),('G',931),('G',932),('G',933),('G',934),('G',935),('G',936),('G',937),('G',938),('G',939),('G',940),('G',941),('G',942),('G',943),('G',944),('G',945),('G',946),('G',947),('G',948),('G',949),('G',950),('G',951),('G',952),('G',953),('G',954),('G',955),('G',956),('G',957),('G',958),('G',959),('G',960),('G',961),('G',962),('G',963),('G',964),('G',965),('G',966),('G',967),('G',968),('G',969),('G',970),('G',971),('G',972),('G',973),('G',974),('G',975),('G',976),('G',977),('G',978),('G',979),('G',980),('G',981),('G',982),('G',983),('G',984),('G',985),('G',986),('G',987),('G',988),('G',989),('G',990),('G',991),('G',992),('G',993),('G',994),('G',995),(
 'G',996)", sql.get(0));
+    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('G',997),('G',998),('G',999)", sql.get(1));
+
+    sqlGenerator = new SQLGenerator(DatabaseProduct.SQLSERVER, conf);
+    String modSql = sqlGenerator.addForUpdateClause("select nl_next from NEXT_LOCK_ID");
+    Assert.assertEquals("select nl_next from NEXT_LOCK_ID with (updlock)", modSql);
+    modSql = sqlGenerator.addForUpdateClause("select MT_COMMENT from AUX_TABLE where MT_KEY1='CheckLock' and MT_KEY2=0");
+    Assert.assertEquals("select MT_COMMENT from AUX_TABLE with (updlock) where MT_KEY1='CheckLock' and MT_KEY2=0", modSql);
+  }
+
+  @Before
+  public void setUp() throws Exception {
+    conf = MetastoreConf.newMetastoreConf();
+    TxnDbUtil.setConfValues(conf);
+    TxnDbUtil.prepDb(conf);
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    TxnDbUtil.cleanDb(conf);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 199acec..807cb2c 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -31,6 +31,7 @@
 
   <properties>
     <commons-lang.version>2.6</commons-lang.version>
+    <commons-logging.version>1.1.3</commons-logging.version>
     <guava.version>14.0.1</guava.version>
     <hadoop.version>2.8.1</hadoop.version>
     <junit.version>4.11</junit.version>
@@ -115,6 +116,12 @@
       <version>${junit.version}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>commons-logging</groupId>
+      <artifactId>commons-logging</artifactId>
+      <version>${commons-logging.version}</version>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
new file mode 100644
index 0000000..94b8c58
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by the compactor.
+ * 
+ * Compaction should only include txns up to smallest open txn (exclussive).
+ * There may be aborted txns in the snapshot represented by this ValidCompactorTxnList.
+ * Thus {@link #isTxnRangeValid(long, long)} returns NONE for any range that inluces any unresolved
+ * transactions.  Any txn above {@code highWatermark} is unresolved.
+ * These produce the logic we need to assure that the compactor only sees records less than the lowest
+ * open transaction when choosing which files to compact, but that it still ignores aborted
+ * records when compacting.
+ * 
+ * See org.apache.hadoop.hive.metastore.txn.TxnUtils#createValidCompactTxnList() for proper
+ * way to construct this.
+ */
+public class ValidCompactorTxnList extends ValidReadTxnList {
+  public ValidCompactorTxnList() {
+    super();
+  }
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
+    this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
+  }
+  /**
+   * @param abortedTxnList list of all aborted transactions
+   * @param abortedBits bitset marking whether the corresponding transaction is aborted
+   * @param highWatermark highest committed transaction to be considered for compaction,
+   *                      equivalently (lowest_open_txn - 1).
+   */
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) {
+    // abortedBits should be all true as everything in exceptions are aborted txns
+    super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
+    if(this.exceptions.length <= 0) {
+      return;
+    }
+    //now that exceptions (aka abortedTxnList) is sorted
+    int idx = Arrays.binarySearch(this.exceptions, highWatermark);
+    int lastElementPos;
+    if(idx < 0) {
+      int insertionPoint = -idx - 1 ;//see Arrays.binarySearch() JavaDoc
+      lastElementPos = insertionPoint - 1;
+    }
+    else {
+      lastElementPos = idx;
+    }
+    /*
+     * ensure that we throw out any exceptions above highWatermark to make
+     * {@link #isTxnValid(long)} faster 
+     */
+    this.exceptions = Arrays.copyOf(this.exceptions, lastElementPos + 1);
+  }
+  public ValidCompactorTxnList(String value) {
+    super(value);
+  }
+  /**
+   * Returns org.apache.hadoop.hive.common.ValidTxnList.RangeResponse.ALL if all txns in
+   * the range are resolved and RangeResponse.NONE otherwise
+   */
+  @Override
+  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+    return highWatermark >= maxTxnId ? RangeResponse.ALL : RangeResponse.NONE;
+  }
+
+  @Override
+  public boolean isTxnAborted(long txnid) {
+    return Arrays.binarySearch(exceptions, txnid) >= 0;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
new file mode 100644
index 0000000..ccdd4b7
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+import java.util.Arrays;
+import java.util.BitSet;
+
+/**
+ * An implementation of {@link org.apache.hadoop.hive.common.ValidTxnList} for use by readers.
+ * This class will view a transaction as valid only if it is committed.  Both open and aborted
+ * transactions will be seen as invalid.
+ */
+public class ValidReadTxnList implements ValidTxnList {
+
+  protected long[] exceptions;
+  protected BitSet abortedBits; // BitSet for flagging aborted transactions. Bit is true if aborted, false if open
+  //default value means there are no open txn in the snapshot
+  private long minOpenTxn = Long.MAX_VALUE;
+  protected long highWatermark;
+
+  public ValidReadTxnList() {
+    this(new long[0], new BitSet(), Long.MAX_VALUE, Long.MAX_VALUE);
+  }
+
+  /**
+   * Used if there are no open transactions in the snapshot
+   */
+  public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark) {
+    this(exceptions, abortedBits, highWatermark, Long.MAX_VALUE);
+  }
+  public ValidReadTxnList(long[] exceptions, BitSet abortedBits, long highWatermark, long minOpenTxn) {
+    if (exceptions.length > 0) {
+      this.minOpenTxn = minOpenTxn;
+    }
+    this.exceptions = exceptions;
+    this.abortedBits = abortedBits;
+    this.highWatermark = highWatermark;
+  }
+
+  public ValidReadTxnList(String value) {
+    readFromString(value);
+  }
+
+  @Override
+  public boolean isTxnValid(long txnid) {
+    if (highWatermark < txnid) {
+      return false;
+    }
+    return Arrays.binarySearch(exceptions, txnid) < 0;
+  }
+
+  /**
+   * We cannot use a base file if its range contains an open txn.
+   * @param txnid from base_xxxx
+   */
+  @Override
+  public boolean isValidBase(long txnid) {
+    return minOpenTxn > txnid && txnid <= highWatermark;
+  }
+  @Override
+  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId) {
+    // check the easy cases first
+    if (highWatermark < minTxnId) {
+      return RangeResponse.NONE;
+    } else if (exceptions.length > 0 && exceptions[0] > maxTxnId) {
+      return RangeResponse.ALL;
+    }
+
+    // since the exceptions and the range in question overlap, count the
+    // exceptions in the range
+    long count = Math.max(0, maxTxnId - highWatermark);
+    for(long txn: exceptions) {
+      if (minTxnId <= txn && txn <= maxTxnId) {
+        count += 1;
+      }
+    }
+
+    if (count == 0) {
+      return RangeResponse.ALL;
+    } else if (count == (maxTxnId - minTxnId + 1)) {
+      return RangeResponse.NONE;
+    } else {
+      return RangeResponse.SOME;
+    }
+  }
+
+  @Override
+  public String toString() {
+    return writeToString();
+  }
+
+  @Override
+  public String writeToString() {
+    StringBuilder buf = new StringBuilder();
+    buf.append(highWatermark);
+    buf.append(':');
+    buf.append(minOpenTxn);
+    if (exceptions.length == 0) {
+      buf.append(':');  // separator for open txns
+      buf.append(':');  // separator for aborted txns
+    } else {
+      StringBuilder open = new StringBuilder();
+      StringBuilder abort = new StringBuilder();
+      for (int i = 0; i < exceptions.length; i++) {
+        if (abortedBits.get(i)) {
+          if (abort.length() > 0) {
+            abort.append(',');
+          }
+          abort.append(exceptions[i]);
+        } else {
+          if (open.length() > 0) {
+            open.append(',');
+          }
+          open.append(exceptions[i]);
+        }
+      }
+      buf.append(':');
+      buf.append(open);
+      buf.append(':');
+      buf.append(abort);
+    }
+    return buf.toString();
+  }
+
+  @Override
+  public void readFromString(String src) {
+    if (src == null || src.length() == 0) {
+      highWatermark = Long.MAX_VALUE;
+      exceptions = new long[0];
+      abortedBits = new BitSet();
+    } else {
+      String[] values = src.split(":");
+      highWatermark = Long.parseLong(values[0]);
+      minOpenTxn = Long.parseLong(values[1]);
+      String[] openTxns = new String[0];
+      String[] abortedTxns = new String[0];
+      if (values.length < 3) {
+        openTxns = new String[0];
+        abortedTxns = new String[0];
+      } else if (values.length == 3) {
+        if (!values[2].isEmpty()) {
+          openTxns = values[2].split(",");
+        }
+      } else {
+        if (!values[2].isEmpty()) {
+          openTxns = values[2].split(",");
+        }
+        if (!values[3].isEmpty()) {
+          abortedTxns = values[3].split(",");
+        }
+      }
+      exceptions = new long[openTxns.length + abortedTxns.length];
+      int i = 0;
+      for (String open : openTxns) {
+        exceptions[i++] = Long.parseLong(open);
+      }
+      for (String abort : abortedTxns) {
+        exceptions[i++] = Long.parseLong(abort);
+      }
+      Arrays.sort(exceptions);
+      abortedBits = new BitSet(exceptions.length);
+      for (String abort : abortedTxns) {
+        int index = Arrays.binarySearch(exceptions, Long.parseLong(abort));
+        abortedBits.set(index);
+      }
+    }
+  }
+
+  @Override
+  public long getHighWatermark() {
+    return highWatermark;
+  }
+
+  @Override
+  public long[] getInvalidTransactions() {
+    return exceptions;
+  }
+
+  @Override
+  public Long getMinOpenTxn() {
+    return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
+  }
+
+  @Override
+  public boolean isTxnAborted(long txnid) {
+    int index = Arrays.binarySearch(exceptions, txnid);
+    return index >= 0 && abortedBits.get(index);
+  }
+
+  @Override
+  public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId) {
+    // check the easy cases first
+    if (highWatermark < minTxnId) {
+      return RangeResponse.NONE;
+    }
+
+    int count = 0;  // number of aborted txns found in exceptions
+
+    // traverse the aborted txns list, starting at first aborted txn index
+    for (int i = abortedBits.nextSetBit(0); i >= 0; i = abortedBits.nextSetBit(i + 1)) {
+      long abortedTxnId = exceptions[i];
+      if (abortedTxnId > maxTxnId) {  // we've already gone beyond the specified range
+        break;
+      }
+      if (abortedTxnId >= minTxnId && abortedTxnId <= maxTxnId) {
+        count++;
+      }
+    }
+
+    if (count == 0) {
+      return RangeResponse.NONE;
+    } else if (count == (maxTxnId - minTxnId + 1)) {
+      return RangeResponse.ALL;
+    } else {
+      return RangeResponse.SOME;
+    }
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
new file mode 100644
index 0000000..108e5ca
--- /dev/null
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.common;
+
+/**
+ * Models the list of transactions that should be included in a snapshot.
+ * It is modelled as a high water mark, which is the largest transaction id that
+ * has been committed and a list of transactions that are not included.
+ */
+public interface ValidTxnList {
+
+  /**
+   * Key used to store valid txn list in a
+   * {@link org.apache.hadoop.conf.Configuration} object.
+   */
+  public static final String VALID_TXNS_KEY = "hive.txn.valid.txns";
+
+  /**
+   * The response to a range query.  NONE means no values in this range match,
+   * SOME mean that some do, and ALL means that every value does.
+   */
+  public enum RangeResponse {NONE, SOME, ALL};
+
+  /**
+   * Indicates whether a given transaction is valid. Note that valid may have different meanings
+   * for different implementations, as some will only want to see committed transactions and some
+   * both committed and aborted.
+   * @param txnid id for the transaction
+   * @return true if valid, false otherwise
+   */
+  public boolean isTxnValid(long txnid);
+
+  /**
+   * Returns {@code true} if such base file can be used to materialize the snapshot represented by
+   * this {@code ValidTxnList}.
+   * @param txnid highest txn in a given base_xxxx file
+   */
+  public boolean isValidBase(long txnid);
+
+  /**
+   * Find out if a range of transaction ids are valid.  Note that valid may have different meanings
+   * for different implementations, as some will only want to see committed transactions and some
+   * both committed and aborted.
+   * @param minTxnId minimum txnid to look for, inclusive
+   * @param maxTxnId maximum txnid to look for, inclusive
+   * @return Indicate whether none, some, or all of these transactions are valid.
+   */
+  public RangeResponse isTxnRangeValid(long minTxnId, long maxTxnId);
+
+  /**
+   * Write this validTxnList into a string. This should produce a string that
+   * can be used by {@link #readFromString(String)} to populate a validTxnsList.
+   */
+  public String writeToString();
+
+  /**
+   * Populate this validTxnList from the string.  It is assumed that the string
+   * was created via {@link #writeToString()} and the exceptions list is sorted.
+   * @param src source string.
+   */
+  public void readFromString(String src);
+
+  /**
+   * Get the largest transaction id used.
+   * @return largest transaction id used
+   */
+  public long getHighWatermark();
+
+  /**
+   * Get the list of transactions under the high water mark that are not valid.  Note that invalid
+   * may have different meanings for different implementations, as some will only want to see open
+   * transactions and some both open and aborted.
+   * @return a list of invalid transaction ids
+   */
+  public long[] getInvalidTransactions();
+
+  /**
+   * Indicates whether a given transaction is aborted.
+   * @param txnid id for the transaction
+   * @return true if aborted, false otherwise
+   */
+  public boolean isTxnAborted(long txnid);
+
+  /**
+   * Find out if a range of transaction ids are aborted.
+   * @param minTxnId minimum txnid to look for, inclusive
+   * @param maxTxnId maximum txnid to look for, inclusive
+   * @return Indicate whether none, some, or all of these transactions are aborted.
+   */
+  public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
+
+  /**
+   * Returns smallest Open transaction in this set, {@code null} if there is none.
+   */
+  Long getMinOpenTxn();
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
new file mode 100644
index 0000000..867b652
--- /dev/null
+++ b/storage-api/src/test/org/apache/hadoop/hive/common/TestValidCompactorTxnList.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.common;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.BitSet;
+
+public class TestValidCompactorTxnList {
+
+  @Test
+  public void minTxnHigh() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 4}, bitSet, 2);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void maxTxnLow() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{13, 14}, bitSet, 12);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
+  }
+
+  @Test
+  public void minTxnHighNoExceptions() {
+    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 5);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void maxTxnLowNoExceptions() {
+    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 15);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
+  }
+
+  @Test
+  public void exceptionsAllBelow() {
+    BitSet bitSet = new BitSet(2);
+    bitSet.set(0, 2);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 6}, bitSet, 3);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void exceptionsInMidst() {
+    BitSet bitSet = new BitSet(1);
+    bitSet.set(0, 1);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8}, bitSet, 7);
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+  }
+  @Test
+  public void exceptionsAbveHighWaterMark() {
+    BitSet bitSet = new BitSet(4);
+    bitSet.set(0, 4);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8, 11, 17, 29}, bitSet, 15);
+    Assert.assertArrayEquals("", new long[]{8, 11}, txns.getInvalidTransactions());
+    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
+    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
+    rsp = txns.isTxnRangeValid(12, 16);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+  }
+
+  @Test
+  public void writeToString() {
+    BitSet bitSet = new BitSet(4);
+    bitSet.set(0, 4);
+    ValidTxnList txns = new ValidCompactorTxnList(new long[]{7, 9, 10, Long.MAX_VALUE}, bitSet, 8);
+    Assert.assertEquals("8:" + Long.MAX_VALUE + "::7", txns.writeToString());
+    txns = new ValidCompactorTxnList();
+    Assert.assertEquals(Long.toString(Long.MAX_VALUE) + ":" + Long.MAX_VALUE + "::", txns.writeToString());
+    txns = new ValidCompactorTxnList(new long[0], new BitSet(), 23);
+    Assert.assertEquals("23:" + Long.MAX_VALUE + "::", txns.writeToString());
+  }
+
+  @Test
+  public void readFromString() {
+    ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10");
+    Assert.assertEquals(37L, txns.getHighWatermark());
+    Assert.assertNull(txns.getMinOpenTxn());
+    Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions());
+    txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
+    Assert.assertEquals(21L, txns.getHighWatermark());
+    Assert.assertNull(txns.getMinOpenTxn());
+    Assert.assertEquals(0, txns.getInvalidTransactions().length);
+  }
+
+  @Test
+  public void testAbortedTxn() throws Exception {
+    ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
+    Assert.assertEquals(5L, txnList.getHighWatermark());
+    Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
+    Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions());
+  }
+
+  @Test
+  public void testAbortedRange() throws Exception {
+    ValidCompactorTxnList txnList = new ValidCompactorTxnList("11:4::5,6,7,8");
+    ValidTxnList.RangeResponse rsp = txnList.isTxnRangeAborted(1L, 3L);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+    rsp = txnList.isTxnRangeAborted(9L, 10L);
+    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
+    rsp = txnList.isTxnRangeAborted(6L, 7L);
+    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
+    rsp = txnList.isTxnRangeAborted(4L, 6L);
+    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
+    rsp = txnList.isTxnRangeAborted(6L, 13L);
+    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
+  }
+}


[07/11] hive git commit: HIVE-17561 Move TxnStore and implementations to standalone metastore (Alan Gates, reviewed by Eugene Koifman)

Posted by ga...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
deleted file mode 100644
index 1497c00..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnUtils.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.DatabaseProduct;
-import org.apache.hadoop.hive.metastore.tools.SQLGenerator;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Tests for TxnUtils
- */
-public class TestTxnUtils {
-  private HiveConf conf;
-
-  public TestTxnUtils() throws Exception {
-  }
-
-  @Test
-  public void testBuildQueryWithINClause() throws Exception {
-    List<String> queries = new ArrayList<String>();
-
-    StringBuilder prefix = new StringBuilder();
-    StringBuilder suffix = new StringBuilder();
-
-    // Note, this is a "real" query that depends on one of the metastore tables
-    prefix.append("select count(*) from TXNS where ");
-    suffix.append(" and TXN_STATE = 'o'");
-
-    // Case 1 - Max in list members: 10; Max query string length: 1KB
-    //          The first query happens to have 2 full batches.
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 10);
-    List<Long> inList = new ArrayList<Long>();
-    for (long i = 1; i <= 200; i++) {
-      inList.add(i);
-    }
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(1, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 2 - Max in list members: 10; Max query string length: 1KB
-    //          The first query has 2 full batches, and the second query only has 1 batch which only contains 1 member
-    queries.clear();
-    inList.add((long)201);
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(2, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 3.1 - Max in list members: 1000, Max query string length: 1KB, and exact 1000 members in a single IN clause
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
-    queries.clear();
-    for (long i = 202; i <= 1000; i++) {
-      inList.add(i);
-    }
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(1, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 3.2 - Max in list members: 1000, Max query string length: 10KB, and exact 1000 members in a single IN clause
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10);
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_IN_CLAUSE, 1000);
-    queries.clear();
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(1, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 3.3 - Now with 2000 entries, try the above settings
-    for (long i = 1001; i <= 2000; i++) {
-      inList.add(i);
-    }
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 1);
-    queries.clear();
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(2, queries.size());
-    runAgainstDerby(queries);
-    conf.setIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_QUERY_LENGTH, 10);
-    queries.clear();
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(1, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 4 - Max in list members: 1000; Max query string length: 10KB
-    queries.clear();
-    for (long i = 2001; i <= 4321; i++) {
-      inList.add(i);
-    }
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, false);
-    Assert.assertEquals(3, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 5 - NOT IN list
-    queries.clear();
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", true, true);
-    Assert.assertEquals(3, queries.size());
-    runAgainstDerby(queries);
-
-    // Case 6 - No parenthesis
-    queries.clear();
-    suffix.setLength(0);
-    suffix.append("");
-    TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, inList, "TXN_ID", false, false);
-    Assert.assertEquals(3, queries.size());
-    runAgainstDerby(queries);
-  }
-
-  /** Verify queries can run against Derby DB.
-   *  As long as Derby doesn't complain, we assume the query is syntactically/semantically correct.
-   */
-  private void runAgainstDerby(List<String> queries) throws Exception {
-    Connection conn = null;
-    Statement stmt = null;
-    ResultSet rs = null;
-
-    try {
-      conn = TxnDbUtil.getConnection();
-      stmt = conn.createStatement();
-      for (String query : queries) {
-        rs = stmt.executeQuery(query);
-        Assert.assertTrue("The query is not valid", rs.next());
-      }
-    } finally {
-      TxnDbUtil.closeResources(conn, stmt, rs);
-    }
-  }
-  @Test
-  public void testSQLGenerator() throws Exception {
-    //teseted on Oracle Database 11g Express Edition Release 11.2.0.2.0 - 64bit Production
-    SQLGenerator sqlGenerator =
-      new SQLGenerator(DatabaseProduct.ORACLE, conf);
-    List<String> rows = new ArrayList<>();
-    rows.add("'yellow', 1");
-    List<String> sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 1, sql.size());
-    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1)", sql.get(0));
-    rows.add("'red', 2");
-    rows.add("'orange', 3");
-    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 1, sql.size());
-    
-    Assert.assertEquals("Wrong stmt", 
-      "insert all into colors(name, category) values('yellow', 1) into colors(name, category) values('red', 2) into colors(name, category) values('orange', 3) select * from dual", sql.get(0));
-    for(int i = 0; i < conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE); i++) {
-      rows.add("\'G\'," + i);
-    }
-    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 2, sql.size());
-    Assert.assertEquals("Wrong stmt", "insert all into colors(name, category) values('yellow', 1) into colors(name, category) values('red', 2) into colors(name, category) values('orange', 3) into colors(name, category) values('G',0) into colors(name, category) values('G',1) into colors(name, category) values('G',2) into colors(name, category) values('G',3) into colors(name, category) values('G',4) into colors(name, category) values('G',5) into colors(name, category) values('G',6) into colors(name, category) values('G',7) into colors(name, category) values('G',8) into colors(name, category) values('G',9) into colors(name, category) values('G',10) into colors(name, category) values('G',11) into colors(name, category) values('G',12) into colors(name, category) values('G',13) into colors(name, category) values('G',14) into colors(name, category) values('G',15) into colors(name, category) values('G',16) into colors(name, category) values('G',17) into colors(name, category) values('G',18)
  into colors(name, category) values('G',19) into colors(name, category) values('G',20) into colors(name, category) values('G',21) into colors(name, category) values('G',22) into colors(name, category) values('G',23) into colors(name, category) values('G',24) into colors(name, category) values('G',25) into colors(name, category) values('G',26) into colors(name, category) values('G',27) into colors(name, category) values('G',28) into colors(name, category) values('G',29) into colors(name, category) values('G',30) into colors(name, category) values('G',31) into colors(name, category) values('G',32) into colors(name, category) values('G',33) into colors(name, category) values('G',34) into colors(name, category) values('G',35) into colors(name, category) values('G',36) into colors(name, category) values('G',37) into colors(name, category) values('G',38) into colors(name, category) values('G',39) into colors(name, category) values('G',40) into colors(name, category) values('G',41) into co
 lors(name, category) values('G',42) into colors(name, category) values('G',43) into colors(name, category) values('G',44) into colors(name, category) values('G',45) into colors(name, category) values('G',46) into colors(name, category) values('G',47) into colors(name, category) values('G',48) into colors(name, category) values('G',49) into colors(name, category) values('G',50) into colors(name, category) values('G',51) into colors(name, category) values('G',52) into colors(name, category) values('G',53) into colors(name, category) values('G',54) into colors(name, category) values('G',55) into colors(name, category) values('G',56) into colors(name, category) values('G',57) into colors(name, category) values('G',58) into colors(name, category) values('G',59) into colors(name, category) values('G',60) into colors(name, category) values('G',61) into colors(name, category) values('G',62) into colors(name, category) values('G',63) into colors(name, category) values('G',64) into colors(nam
 e, category) values('G',65) into colors(name, category) values('G',66) into colors(name, category) values('G',67) into colors(name, category) values('G',68) into colors(name, category) values('G',69) into colors(name, category) values('G',70) into colors(name, category) values('G',71) into colors(name, category) values('G',72) into colors(name, category) values('G',73) into colors(name, category) values('G',74) into colors(name, category) values('G',75) into colors(name, category) values('G',76) into colors(name, category) values('G',77) into colors(name, category) values('G',78) into colors(name, category) values('G',79) into colors(name, category) values('G',80) into colors(name, category) values('G',81) into colors(name, category) values('G',82) into colors(name, category) values('G',83) into colors(name, category) values('G',84) into colors(name, category) values('G',85) into colors(name, category) values('G',86) into colors(name, category) values('G',87) into colors(name, categ
 ory) values('G',88) into colors(name, category) values('G',89) into colors(name, category) values('G',90) into colors(name, category) values('G',91) into colors(name, category) values('G',92) into colors(name, category) values('G',93) into colors(name, category) values('G',94) into colors(name, category) values('G',95) into colors(name, category) values('G',96) into colors(name, category) values('G',97) into colors(name, category) values('G',98) into colors(name, category) values('G',99) into colors(name, category) values('G',100) into colors(name, category) values('G',101) into colors(name, category) values('G',102) into colors(name, category) values('G',103) into colors(name, category) values('G',104) into colors(name, category) values('G',105) into colors(name, category) values('G',106) into colors(name, category) values('G',107) into colors(name, category) values('G',108) into colors(name, category) values('G',109) into colors(name, category) values('G',110) into colors(name, ca
 tegory) values('G',111) into colors(name, category) values('G',112) into colors(name, category) values('G',113) into colors(name, category) values('G',114) into colors(name, category) values('G',115) into colors(name, category) values('G',116) into colors(name, category) values('G',117) into colors(name, category) values('G',118) into colors(name, category) values('G',119) into colors(name, category) values('G',120) into colors(name, category) values('G',121) into colors(name, category) values('G',122) into colors(name, category) values('G',123) into colors(name, category) values('G',124) into colors(name, category) values('G',125) into colors(name, category) values('G',126) into colors(name, category) values('G',127) into colors(name, category) values('G',128) into colors(name, category) values('G',129) into colors(name, category) values('G',130) into colors(name, category) values('G',131) into colors(name, category) values('G',132) into colors(name, category) values('G',133) into 
 colors(name, category) values('G',134) into colors(name, category) values('G',135) into colors(name, category) values('G',136) into colors(name, category) values('G',137) into colors(name, category) values('G',138) into colors(name, category) values('G',139) into colors(name, category) values('G',140) into colors(name, category) values('G',141) into colors(name, category) values('G',142) into colors(name, category) values('G',143) into colors(name, category) values('G',144) into colors(name, category) values('G',145) into colors(name, category) values('G',146) into colors(name, category) values('G',147) into colors(name, category) values('G',148) into colors(name, category) values('G',149) into colors(name, category) values('G',150) into colors(name, category) values('G',151) into colors(name, category) values('G',152) into colors(name, category) values('G',153) into colors(name, category) values('G',154) into colors(name, category) values('G',155) into colors(name, category) values
 ('G',156) into colors(name, category) values('G',157) into colors(name, category) values('G',158) into colors(name, category) values('G',159) into colors(name, category) values('G',160) into colors(name, category) values('G',161) into colors(name, category) values('G',162) into colors(name, category) values('G',163) into colors(name, category) values('G',164) into colors(name, category) values('G',165) into colors(name, category) values('G',166) into colors(name, category) values('G',167) into colors(name, category) values('G',168) into colors(name, category) values('G',169) into colors(name, category) values('G',170) into colors(name, category) values('G',171) into colors(name, category) values('G',172) into colors(name, category) values('G',173) into colors(name, category) values('G',174) into colors(name, category) values('G',175) into colors(name, category) values('G',176) into colors(name, category) values('G',177) into colors(name, category) values('G',178) into colors(name, c
 ategory) values('G',179) into colors(name, category) values('G',180) into colors(name, category) values('G',181) into colors(name, category) values('G',182) into colors(name, category) values('G',183) into colors(name, category) values('G',184) into colors(name, category) values('G',185) into colors(name, category) values('G',186) into colors(name, category) values('G',187) into colors(name, category) values('G',188) into colors(name, category) values('G',189) into colors(name, category) values('G',190) into colors(name, category) values('G',191) into colors(name, category) values('G',192) into colors(name, category) values('G',193) into colors(name, category) values('G',194) into colors(name, category) values('G',195) into colors(name, category) values('G',196) into colors(name, category) values('G',197) into colors(name, category) values('G',198) into colors(name, category) values('G',199) into colors(name, category) values('G',200) into colors(name, category) values('G',201) into
  colors(name, category) values('G',202) into colors(name, category) values('G',203) into colors(name, category) values('G',204) into colors(name, category) values('G',205) into colors(name, category) values('G',206) into colors(name, category) values('G',207) into colors(name, category) values('G',208) into colors(name, category) values('G',209) into colors(name, category) values('G',210) into colors(name, category) values('G',211) into colors(name, category) values('G',212) into colors(name, category) values('G',213) into colors(name, category) values('G',214) into colors(name, category) values('G',215) into colors(name, category) values('G',216) into colors(name, category) values('G',217) into colors(name, category) values('G',218) into colors(name, category) values('G',219) into colors(name, category) values('G',220) into colors(name, category) values('G',221) into colors(name, category) values('G',222) into colors(name, category) values('G',223) into colors(name, category) value
 s('G',224) into colors(name, category) values('G',225) into colors(name, category) values('G',226) into colors(name, category) values('G',227) into colors(name, category) values('G',228) into colors(name, category) values('G',229) into colors(name, category) values('G',230) into colors(name, category) values('G',231) into colors(name, category) values('G',232) into colors(name, category) values('G',233) into colors(name, category) values('G',234) into colors(name, category) values('G',235) into colors(name, category) values('G',236) into colors(name, category) values('G',237) into colors(name, category) values('G',238) into colors(name, category) values('G',239) into colors(name, category) values('G',240) into colors(name, category) values('G',241) into colors(name, category) values('G',242) into colors(name, category) values('G',243) into colors(name, category) values('G',244) into colors(name, category) values('G',245) into colors(name, category) values('G',246) into colors(name, 
 category) values('G',247) into colors(name, category) values('G',248) into colors(name, category) values('G',249) into colors(name, category) values('G',250) into colors(name, category) values('G',251) into colors(name, category) values('G',252) into colors(name, category) values('G',253) into colors(name, category) values('G',254) into colors(name, category) values('G',255) into colors(name, category) values('G',256) into colors(name, category) values('G',257) into colors(name, category) values('G',258) into colors(name, category) values('G',259) into colors(name, category) values('G',260) into colors(name, category) values('G',261) into colors(name, category) values('G',262) into colors(name, category) values('G',263) into colors(name, category) values('G',264) into colors(name, category) values('G',265) into colors(name, category) values('G',266) into colors(name, category) values('G',267) into colors(name, category) values('G',268) into colors(name, category) values('G',269) int
 o colors(name, category) values('G',270) into colors(name, category) values('G',271) into colors(name, category) values('G',272) into colors(name, category) values('G',273) into colors(name, category) values('G',274) into colors(name, category) values('G',275) into colors(name, category) values('G',276) into colors(name, category) values('G',277) into colors(name, category) values('G',278) into colors(name, category) values('G',279) into colors(name, category) values('G',280) into colors(name, category) values('G',281) into colors(name, category) values('G',282) into colors(name, category) values('G',283) into colors(name, category) values('G',284) into colors(name, category) values('G',285) into colors(name, category) values('G',286) into colors(name, category) values('G',287) into colors(name, category) values('G',288) into colors(name, category) values('G',289) into colors(name, category) values('G',290) into colors(name, category) values('G',291) into colors(name, category) valu
 es('G',292) into colors(name, category) values('G',293) into colors(name, category) values('G',294) into colors(name, category) values('G',295) into colors(name, category) values('G',296) into colors(name, category) values('G',297) into colors(name, category) values('G',298) into colors(name, category) values('G',299) into colors(name, category) values('G',300) into colors(name, category) values('G',301) into colors(name, category) values('G',302) into colors(name, category) values('G',303) into colors(name, category) values('G',304) into colors(name, category) values('G',305) into colors(name, category) values('G',306) into colors(name, category) values('G',307) into colors(name, category) values('G',308) into colors(name, category) values('G',309) into colors(name, category) values('G',310) into colors(name, category) values('G',311) into colors(name, category) values('G',312) into colors(name, category) values('G',313) into colors(name, category) values('G',314) into colors(name,
  category) values('G',315) into colors(name, category) values('G',316) into colors(name, category) values('G',317) into colors(name, category) values('G',318) into colors(name, category) values('G',319) into colors(name, category) values('G',320) into colors(name, category) values('G',321) into colors(name, category) values('G',322) into colors(name, category) values('G',323) into colors(name, category) values('G',324) into colors(name, category) values('G',325) into colors(name, category) values('G',326) into colors(name, category) values('G',327) into colors(name, category) values('G',328) into colors(name, category) values('G',329) into colors(name, category) values('G',330) into colors(name, category) values('G',331) into colors(name, category) values('G',332) into colors(name, category) values('G',333) into colors(name, category) values('G',334) into colors(name, category) values('G',335) into colors(name, category) values('G',336) into colors(name, category) values('G',337) in
 to colors(name, category) values('G',338) into colors(name, category) values('G',339) into colors(name, category) values('G',340) into colors(name, category) values('G',341) into colors(name, category) values('G',342) into colors(name, category) values('G',343) into colors(name, category) values('G',344) into colors(name, category) values('G',345) into colors(name, category) values('G',346) into colors(name, category) values('G',347) into colors(name, category) values('G',348) into colors(name, category) values('G',349) into colors(name, category) values('G',350) into colors(name, category) values('G',351) into colors(name, category) values('G',352) into colors(name, category) values('G',353) into colors(name, category) values('G',354) into colors(name, category) values('G',355) into colors(name, category) values('G',356) into colors(name, category) values('G',357) into colors(name, category) values('G',358) into colors(name, category) values('G',359) into colors(name, category) val
 ues('G',360) into colors(name, category) values('G',361) into colors(name, category) values('G',362) into colors(name, category) values('G',363) into colors(name, category) values('G',364) into colors(name, category) values('G',365) into colors(name, category) values('G',366) into colors(name, category) values('G',367) into colors(name, category) values('G',368) into colors(name, category) values('G',369) into colors(name, category) values('G',370) into colors(name, category) values('G',371) into colors(name, category) values('G',372) into colors(name, category) values('G',373) into colors(name, category) values('G',374) into colors(name, category) values('G',375) into colors(name, category) values('G',376) into colors(name, category) values('G',377) into colors(name, category) values('G',378) into colors(name, category) values('G',379) into colors(name, category) values('G',380) into colors(name, category) values('G',381) into colors(name, category) values('G',382) into colors(name
 , category) values('G',383) into colors(name, category) values('G',384) into colors(name, category) values('G',385) into colors(name, category) values('G',386) into colors(name, category) values('G',387) into colors(name, category) values('G',388) into colors(name, category) values('G',389) into colors(name, category) values('G',390) into colors(name, category) values('G',391) into colors(name, category) values('G',392) into colors(name, category) values('G',393) into colors(name, category) values('G',394) into colors(name, category) values('G',395) into colors(name, category) values('G',396) into colors(name, category) values('G',397) into colors(name, category) values('G',398) into colors(name, category) values('G',399) into colors(name, category) values('G',400) into colors(name, category) values('G',401) into colors(name, category) values('G',402) into colors(name, category) values('G',403) into colors(name, category) values('G',404) into colors(name, category) values('G',405) i
 nto colors(name, category) values('G',406) into colors(name, category) values('G',407) into colors(name, category) values('G',408) into colors(name, category) values('G',409) into colors(name, category) values('G',410) into colors(name, category) values('G',411) into colors(name, category) values('G',412) into colors(name, category) values('G',413) into colors(name, category) values('G',414) into colors(name, category) values('G',415) into colors(name, category) values('G',416) into colors(name, category) values('G',417) into colors(name, category) values('G',418) into colors(name, category) values('G',419) into colors(name, category) values('G',420) into colors(name, category) values('G',421) into colors(name, category) values('G',422) into colors(name, category) values('G',423) into colors(name, category) values('G',424) into colors(name, category) values('G',425) into colors(name, category) values('G',426) into colors(name, category) values('G',427) into colors(name, category) va
 lues('G',428) into colors(name, category) values('G',429) into colors(name, category) values('G',430) into colors(name, category) values('G',431) into colors(name, category) values('G',432) into colors(name, category) values('G',433) into colors(name, category) values('G',434) into colors(name, category) values('G',435) into colors(name, category) values('G',436) into colors(name, category) values('G',437) into colors(name, category) values('G',438) into colors(name, category) values('G',439) into colors(name, category) values('G',440) into colors(name, category) values('G',441) into colors(name, category) values('G',442) into colors(name, category) values('G',443) into colors(name, category) values('G',444) into colors(name, category) values('G',445) into colors(name, category) values('G',446) into colors(name, category) values('G',447) into colors(name, category) values('G',448) into colors(name, category) values('G',449) into colors(name, category) values('G',450) into colors(nam
 e, category) values('G',451) into colors(name, category) values('G',452) into colors(name, category) values('G',453) into colors(name, category) values('G',454) into colors(name, category) values('G',455) into colors(name, category) values('G',456) into colors(name, category) values('G',457) into colors(name, category) values('G',458) into colors(name, category) values('G',459) into colors(name, category) values('G',460) into colors(name, category) values('G',461) into colors(name, category) values('G',462) into colors(name, category) values('G',463) into colors(name, category) values('G',464) into colors(name, category) values('G',465) into colors(name, category) values('G',466) into colors(name, category) values('G',467) into colors(name, category) values('G',468) into colors(name, category) values('G',469) into colors(name, category) values('G',470) into colors(name, category) values('G',471) into colors(name, category) values('G',472) into colors(name, category) values('G',473) 
 into colors(name, category) values('G',474) into colors(name, category) values('G',475) into colors(name, category) values('G',476) into colors(name, category) values('G',477) into colors(name, category) values('G',478) into colors(name, category) values('G',479) into colors(name, category) values('G',480) into colors(name, category) values('G',481) into colors(name, category) values('G',482) into colors(name, category) values('G',483) into colors(name, category) values('G',484) into colors(name, category) values('G',485) into colors(name, category) values('G',486) into colors(name, category) values('G',487) into colors(name, category) values('G',488) into colors(name, category) values('G',489) into colors(name, category) values('G',490) into colors(name, category) values('G',491) into colors(name, category) values('G',492) into colors(name, category) values('G',493) into colors(name, category) values('G',494) into colors(name, category) values('G',495) into colors(name, category) v
 alues('G',496) into colors(name, category) values('G',497) into colors(name, category) values('G',498) into colors(name, category) values('G',499) into colors(name, category) values('G',500) into colors(name, category) values('G',501) into colors(name, category) values('G',502) into colors(name, category) values('G',503) into colors(name, category) values('G',504) into colors(name, category) values('G',505) into colors(name, category) values('G',506) into colors(name, category) values('G',507) into colors(name, category) values('G',508) into colors(name, category) values('G',509) into colors(name, category) values('G',510) into colors(name, category) values('G',511) into colors(name, category) values('G',512) into colors(name, category) values('G',513) into colors(name, category) values('G',514) into colors(name, category) values('G',515) into colors(name, category) values('G',516) into colors(name, category) values('G',517) into colors(name, category) values('G',518) into colors(na
 me, category) values('G',519) into colors(name, category) values('G',520) into colors(name, category) values('G',521) into colors(name, category) values('G',522) into colors(name, category) values('G',523) into colors(name, category) values('G',524) into colors(name, category) values('G',525) into colors(name, category) values('G',526) into colors(name, category) values('G',527) into colors(name, category) values('G',528) into colors(name, category) values('G',529) into colors(name, category) values('G',530) into colors(name, category) values('G',531) into colors(name, category) values('G',532) into colors(name, category) values('G',533) into colors(name, category) values('G',534) into colors(name, category) values('G',535) into colors(name, category) values('G',536) into colors(name, category) values('G',537) into colors(name, category) values('G',538) into colors(name, category) values('G',539) into colors(name, category) values('G',540) into colors(name, category) values('G',541)
  into colors(name, category) values('G',542) into colors(name, category) values('G',543) into colors(name, category) values('G',544) into colors(name, category) values('G',545) into colors(name, category) values('G',546) into colors(name, category) values('G',547) into colors(name, category) values('G',548) into colors(name, category) values('G',549) into colors(name, category) values('G',550) into colors(name, category) values('G',551) into colors(name, category) values('G',552) into colors(name, category) values('G',553) into colors(name, category) values('G',554) into colors(name, category) values('G',555) into colors(name, category) values('G',556) into colors(name, category) values('G',557) into colors(name, category) values('G',558) into colors(name, category) values('G',559) into colors(name, category) values('G',560) into colors(name, category) values('G',561) into colors(name, category) values('G',562) into colors(name, category) values('G',563) into colors(name, category) 
 values('G',564) into colors(name, category) values('G',565) into colors(name, category) values('G',566) into colors(name, category) values('G',567) into colors(name, category) values('G',568) into colors(name, category) values('G',569) into colors(name, category) values('G',570) into colors(name, category) values('G',571) into colors(name, category) values('G',572) into colors(name, category) values('G',573) into colors(name, category) values('G',574) into colors(name, category) values('G',575) into colors(name, category) values('G',576) into colors(name, category) values('G',577) into colors(name, category) values('G',578) into colors(name, category) values('G',579) into colors(name, category) values('G',580) into colors(name, category) values('G',581) into colors(name, category) values('G',582) into colors(name, category) values('G',583) into colors(name, category) values('G',584) into colors(name, category) values('G',585) into colors(name, category) values('G',586) into colors(n
 ame, category) values('G',587) into colors(name, category) values('G',588) into colors(name, category) values('G',589) into colors(name, category) values('G',590) into colors(name, category) values('G',591) into colors(name, category) values('G',592) into colors(name, category) values('G',593) into colors(name, category) values('G',594) into colors(name, category) values('G',595) into colors(name, category) values('G',596) into colors(name, category) values('G',597) into colors(name, category) values('G',598) into colors(name, category) values('G',599) into colors(name, category) values('G',600) into colors(name, category) values('G',601) into colors(name, category) values('G',602) into colors(name, category) values('G',603) into colors(name, category) values('G',604) into colors(name, category) values('G',605) into colors(name, category) values('G',606) into colors(name, category) values('G',607) into colors(name, category) values('G',608) into colors(name, category) values('G',609
 ) into colors(name, category) values('G',610) into colors(name, category) values('G',611) into colors(name, category) values('G',612) into colors(name, category) values('G',613) into colors(name, category) values('G',614) into colors(name, category) values('G',615) into colors(name, category) values('G',616) into colors(name, category) values('G',617) into colors(name, category) values('G',618) into colors(name, category) values('G',619) into colors(name, category) values('G',620) into colors(name, category) values('G',621) into colors(name, category) values('G',622) into colors(name, category) values('G',623) into colors(name, category) values('G',624) into colors(name, category) values('G',625) into colors(name, category) values('G',626) into colors(name, category) values('G',627) into colors(name, category) values('G',628) into colors(name, category) values('G',629) into colors(name, category) values('G',630) into colors(name, category) values('G',631) into colors(name, category)
  values('G',632) into colors(name, category) values('G',633) into colors(name, category) values('G',634) into colors(name, category) values('G',635) into colors(name, category) values('G',636) into colors(name, category) values('G',637) into colors(name, category) values('G',638) into colors(name, category) values('G',639) into colors(name, category) values('G',640) into colors(name, category) values('G',641) into colors(name, category) values('G',642) into colors(name, category) values('G',643) into colors(name, category) values('G',644) into colors(name, category) values('G',645) into colors(name, category) values('G',646) into colors(name, category) values('G',647) into colors(name, category) values('G',648) into colors(name, category) values('G',649) into colors(name, category) values('G',650) into colors(name, category) values('G',651) into colors(name, category) values('G',652) into colors(name, category) values('G',653) into colors(name, category) values('G',654) into colors(
 name, category) values('G',655) into colors(name, category) values('G',656) into colors(name, category) values('G',657) into colors(name, category) values('G',658) into colors(name, category) values('G',659) into colors(name, category) values('G',660) into colors(name, category) values('G',661) into colors(name, category) values('G',662) into colors(name, category) values('G',663) into colors(name, category) values('G',664) into colors(name, category) values('G',665) into colors(name, category) values('G',666) into colors(name, category) values('G',667) into colors(name, category) values('G',668) into colors(name, category) values('G',669) into colors(name, category) values('G',670) into colors(name, category) values('G',671) into colors(name, category) values('G',672) into colors(name, category) values('G',673) into colors(name, category) values('G',674) into colors(name, category) values('G',675) into colors(name, category) values('G',676) into colors(name, category) values('G',67
 7) into colors(name, category) values('G',678) into colors(name, category) values('G',679) into colors(name, category) values('G',680) into colors(name, category) values('G',681) into colors(name, category) values('G',682) into colors(name, category) values('G',683) into colors(name, category) values('G',684) into colors(name, category) values('G',685) into colors(name, category) values('G',686) into colors(name, category) values('G',687) into colors(name, category) values('G',688) into colors(name, category) values('G',689) into colors(name, category) values('G',690) into colors(name, category) values('G',691) into colors(name, category) values('G',692) into colors(name, category) values('G',693) into colors(name, category) values('G',694) into colors(name, category) values('G',695) into colors(name, category) values('G',696) into colors(name, category) values('G',697) into colors(name, category) values('G',698) into colors(name, category) values('G',699) into colors(name, category
 ) values('G',700) into colors(name, category) values('G',701) into colors(name, category) values('G',702) into colors(name, category) values('G',703) into colors(name, category) values('G',704) into colors(name, category) values('G',705) into colors(name, category) values('G',706) into colors(name, category) values('G',707) into colors(name, category) values('G',708) into colors(name, category) values('G',709) into colors(name, category) values('G',710) into colors(name, category) values('G',711) into colors(name, category) values('G',712) into colors(name, category) values('G',713) into colors(name, category) values('G',714) into colors(name, category) values('G',715) into colors(name, category) values('G',716) into colors(name, category) values('G',717) into colors(name, category) values('G',718) into colors(name, category) values('G',719) into colors(name, category) values('G',720) into colors(name, category) values('G',721) into colors(name, category) values('G',722) into colors
 (name, category) values('G',723) into colors(name, category) values('G',724) into colors(name, category) values('G',725) into colors(name, category) values('G',726) into colors(name, category) values('G',727) into colors(name, category) values('G',728) into colors(name, category) values('G',729) into colors(name, category) values('G',730) into colors(name, category) values('G',731) into colors(name, category) values('G',732) into colors(name, category) values('G',733) into colors(name, category) values('G',734) into colors(name, category) values('G',735) into colors(name, category) values('G',736) into colors(name, category) values('G',737) into colors(name, category) values('G',738) into colors(name, category) values('G',739) into colors(name, category) values('G',740) into colors(name, category) values('G',741) into colors(name, category) values('G',742) into colors(name, category) values('G',743) into colors(name, category) values('G',744) into colors(name, category) values('G',7
 45) into colors(name, category) values('G',746) into colors(name, category) values('G',747) into colors(name, category) values('G',748) into colors(name, category) values('G',749) into colors(name, category) values('G',750) into colors(name, category) values('G',751) into colors(name, category) values('G',752) into colors(name, category) values('G',753) into colors(name, category) values('G',754) into colors(name, category) values('G',755) into colors(name, category) values('G',756) into colors(name, category) values('G',757) into colors(name, category) values('G',758) into colors(name, category) values('G',759) into colors(name, category) values('G',760) into colors(name, category) values('G',761) into colors(name, category) values('G',762) into colors(name, category) values('G',763) into colors(name, category) values('G',764) into colors(name, category) values('G',765) into colors(name, category) values('G',766) into colors(name, category) values('G',767) into colors(name, categor
 y) values('G',768) into colors(name, category) values('G',769) into colors(name, category) values('G',770) into colors(name, category) values('G',771) into colors(name, category) values('G',772) into colors(name, category) values('G',773) into colors(name, category) values('G',774) into colors(name, category) values('G',775) into colors(name, category) values('G',776) into colors(name, category) values('G',777) into colors(name, category) values('G',778) into colors(name, category) values('G',779) into colors(name, category) values('G',780) into colors(name, category) values('G',781) into colors(name, category) values('G',782) into colors(name, category) values('G',783) into colors(name, category) values('G',784) into colors(name, category) values('G',785) into colors(name, category) values('G',786) into colors(name, category) values('G',787) into colors(name, category) values('G',788) into colors(name, category) values('G',789) into colors(name, category) values('G',790) into color
 s(name, category) values('G',791) into colors(name, category) values('G',792) into colors(name, category) values('G',793) into colors(name, category) values('G',794) into colors(name, category) values('G',795) into colors(name, category) values('G',796) into colors(name, category) values('G',797) into colors(name, category) values('G',798) into colors(name, category) values('G',799) into colors(name, category) values('G',800) into colors(name, category) values('G',801) into colors(name, category) values('G',802) into colors(name, category) values('G',803) into colors(name, category) values('G',804) into colors(name, category) values('G',805) into colors(name, category) values('G',806) into colors(name, category) values('G',807) into colors(name, category) values('G',808) into colors(name, category) values('G',809) into colors(name, category) values('G',810) into colors(name, category) values('G',811) into colors(name, category) values('G',812) into colors(name, category) values('G',
 813) into colors(name, category) values('G',814) into colors(name, category) values('G',815) into colors(name, category) values('G',816) into colors(name, category) values('G',817) into colors(name, category) values('G',818) into colors(name, category) values('G',819) into colors(name, category) values('G',820) into colors(name, category) values('G',821) into colors(name, category) values('G',822) into colors(name, category) values('G',823) into colors(name, category) values('G',824) into colors(name, category) values('G',825) into colors(name, category) values('G',826) into colors(name, category) values('G',827) into colors(name, category) values('G',828) into colors(name, category) values('G',829) into colors(name, category) values('G',830) into colors(name, category) values('G',831) into colors(name, category) values('G',832) into colors(name, category) values('G',833) into colors(name, category) values('G',834) into colors(name, category) values('G',835) into colors(name, catego
 ry) values('G',836) into colors(name, category) values('G',837) into colors(name, category) values('G',838) into colors(name, category) values('G',839) into colors(name, category) values('G',840) into colors(name, category) values('G',841) into colors(name, category) values('G',842) into colors(name, category) values('G',843) into colors(name, category) values('G',844) into colors(name, category) values('G',845) into colors(name, category) values('G',846) into colors(name, category) values('G',847) into colors(name, category) values('G',848) into colors(name, category) values('G',849) into colors(name, category) values('G',850) into colors(name, category) values('G',851) into colors(name, category) values('G',852) into colors(name, category) values('G',853) into colors(name, category) values('G',854) into colors(name, category) values('G',855) into colors(name, category) values('G',856) into colors(name, category) values('G',857) into colors(name, category) values('G',858) into colo
 rs(name, category) values('G',859) into colors(name, category) values('G',860) into colors(name, category) values('G',861) into colors(name, category) values('G',862) into colors(name, category) values('G',863) into colors(name, category) values('G',864) into colors(name, category) values('G',865) into colors(name, category) values('G',866) into colors(name, category) values('G',867) into colors(name, category) values('G',868) into colors(name, category) values('G',869) into colors(name, category) values('G',870) into colors(name, category) values('G',871) into colors(name, category) values('G',872) into colors(name, category) values('G',873) into colors(name, category) values('G',874) into colors(name, category) values('G',875) into colors(name, category) values('G',876) into colors(name, category) values('G',877) into colors(name, category) values('G',878) into colors(name, category) values('G',879) into colors(name, category) values('G',880) into colors(name, category) values('G'
 ,881) into colors(name, category) values('G',882) into colors(name, category) values('G',883) into colors(name, category) values('G',884) into colors(name, category) values('G',885) into colors(name, category) values('G',886) into colors(name, category) values('G',887) into colors(name, category) values('G',888) into colors(name, category) values('G',889) into colors(name, category) values('G',890) into colors(name, category) values('G',891) into colors(name, category) values('G',892) into colors(name, category) values('G',893) into colors(name, category) values('G',894) into colors(name, category) values('G',895) into colors(name, category) values('G',896) into colors(name, category) values('G',897) into colors(name, category) values('G',898) into colors(name, category) values('G',899) into colors(name, category) values('G',900) into colors(name, category) values('G',901) into colors(name, category) values('G',902) into colors(name, category) values('G',903) into colors(name, categ
 ory) values('G',904) into colors(name, category) values('G',905) into colors(name, category) values('G',906) into colors(name, category) values('G',907) into colors(name, category) values('G',908) into colors(name, category) values('G',909) into colors(name, category) values('G',910) into colors(name, category) values('G',911) into colors(name, category) values('G',912) into colors(name, category) values('G',913) into colors(name, category) values('G',914) into colors(name, category) values('G',915) into colors(name, category) values('G',916) into colors(name, category) values('G',917) into colors(name, category) values('G',918) into colors(name, category) values('G',919) into colors(name, category) values('G',920) into colors(name, category) values('G',921) into colors(name, category) values('G',922) into colors(name, category) values('G',923) into colors(name, category) values('G',924) into colors(name, category) values('G',925) into colors(name, category) values('G',926) into col
 ors(name, category) values('G',927) into colors(name, category) values('G',928) into colors(name, category) values('G',929) into colors(name, category) values('G',930) into colors(name, category) values('G',931) into colors(name, category) values('G',932) into colors(name, category) values('G',933) into colors(name, category) values('G',934) into colors(name, category) values('G',935) into colors(name, category) values('G',936) into colors(name, category) values('G',937) into colors(name, category) values('G',938) into colors(name, category) values('G',939) into colors(name, category) values('G',940) into colors(name, category) values('G',941) into colors(name, category) values('G',942) into colors(name, category) values('G',943) into colors(name, category) values('G',944) into colors(name, category) values('G',945) into colors(name, category) values('G',946) into colors(name, category) values('G',947) into colors(name, category) values('G',948) into colors(name, category) values('G
 ',949) into colors(name, category) values('G',950) into colors(name, category) values('G',951) into colors(name, category) values('G',952) into colors(name, category) values('G',953) into colors(name, category) values('G',954) into colors(name, category) values('G',955) into colors(name, category) values('G',956) into colors(name, category) values('G',957) into colors(name, category) values('G',958) into colors(name, category) values('G',959) into colors(name, category) values('G',960) into colors(name, category) values('G',961) into colors(name, category) values('G',962) into colors(name, category) values('G',963) into colors(name, category) values('G',964) into colors(name, category) values('G',965) into colors(name, category) values('G',966) into colors(name, category) values('G',967) into colors(name, category) values('G',968) into colors(name, category) values('G',969) into colors(name, category) values('G',970) into colors(name, category) values('G',971) into colors(name, cate
 gory) values('G',972) into colors(name, category) values('G',973) into colors(name, category) values('G',974) into colors(name, category) values('G',975) into colors(name, category) values('G',976) into colors(name, category) values('G',977) into colors(name, category) values('G',978) into colors(name, category) values('G',979) into colors(name, category) values('G',980) into colors(name, category) values('G',981) into colors(name, category) values('G',982) into colors(name, category) values('G',983) into colors(name, category) values('G',984) into colors(name, category) values('G',985) into colors(name, category) values('G',986) into colors(name, category) values('G',987) into colors(name, category) values('G',988) into colors(name, category) values('G',989) into colors(name, category) values('G',990) into colors(name, category) values('G',991) into colors(name, category) values('G',992) into colors(name, category) values('G',993) into colors(name, category) values('G',994) into co
 lors(name, category) values('G',995) into colors(name, category) values('G',996)  select * from dual", sql.get(0));
-    Assert.assertEquals("Wrong stmt", "insert all into colors(name, category) values('G',997) into colors(name, category) values('G',998) into colors(name, category) values('G',999) select * from dual", sql.get(1));
-    
-    sqlGenerator =
-      new SQLGenerator(DatabaseProduct.MYSQL, conf);
-    rows.clear();
-    rows.add("'yellow', 1");
-    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 1, sql.size());
-    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1)", sql.get(0));
-    rows.add("'red', 2");
-    rows.add("'orange', 3");
-    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 1, sql.size());
-    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1),('red', 2),('orange', 3)", sql.get(0));
-    for(int i = 0; i < conf.getIntVar(HiveConf.ConfVars.METASTORE_DIRECT_SQL_MAX_ELEMENTS_VALUES_CLAUSE); i++) {
-      rows.add("\'G\'," + i);
-    }
-    sql = sqlGenerator.createInsertValuesStmt("colors(name, category)", rows);
-    Assert.assertEquals("Number of stmts", 2, sql.size());
-    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('yellow', 1),('red', 2),('orange', 3),('G',0),('G',1),('G',2),('G',3),('G',4),('G',5),('G',6),('G',7),('G',8),('G',9),('G',10),('G',11),('G',12),('G',13),('G',14),('G',15),('G',16),('G',17),('G',18),('G',19),('G',20),('G',21),('G',22),('G',23),('G',24),('G',25),('G',26),('G',27),('G',28),('G',29),('G',30),('G',31),('G',32),('G',33),('G',34),('G',35),('G',36),('G',37),('G',38),('G',39),('G',40),('G',41),('G',42),('G',43),('G',44),('G',45),('G',46),('G',47),('G',48),('G',49),('G',50),('G',51),('G',52),('G',53),('G',54),('G',55),('G',56),('G',57),('G',58),('G',59),('G',60),('G',61),('G',62),('G',63),('G',64),('G',65),('G',66),('G',67),('G',68),('G',69),('G',70),('G',71),('G',72),('G',73),('G',74),('G',75),('G',76),('G',77),('G',78),('G',79),('G',80),('G',81),('G',82),('G',83),('G',84),('G',85),('G',86),('G',87),('G',88),('G',89),('G',90),('G',91),('G',92),('G',93),('G',94),('G',95),('G',96),('G',97),('G',9
 8),('G',99),('G',100),('G',101),('G',102),('G',103),('G',104),('G',105),('G',106),('G',107),('G',108),('G',109),('G',110),('G',111),('G',112),('G',113),('G',114),('G',115),('G',116),('G',117),('G',118),('G',119),('G',120),('G',121),('G',122),('G',123),('G',124),('G',125),('G',126),('G',127),('G',128),('G',129),('G',130),('G',131),('G',132),('G',133),('G',134),('G',135),('G',136),('G',137),('G',138),('G',139),('G',140),('G',141),('G',142),('G',143),('G',144),('G',145),('G',146),('G',147),('G',148),('G',149),('G',150),('G',151),('G',152),('G',153),('G',154),('G',155),('G',156),('G',157),('G',158),('G',159),('G',160),('G',161),('G',162),('G',163),('G',164),('G',165),('G',166),('G',167),('G',168),('G',169),('G',170),('G',171),('G',172),('G',173),('G',174),('G',175),('G',176),('G',177),('G',178),('G',179),('G',180),('G',181),('G',182),('G',183),('G',184),('G',185),('G',186),('G',187),('G',188),('G',189),('G',190),('G',191),('G',192),('G',193),('G',194),('G',195),('G',196),('G',197),('G',
 198),('G',199),('G',200),('G',201),('G',202),('G',203),('G',204),('G',205),('G',206),('G',207),('G',208),('G',209),('G',210),('G',211),('G',212),('G',213),('G',214),('G',215),('G',216),('G',217),('G',218),('G',219),('G',220),('G',221),('G',222),('G',223),('G',224),('G',225),('G',226),('G',227),('G',228),('G',229),('G',230),('G',231),('G',232),('G',233),('G',234),('G',235),('G',236),('G',237),('G',238),('G',239),('G',240),('G',241),('G',242),('G',243),('G',244),('G',245),('G',246),('G',247),('G',248),('G',249),('G',250),('G',251),('G',252),('G',253),('G',254),('G',255),('G',256),('G',257),('G',258),('G',259),('G',260),('G',261),('G',262),('G',263),('G',264),('G',265),('G',266),('G',267),('G',268),('G',269),('G',270),('G',271),('G',272),('G',273),('G',274),('G',275),('G',276),('G',277),('G',278),('G',279),('G',280),('G',281),('G',282),('G',283),('G',284),('G',285),('G',286),('G',287),('G',288),('G',289),('G',290),('G',291),('G',292),('G',293),('G',294),('G',295),('G',296),('G',297),('
 G',298),('G',299),('G',300),('G',301),('G',302),('G',303),('G',304),('G',305),('G',306),('G',307),('G',308),('G',309),('G',310),('G',311),('G',312),('G',313),('G',314),('G',315),('G',316),('G',317),('G',318),('G',319),('G',320),('G',321),('G',322),('G',323),('G',324),('G',325),('G',326),('G',327),('G',328),('G',329),('G',330),('G',331),('G',332),('G',333),('G',334),('G',335),('G',336),('G',337),('G',338),('G',339),('G',340),('G',341),('G',342),('G',343),('G',344),('G',345),('G',346),('G',347),('G',348),('G',349),('G',350),('G',351),('G',352),('G',353),('G',354),('G',355),('G',356),('G',357),('G',358),('G',359),('G',360),('G',361),('G',362),('G',363),('G',364),('G',365),('G',366),('G',367),('G',368),('G',369),('G',370),('G',371),('G',372),('G',373),('G',374),('G',375),('G',376),('G',377),('G',378),('G',379),('G',380),('G',381),('G',382),('G',383),('G',384),('G',385),('G',386),('G',387),('G',388),('G',389),('G',390),('G',391),('G',392),('G',393),('G',394),('G',395),('G',396),('G',397)
 ,('G',398),('G',399),('G',400),('G',401),('G',402),('G',403),('G',404),('G',405),('G',406),('G',407),('G',408),('G',409),('G',410),('G',411),('G',412),('G',413),('G',414),('G',415),('G',416),('G',417),('G',418),('G',419),('G',420),('G',421),('G',422),('G',423),('G',424),('G',425),('G',426),('G',427),('G',428),('G',429),('G',430),('G',431),('G',432),('G',433),('G',434),('G',435),('G',436),('G',437),('G',438),('G',439),('G',440),('G',441),('G',442),('G',443),('G',444),('G',445),('G',446),('G',447),('G',448),('G',449),('G',450),('G',451),('G',452),('G',453),('G',454),('G',455),('G',456),('G',457),('G',458),('G',459),('G',460),('G',461),('G',462),('G',463),('G',464),('G',465),('G',466),('G',467),('G',468),('G',469),('G',470),('G',471),('G',472),('G',473),('G',474),('G',475),('G',476),('G',477),('G',478),('G',479),('G',480),('G',481),('G',482),('G',483),('G',484),('G',485),('G',486),('G',487),('G',488),('G',489),('G',490),('G',491),('G',492),('G',493),('G',494),('G',495),('G',496),('G',4
 97),('G',498),('G',499),('G',500),('G',501),('G',502),('G',503),('G',504),('G',505),('G',506),('G',507),('G',508),('G',509),('G',510),('G',511),('G',512),('G',513),('G',514),('G',515),('G',516),('G',517),('G',518),('G',519),('G',520),('G',521),('G',522),('G',523),('G',524),('G',525),('G',526),('G',527),('G',528),('G',529),('G',530),('G',531),('G',532),('G',533),('G',534),('G',535),('G',536),('G',537),('G',538),('G',539),('G',540),('G',541),('G',542),('G',543),('G',544),('G',545),('G',546),('G',547),('G',548),('G',549),('G',550),('G',551),('G',552),('G',553),('G',554),('G',555),('G',556),('G',557),('G',558),('G',559),('G',560),('G',561),('G',562),('G',563),('G',564),('G',565),('G',566),('G',567),('G',568),('G',569),('G',570),('G',571),('G',572),('G',573),('G',574),('G',575),('G',576),('G',577),('G',578),('G',579),('G',580),('G',581),('G',582),('G',583),('G',584),('G',585),('G',586),('G',587),('G',588),('G',589),('G',590),('G',591),('G',592),('G',593),('G',594),('G',595),('G',596),('G
 ',597),('G',598),('G',599),('G',600),('G',601),('G',602),('G',603),('G',604),('G',605),('G',606),('G',607),('G',608),('G',609),('G',610),('G',611),('G',612),('G',613),('G',614),('G',615),('G',616),('G',617),('G',618),('G',619),('G',620),('G',621),('G',622),('G',623),('G',624),('G',625),('G',626),('G',627),('G',628),('G',629),('G',630),('G',631),('G',632),('G',633),('G',634),('G',635),('G',636),('G',637),('G',638),('G',639),('G',640),('G',641),('G',642),('G',643),('G',644),('G',645),('G',646),('G',647),('G',648),('G',649),('G',650),('G',651),('G',652),('G',653),('G',654),('G',655),('G',656),('G',657),('G',658),('G',659),('G',660),('G',661),('G',662),('G',663),('G',664),('G',665),('G',666),('G',667),('G',668),('G',669),('G',670),('G',671),('G',672),('G',673),('G',674),('G',675),('G',676),('G',677),('G',678),('G',679),('G',680),('G',681),('G',682),('G',683),('G',684),('G',685),('G',686),('G',687),('G',688),('G',689),('G',690),('G',691),('G',692),('G',693),('G',694),('G',695),('G',696),
 ('G',697),('G',698),('G',699),('G',700),('G',701),('G',702),('G',703),('G',704),('G',705),('G',706),('G',707),('G',708),('G',709),('G',710),('G',711),('G',712),('G',713),('G',714),('G',715),('G',716),('G',717),('G',718),('G',719),('G',720),('G',721),('G',722),('G',723),('G',724),('G',725),('G',726),('G',727),('G',728),('G',729),('G',730),('G',731),('G',732),('G',733),('G',734),('G',735),('G',736),('G',737),('G',738),('G',739),('G',740),('G',741),('G',742),('G',743),('G',744),('G',745),('G',746),('G',747),('G',748),('G',749),('G',750),('G',751),('G',752),('G',753),('G',754),('G',755),('G',756),('G',757),('G',758),('G',759),('G',760),('G',761),('G',762),('G',763),('G',764),('G',765),('G',766),('G',767),('G',768),('G',769),('G',770),('G',771),('G',772),('G',773),('G',774),('G',775),('G',776),('G',777),('G',778),('G',779),('G',780),('G',781),('G',782),('G',783),('G',784),('G',785),('G',786),('G',787),('G',788),('G',789),('G',790),('G',791),('G',792),('G',793),('G',794),('G',795),('G',79
 6),('G',797),('G',798),('G',799),('G',800),('G',801),('G',802),('G',803),('G',804),('G',805),('G',806),('G',807),('G',808),('G',809),('G',810),('G',811),('G',812),('G',813),('G',814),('G',815),('G',816),('G',817),('G',818),('G',819),('G',820),('G',821),('G',822),('G',823),('G',824),('G',825),('G',826),('G',827),('G',828),('G',829),('G',830),('G',831),('G',832),('G',833),('G',834),('G',835),('G',836),('G',837),('G',838),('G',839),('G',840),('G',841),('G',842),('G',843),('G',844),('G',845),('G',846),('G',847),('G',848),('G',849),('G',850),('G',851),('G',852),('G',853),('G',854),('G',855),('G',856),('G',857),('G',858),('G',859),('G',860),('G',861),('G',862),('G',863),('G',864),('G',865),('G',866),('G',867),('G',868),('G',869),('G',870),('G',871),('G',872),('G',873),('G',874),('G',875),('G',876),('G',877),('G',878),('G',879),('G',880),('G',881),('G',882),('G',883),('G',884),('G',885),('G',886),('G',887),('G',888),('G',889),('G',890),('G',891),('G',892),('G',893),('G',894),('G',895),('G'
 ,896),('G',897),('G',898),('G',899),('G',900),('G',901),('G',902),('G',903),('G',904),('G',905),('G',906),('G',907),('G',908),('G',909),('G',910),('G',911),('G',912),('G',913),('G',914),('G',915),('G',916),('G',917),('G',918),('G',919),('G',920),('G',921),('G',922),('G',923),('G',924),('G',925),('G',926),('G',927),('G',928),('G',929),('G',930),('G',931),('G',932),('G',933),('G',934),('G',935),('G',936),('G',937),('G',938),('G',939),('G',940),('G',941),('G',942),('G',943),('G',944),('G',945),('G',946),('G',947),('G',948),('G',949),('G',950),('G',951),('G',952),('G',953),('G',954),('G',955),('G',956),('G',957),('G',958),('G',959),('G',960),('G',961),('G',962),('G',963),('G',964),('G',965),('G',966),('G',967),('G',968),('G',969),('G',970),('G',971),('G',972),('G',973),('G',974),('G',975),('G',976),('G',977),('G',978),('G',979),('G',980),('G',981),('G',982),('G',983),('G',984),('G',985),('G',986),('G',987),('G',988),('G',989),('G',990),('G',991),('G',992),('G',993),('G',994),('G',995),(
 'G',996)", sql.get(0));
-    Assert.assertEquals("Wrong stmt", "insert into colors(name, category) values('G',997),('G',998),('G',999)", sql.get(1));
-
-    sqlGenerator = new SQLGenerator(DatabaseProduct.SQLSERVER, conf);
-    String modSql = sqlGenerator.addForUpdateClause("select nl_next from NEXT_LOCK_ID");
-    Assert.assertEquals("select nl_next from NEXT_LOCK_ID with (updlock)", modSql);
-    modSql = sqlGenerator.addForUpdateClause("select MT_COMMENT from AUX_TABLE where MT_KEY1='CheckLock' and MT_KEY2=0");
-    Assert.assertEquals("select MT_COMMENT from AUX_TABLE with (updlock) where MT_KEY1='CheckLock' and MT_KEY2=0", modSql);
-  }
-
-  @Before
-  public void setUp() throws Exception {
-    tearDown();
-    conf = new HiveConf(this.getClass());
-    TxnDbUtil.setConfValues(conf);
-    TxnDbUtil.prepDb();
-  }
-
-  @After
-  public void tearDown() throws Exception {
-    TxnDbUtil.cleanDb();
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
deleted file mode 100644
index eb88e32..0000000
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.metastore.txn;
-
-import org.apache.hadoop.hive.common.ValidCompactorTxnList;
-import org.apache.hadoop.hive.common.ValidTxnList;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.BitSet;
-
-public class TestValidCompactorTxnList {
-
-  @Test
-  public void minTxnHigh() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 4}, bitSet, 2);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void maxTxnLow() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{13, 14}, bitSet, 12);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-  }
-
-  @Test
-  public void minTxnHighNoExceptions() {
-    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 5);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void maxTxnLowNoExceptions() {
-    ValidTxnList txns = new ValidCompactorTxnList(new long[0], new BitSet(), 15);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-  }
-
-  @Test
-  public void exceptionsAllBelow() {
-    BitSet bitSet = new BitSet(2);
-    bitSet.set(0, 2);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{3, 6}, bitSet, 3);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void exceptionsInMidst() {
-    BitSet bitSet = new BitSet(1);
-    bitSet.set(0, 1);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8}, bitSet, 7);
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-  @Test
-  public void exceptionsAbveHighWaterMark() {
-    BitSet bitSet = new BitSet(4);
-    bitSet.set(0, 4);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{8, 11, 17, 29}, bitSet, 15);
-    Assert.assertArrayEquals("", new long[]{8, 11}, txns.getInvalidTransactions());
-    ValidTxnList.RangeResponse rsp = txns.isTxnRangeValid(7, 9);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-    rsp = txns.isTxnRangeValid(12, 16);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-  }
-
-  @Test
-  public void writeToString() {
-    BitSet bitSet = new BitSet(4);
-    bitSet.set(0, 4);
-    ValidTxnList txns = new ValidCompactorTxnList(new long[]{7, 9, 10, Long.MAX_VALUE}, bitSet, 8);
-    Assert.assertEquals("8:" + Long.MAX_VALUE + "::7", txns.writeToString());
-    txns = new ValidCompactorTxnList();
-    Assert.assertEquals(Long.toString(Long.MAX_VALUE) + ":" + Long.MAX_VALUE + "::", txns.writeToString());
-    txns = new ValidCompactorTxnList(new long[0], new BitSet(), 23);
-    Assert.assertEquals("23:" + Long.MAX_VALUE + "::", txns.writeToString());
-  }
-
-  @Test
-  public void readFromString() {
-    ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10");
-    Assert.assertEquals(37L, txns.getHighWatermark());
-    Assert.assertNull(txns.getMinOpenTxn());
-    Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions());
-    txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
-    Assert.assertEquals(21L, txns.getHighWatermark());
-    Assert.assertNull(txns.getMinOpenTxn());
-    Assert.assertEquals(0, txns.getInvalidTransactions().length);
-  }
-
-  @Test
-  public void testAbortedTxn() throws Exception {
-    ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
-    Assert.assertEquals(5L, txnList.getHighWatermark());
-    Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
-    Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions());
-  }
-
-  @Test
-  public void testAbortedRange() throws Exception {
-    ValidCompactorTxnList txnList = new ValidCompactorTxnList("11:4::5,6,7,8");
-    ValidTxnList.RangeResponse rsp = txnList.isTxnRangeAborted(1L, 3L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-    rsp = txnList.isTxnRangeAborted(9L, 10L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.NONE, rsp);
-    rsp = txnList.isTxnRangeAborted(6L, 7L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.ALL, rsp);
-    rsp = txnList.isTxnRangeAborted(4L, 6L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
-    rsp = txnList.isTxnRangeAborted(6L, 13L);
-    Assert.assertEquals(ValidTxnList.RangeResponse.SOME, rsp);
-  }
-}

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
deleted file mode 100644
index 08fcff4..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/AcidOpenTxnsCounterService.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.ql.txn;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.txn.TxnStore;
-import org.apache.hadoop.hive.metastore.txn.TxnUtils;
-import org.apache.hadoop.hive.ql.txn.compactor.HouseKeeperServiceBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-/**
- * Background running thread, periodically updating number of open transactions.
- * Runs inside Hive Metastore Service.
- */
-public class AcidOpenTxnsCounterService extends HouseKeeperServiceBase {
-  private static final Logger LOG = LoggerFactory.getLogger(AcidOpenTxnsCounterService.class);
-  @Override
-  protected long getStartDelayMs() {
-    return 100;  // in miliseconds
-  }
-  @Override
-  protected long getIntervalMs() {
-    return hiveConf.getTimeVar(HiveConf.ConfVars.HIVE_COUNT_OPEN_TXNS_INTERVAL, TimeUnit.MILLISECONDS);
-  }
-  @Override
-  protected Runnable getScheduedAction(HiveConf hiveConf, AtomicInteger isAliveCounter) {
-    return new OpenTxnsCounter(hiveConf, isAliveCounter);
-  }
-  @Override
-  public String getServiceDescription() {
-    return "Count number of open transactions";
-  }
-  private static final class OpenTxnsCounter implements Runnable {
-    private static volatile long lastLogTime = 0;
-    private final TxnStore txnHandler;
-    private final AtomicInteger isAliveCounter;
-    private OpenTxnsCounter(HiveConf hiveConf, AtomicInteger isAliveCounter) {
-      txnHandler = TxnUtils.getTxnStore(hiveConf);
-      this.isAliveCounter = isAliveCounter;
-    }
-    @Override
-    public void run() {
-      try {
-        long startTime = System.currentTimeMillis();
-        txnHandler.countOpenTxns();
-        int count = isAliveCounter.incrementAndGet();
-        if(System.currentTimeMillis() - lastLogTime > 60*1000) {
-          //don't flood the logs with too many msgs
-          LOG.info("OpenTxnsCounter ran for " + (System.currentTimeMillis() - startTime) / 1000 +
-            "seconds.  isAliveCounter=" + count);
-          lastLogTime = System.currentTimeMillis();
-        }
-      }
-      catch(Throwable t) {
-        LOG.error("Serious error in {}", Thread.currentThread().getName(), ": {}" + t.getMessage(), t);
-      }
-    }
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index f8ae86b..96005b4 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -450,13 +450,13 @@ public class TestCompactionTxnHandler {
 
   @Before
   public void setUp() throws Exception {
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(conf);
     txnHandler = TxnUtils.getTxnStore(conf);
   }
 
   @After
   public void tearDown() throws Exception {
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
   }
 
   private long openTxn() throws MetaException {

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
index 23efce0..6f607bb 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandler.java
@@ -1543,13 +1543,13 @@ public class TestTxnHandler {
 
   @Before
   public void setUp() throws Exception {
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(conf);
     txnHandler = TxnUtils.getTxnStore(conf);
   }
 
   @After
   public void tearDown() throws Exception {
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
   }
 
   private long openTxn() throws MetaException {

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
index e5f4dde..a9cf3df 100644
--- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
+++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestTxnHandlerNoConnectionPool.java
@@ -50,18 +50,18 @@ public class TestTxnHandlerNoConnectionPool {
     conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_POOLING_TYPE, "None");
     TxnDbUtil.setConfValues(conf);
     try {
-      TxnDbUtil.prepDb();
+      TxnDbUtil.prepDb(conf);
     } catch (SQLException e) {
       // Usually this means we've already created the tables, so clean them and then try again
       tearDown();
-      TxnDbUtil.prepDb();
+      TxnDbUtil.prepDb(conf);
     }
     txnHandler = TxnUtils.getTxnStore(conf);
   }
 
   @After
   public void tearDown() throws Exception {
-    TxnDbUtil.cleanDb();
+    TxnDbUtil.cleanDb(conf);
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
index 4f1c7d8..2e53155 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands.java
@@ -71,25 +71,6 @@ public class TestTxnCommands extends TxnCommandsBaseForTests {
     return TEST_DATA_DIR;
   }
 
-  private void dropTables() throws Exception {
-    for(Table t : Table.values()) {
-      runStatementOnDriver("drop table if exists " + t);
-    }
-  }
-  @After
-  public void tearDown() throws Exception {
-    try {
-      if (d != null) {
-        dropTables();
-        d.destroy();
-        d.close();
-        d = null;
-      }
-    } finally {
-      TxnDbUtil.cleanDb();
-      FileUtils.deleteDirectory(new File(TEST_DATA_DIR));
-    }
-  }
   @Test//todo: what is this for?
   public void testInsertOverwrite() throws Exception {
     runStatementOnDriver("insert overwrite table " + Table.NONACIDORCTBL + " select a,b from " + Table.NONACIDORCTBL2);
@@ -388,7 +369,7 @@ public class TestTxnCommands extends TxnCommandsBaseForTests {
     Assert.assertNotNull(txnInfo);
     Assert.assertEquals(14, txnInfo.getId());
     Assert.assertEquals(TxnState.OPEN, txnInfo.getState());
-    String s =TxnDbUtil.queryToString("select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false);
+    String s =TxnDbUtil.queryToString(hiveConf, "select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false);
     String[] vals = s.split("\\s+");
     Assert.assertEquals("Didn't get expected timestamps", 2, vals.length);
     long lastHeartbeat = Long.parseLong(vals[1]);
@@ -412,7 +393,7 @@ public class TestTxnCommands extends TxnCommandsBaseForTests {
     TestDbTxnManager2.checkLock(LockType.SHARED_READ, LockState.ACQUIRED, "default", Table.ACIDTBL.name, null, slr.getLocks());
 
     //should've done several heartbeats
-    s =TxnDbUtil.queryToString("select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false);
+    s =TxnDbUtil.queryToString(hiveConf, "select TXN_STARTED, TXN_LAST_HEARTBEAT from TXNS where TXN_ID = " + txnInfo.getId(), false);
     vals = s.split("\\s+");
     Assert.assertEquals("Didn't get expected timestamps", 2, vals.length);
     Assert.assertTrue("Heartbeat didn't progress: (old,new) (" + lastHeartbeat + "," + vals[1]+ ")",

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index d08371d..f620283 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -55,7 +55,7 @@ import org.apache.hadoop.hive.ql.io.HiveInputFormat;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.txn.AcidCompactionHistoryService;
-import org.apache.hadoop.hive.ql.txn.AcidOpenTxnsCounterService;
+import org.apache.hadoop.hive.metastore.txn.AcidOpenTxnsCounterService;
 import org.apache.hadoop.hive.ql.txn.compactor.Cleaner;
 import org.apache.hadoop.hive.ql.txn.compactor.Initiator;
 import org.apache.hadoop.hive.ql.txn.compactor.Worker;
@@ -70,6 +70,10 @@ import org.junit.rules.TestName;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+/**
+ * TODO: this should be merged with TestTxnCommands once that is checked in
+ * specifically the tests; the supporting code here is just a clone of TestTxnCommands
+ */
 public class TestTxnCommands2 {
   static final private Logger LOG = LoggerFactory.getLogger(TestTxnCommands2.class);
   protected static final String TEST_DATA_DIR = new File(System.getProperty("java.io.tmpdir") +
@@ -118,7 +122,6 @@ public class TestTxnCommands2 {
   }
 
   protected void setUpWithTableProperties(String tableProperties) throws Exception {
-    tearDown();
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
@@ -131,7 +134,7 @@ public class TestTxnCommands2 {
     hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true);
 
     TxnDbUtil.setConfValues(hiveConf);
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(hiveConf);
     File f = new File(TEST_WAREHOUSE_DIR);
     if (f.exists()) {
       FileUtil.fullyDelete(f);
@@ -168,7 +171,7 @@ public class TestTxnCommands2 {
         d.close();
         d = null;
       }
-      TxnDbUtil.cleanDb();
+      TxnDbUtil.cleanDb(hiveConf);
     } finally {
       FileUtils.deleteDirectory(new File(TEST_DATA_DIR));
     }
@@ -1284,7 +1287,8 @@ public class TestTxnCommands2 {
     OpenTxnsResponse openTxnsResponse = txnHandler.openTxns(new OpenTxnRequest(3, "me", "localhost"));
 
     AcidOpenTxnsCounterService openTxnsCounterService = new AcidOpenTxnsCounterService();
-    runHouseKeeperService(openTxnsCounterService, hiveConf);  // will update current number of open txns to 3
+    openTxnsCounterService.setConf(hiveConf);
+    openTxnsCounterService.run();  // will update current number of open txns to 3
 
     MetaException exception = null;
     // This should fail once it finds out the threshold has been reached
@@ -1301,7 +1305,7 @@ public class TestTxnCommands2 {
     for (long txnid : openTxnsResponse.getTxn_ids()) {
       txnHandler.commitTxn(new CommitTxnRequest(txnid));
     }
-    runHouseKeeperService(openTxnsCounterService, hiveConf);  // will update current number of open txns back to 0
+    openTxnsCounterService.run();  // will update current number of open txns back to 0
     exception = null;
     try {
       txnHandler.openTxns(new OpenTxnRequest(1, "him", "localhost"));

http://git-wip-us.apache.org/repos/asf/hive/blob/f4a12a56/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
index ad2aac5..e0344c5 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TxnCommandsBaseForTests.java
@@ -53,7 +53,6 @@ public abstract class TxnCommandsBaseForTests {
     setUpInternal();
   }
   void setUpInternal() throws Exception {
-    tearDown();
     hiveConf = new HiveConf(this.getClass());
     hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
     hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
@@ -65,7 +64,7 @@ public abstract class TxnCommandsBaseForTests {
         "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory");
     hiveConf.setBoolVar(HiveConf.ConfVars.MERGE_CARDINALITY_VIOLATION_CHECK, true);
     TxnDbUtil.setConfValues(hiveConf);
-    TxnDbUtil.prepDb();
+    TxnDbUtil.prepDb(hiveConf);
     File f = new File(getWarehouseDir());
     if (f.exists()) {
       FileUtil.fullyDelete(f);
@@ -99,7 +98,7 @@ public abstract class TxnCommandsBaseForTests {
         d = null;
       }
     } finally {
-      TxnDbUtil.cleanDb();
+      TxnDbUtil.cleanDb(hiveConf);
       FileUtils.deleteDirectory(new File(getTestDataDir()));
     }
   }