[hive] branch master updated: HIVE-21226: Exclude read-only transactions from ValidTxnList (Denys Kuzmenko reviewed by Peter Vary)

[pv...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new ad2bb41  HIVE-21226: Exclude read-only transactions from ValidTxnList (Denys Kuzmenko reviewed by Peter Vary)
ad2bb41 is described below

commit ad2bb41e14bc95e2b9a2b661dc0cd929797e615d
Author: denys kuzmenko <dk...@cloudera.com>
AuthorDate: Wed Nov 20 13:07:06 2019 +0100

    HIVE-21226: Exclude read-only transactions from ValidTxnList (Denys Kuzmenko reviewed by Peter Vary)
---
 .../apache/hadoop/hive/metastore/txn/TxnCommonUtils.java |  6 +++---
 .../org/apache/hadoop/hive/metastore/txn/TxnHandler.java | 16 ++++++++++------
 .../hadoop/hive/metastore/TestHiveMetaStoreTxns.java     | 13 +++++++++++++
 3 files changed, 26 insertions(+), 9 deletions(-)

diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
index 43cc805..582b41c 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnCommonUtils.java
@@ -58,11 +58,11 @@ public class TxnCommonUtils {
 
     // We care only about open/aborted txns below currentTxn and hence the size should be determined
     // for the exceptions list. The currentTxn will be missing in openTxns list only in rare case like
-    // txn is aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns.
+    // txn is read-only or aborted by AcidHouseKeeperService and compactor actually cleans up the aborted txns.
     // So, for such cases, we get negative value for sizeToHwm with found position for currentTxn, and so,
     // we just negate it to get the size.
-    int sizeToHwm = (currentTxn > 0) ? Collections.binarySearch(openTxns, currentTxn) : openTxns.size();
-    sizeToHwm = (sizeToHwm < 0) ? (-sizeToHwm) : sizeToHwm;
+    int sizeToHwm = (currentTxn > 0) ? Math.abs(Collections.binarySearch(openTxns, currentTxn)) : openTxns.size();
+    sizeToHwm = Math.min(sizeToHwm, openTxns.size());
     long[] exceptions = new long[sizeToHwm];
     BitSet inAbortedBits = BitSet.valueOf(txns.getAbortedBits());
     BitSet outAbortedBits = new BitSet();
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index 6281208..2680387 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -457,19 +457,23 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI {
         close(rs);
         List<Long> openList = new ArrayList<>();
         //need the WHERE clause below to ensure consistent results with READ_COMMITTED
-        s = "select txn_id, txn_state from TXNS where txn_id <= " + hwm + " order by txn_id";
+        s = "select txn_id, txn_state, txn_type from TXNS where txn_id <= " + hwm + " order by txn_id";
         LOG.debug("Going to execute query<" + s + ">");
         rs = stmt.executeQuery(s);
         long minOpenTxn = Long.MAX_VALUE;
         BitSet abortedBits = new BitSet();
         while (rs.next()) {
           long txnId = rs.getLong(1);
-          openList.add(txnId);
-          char c = rs.getString(2).charAt(0);
-          if(c == TXN_OPEN) {
+          char txnState = rs.getString(2).charAt(0);
+          if (txnState == TXN_OPEN) {
             minOpenTxn = Math.min(minOpenTxn, txnId);
-          } else if (c == TXN_ABORTED) {
-            abortedBits.set(openList.size() - 1);
+          }
+          TxnType txnType = TxnType.findByValue(rs.getInt(3));
+          if (txnType != TxnType.READ_ONLY) {
+            openList.add(txnId);
+            if (txnState == TXN_ABORTED) {
+              abortedBits.set(openList.size() - 1);
+            }
           }
         }
         LOG.debug("Going to rollback");
diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
index fc08dbc..baa6247 100644
--- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
+++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java
@@ -100,6 +100,19 @@ public class TestHiveMetaStoreTxns {
   }
 
   @Test
+  public void testOpenReadOnlyTxnExcluded() throws Exception {
+    client.openTxn("me", TxnType.READ_ONLY);
+    client.openTxns("me", 3);
+    client.rollbackTxn(2);
+    client.commitTxn(3);
+    ValidTxnList validTxns = client.getValidTxns(4);
+    Assert.assertTrue(validTxns.isTxnValid(1));
+    Assert.assertFalse(validTxns.isTxnValid(2));
+    Assert.assertTrue(validTxns.isTxnValid(3));
+    Assert.assertTrue(validTxns.isTxnValid(4));
+  }
+
+  @Test
   public void testTxNWithKeyValue() throws Exception {
     Statement stm = conn.createStatement();