You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/09/26 16:16:47 UTC

[1/3] hive git commit: HIVE-16084 SHOW COMPACTIONS should display CompactionID (Eugene Koifman, reviewed by Alan Gates)

Repository: hive
Updated Branches:
  refs/heads/master 1a7f853f3 -> a0d3aca51


HIVE-16084 SHOW COMPACTIONS should display CompactionID (Eugene Koifman, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a0d3aca5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a0d3aca5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a0d3aca5

Branch: refs/heads/master
Commit: a0d3aca51c7bf3d4d037fa0d99483f40f0491d1f
Parents: c396cd2
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 09:11:45 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java         | 4 ++++
 .../org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java     | 5 +++--
 ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out     | 4 ++--
 3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 443be54..fd91974 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2926,6 +2926,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
     DataOutputStream os = getOutputStream(desc.getResFile());
     try {
       // Write a header
+      os.writeBytes("CompactionId");
+      os.write(separator);
       os.writeBytes("Database");
       os.write(separator);
       os.writeBytes("Table");
@@ -2947,6 +2949,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
 
       if (rsp.getCompacts() != null) {
         for (ShowCompactResponseElement e : rsp.getCompacts()) {
+          os.writeBytes(Long.toString(e.getId()));
+          os.write(separator);
           os.writeBytes(e.getDbname());
           os.write(separator);
           os.writeBytes(e.getTablename());

http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
index dc47a38..c524607 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
@@ -27,8 +27,9 @@ import java.io.Serializable;
 public class ShowCompactionsDesc extends DDLDesc implements Serializable {
 
   private static final long serialVersionUID = 1L;
-  private static final String schema = "dbname,tabname,partname,type,state,workerid," +
-      "starttime,duration,hadoopjobid#string:string:string:string:string:string:string:string:string";
+  private static final String schema =
+    "compactionid,dbname,tabname,partname,type,state,workerid,starttime,duration,hadoopjobid#" +
+      "string:string:string:string:string:string:string:string:string:string";
 
   private String resFile;
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
index a9814f3..90ab961 100644
--- a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
+++ b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
@@ -67,8 +67,8 @@ PREHOOK: query: show compactions
 PREHOOK: type: SHOW COMPACTIONS
 POSTHOOK: query: show compactions
 POSTHOOK: type: SHOW COMPACTIONS
-Database	Table	Partition	Type	State	Worker	Start Time	Duration(ms)	HadoopJobId
-default	partitioned_acid_table	p=abc	MINOR	initiated	 --- 	 --- 	 --- 	 --- 
+CompactionId	Database	Table	Partition	Type	State	Worker	Start Time	Duration(ms)	HadoopJobId
+1	default	partitioned_acid_table	p=abc	MINOR	initiated	 --- 	 --- 	 --- 	 --- 
 PREHOOK: query: drop table partitioned_acid_table
 PREHOOK: type: DROPTABLE
 PREHOOK: Input: default@partitioned_acid_table


[2/3] hive git commit: HIVE-17110 BucketCodec should enforce value ranges (Eugene Koifman, reviewed by Gopal V)

Posted by ek...@apache.org.
HIVE-17110 BucketCodec should enforce value ranges (Eugene Koifman, reviewed by Gopal V)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c396cd2b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c396cd2b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c396cd2b

Branch: refs/heads/master
Commit: c396cd2b8bc46e512ccdf54af8817bb82cf5340e
Parents: a3b4bf8
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 08:51:02 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/ql/io/BucketCodec.java   | 23 +++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/c396cd2b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
index 343f41f..3af16e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
@@ -86,11 +86,28 @@ public enum BucketCodec {
     }
     @Override
     public int encode(AcidOutputFormat.Options options) {
-      return this.version << 29 | options.getBucketId() << 16 |
-        (options.getStatementId() >= 0 ? options.getStatementId() : 0);
+      int statementId = options.getStatementId() >= 0 ? options.getStatementId() : 0;
+
+      assert this.version >=0 && this.version <= MAX_VERSION
+        : "Version out of range: " + version;
+      if(!(options.getBucketId() >= 0 && options.getBucketId() <= MAX_BUCKET_ID)) {
+        throw new IllegalArgumentException("bucketId out of range: " + options.getBucketId());
+      }
+      if(!(statementId >= 0 && statementId <= MAX_STATEMENT_ID)) {
+        throw new IllegalArgumentException("statementId out of range: " + statementId);
+      }
+      return this.version << (1 + NUM_BUCKET_ID_BITS + 4 + NUM_STATEMENT_ID_BITS) |
+        options.getBucketId() << (4 + NUM_STATEMENT_ID_BITS) | statementId;
     }
   };
-  private static int TOP3BITS_MASK = 0b1110_0000_0000_0000_0000_0000_0000_0000;
+  private static final int TOP3BITS_MASK = 0b1110_0000_0000_0000_0000_0000_0000_0000;
+  private static final int NUM_VERSION_BITS = 3;
+  private static final int NUM_BUCKET_ID_BITS = 12;
+  private static final int NUM_STATEMENT_ID_BITS = 12;
+  private static final int MAX_VERSION = (1 << NUM_VERSION_BITS) - 1;
+  private static final int MAX_BUCKET_ID = (1 << NUM_BUCKET_ID_BITS) - 1;
+  private static final int MAX_STATEMENT_ID = (1 << NUM_STATEMENT_ID_BITS) - 1;
+
   public static BucketCodec determineVersion(int bucket) {
     assert 7 << 29 == BucketCodec.TOP3BITS_MASK;
     //look at top 3 bits and return appropriate enum


[3/3] hive git commit: HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)

Posted by ek...@apache.org.
HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3b4bf89
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3b4bf89
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3b4bf89

Branch: refs/heads/master
Commit: a3b4bf89c4200e0a6eb6de47ecb12940a0f8b885
Parents: 1a7f853
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 08:48:05 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/hive/common/ValidCompactorTxnList.java    | 8 ++++++--
 .../org/apache/hadoop/hive/common/ValidReadTxnList.java     | 7 ++++---
 .../java/org/apache/hadoop/hive/common/ValidTxnList.java    | 4 ++++
 .../apache/hadoop/hive/ql/txn/compactor/TestCompactor.java  | 4 +++-
 .../java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java | 8 +++++++-
 .../hive/metastore/txn/TestValidCompactorTxnList.java       | 6 +++---
 .../apache/hadoop/hive/ql/txn/compactor/CompactorMR.java    | 9 +++++++--
 7 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
index c022577..eaa0b34 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -39,14 +39,18 @@ public class ValidCompactorTxnList extends ValidReadTxnList {
   public ValidCompactorTxnList() {
     super();
   }
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
+    this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
+  }
   /**
    * @param abortedTxnList list of all aborted transactions
    * @param abortedBits bitset marking whether the corresponding transaction is aborted
    * @param highWatermark highest committed transaction to be considered for compaction,
    *                      equivalently (lowest_open_txn - 1).
    */
-  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
-    super(abortedTxnList, abortedBits, highWatermark); // abortedBits should be all true as everything in exceptions are aborted txns
+  public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) {
+    // abortedBits should be all true as everything in exceptions are aborted txns
+    super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
     if(this.exceptions.length <= 0) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
index 4e57772..002afd6 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
@@ -193,9 +193,10 @@ public class ValidReadTxnList implements ValidTxnList {
   public long[] getInvalidTransactions() {
     return exceptions;
   }
-  @VisibleForTesting
-  public long getMinOpenTxn() {
-    return minOpenTxn;
+
+  @Override
+  public Long getMinOpenTxn() {
+    return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
index d4ac02c..108e5ca 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -105,4 +105,8 @@ public interface ValidTxnList {
    */
   public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
 
+  /**
+   * Returns smallest Open transaction in this set, {@code null} if there is none.
+   */
+  Long getMinOpenTxn();
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 4bfc5be..aea1dfc 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1150,7 +1150,6 @@ public class TestCompactor {
   /**
    * Users have the choice of specifying compaction related tblproperties either in CREATE TABLE
    * statement or in ALTER TABLE .. COMPACT statement. This tests both cases.
-   * @throws Exception
    */
   @Test
   public void testTableProperties() throws Exception {
@@ -1320,6 +1319,9 @@ public class TestCompactor {
       }
 
       @Override
+      public Long getMinOpenTxn() { return null; }
+
+      @Override
       public long getHighWatermark() {
         return  Long.MAX_VALUE;
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index e6c62d3..30b155f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -81,6 +81,7 @@ public class TxnUtils {
    * @return a valid txn list.
    */
   public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
+    //highWater is the last txn id that has been allocated
     long highWater = txns.getTxn_high_water_mark();
     long minOpenTxn = Long.MAX_VALUE;
     long[] exceptions = new long[txns.getOpen_txnsSize()];
@@ -100,7 +101,12 @@ public class TxnUtils {
     highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
     BitSet bitSet = new BitSet(exceptions.length);
     bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted
-    return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+    if(minOpenTxn == Long.MAX_VALUE) {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+    }
+    else {
+      return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
+    }
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
index 91d6215..eb88e32 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
@@ -103,11 +103,11 @@ public class TestValidCompactorTxnList {
   public void readFromString() {
     ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10");
     Assert.assertEquals(37L, txns.getHighWatermark());
-    Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+    Assert.assertNull(txns.getMinOpenTxn());
     Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions());
     txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
     Assert.assertEquals(21L, txns.getHighWatermark());
-    Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+    Assert.assertNull(txns.getMinOpenTxn());
     Assert.assertEquals(0, txns.getInvalidTransactions().length);
   }
 
@@ -115,7 +115,7 @@ public class TestValidCompactorTxnList {
   public void testAbortedTxn() throws Exception {
     ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
     Assert.assertEquals(5L, txnList.getHighWatermark());
-    Assert.assertEquals(4, txnList.getMinOpenTxn());
+    Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
     Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions());
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 04ef7fc..7f5c6d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -265,10 +265,15 @@ public class CompactorMR {
         dirsToSearch.add(baseDir);
       }
     }
-
     if (parsedDeltas.size() == 0 && dir.getOriginalFiles().size() == 0) {
       // Skip compaction if there's no delta files AND there's no original files
-      LOG.error("No delta files or original files found to compact in " + sd.getLocation() + " for compactionId=" + ci.id);
+      String minOpenInfo = ".";
+      if(txns.getMinOpenTxn() != null) {
+        minOpenInfo = " with min Open " + JavaUtils.txnIdToString(txns.getMinOpenTxn()) +
+          ".  Compaction cannot compact above this txnid";
+      }
+      LOG.error("No delta files or original files found to compact in " + sd.getLocation() +
+        " for compactionId=" + ci.id + minOpenInfo);
       return;
     }