You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2017/09/26 16:16:47 UTC
[1/3] hive git commit: HIVE-16084 SHOW COMPACTIONS should display
CompactionID (Eugene Koifman, reviewed by Alan Gates)
Repository: hive
Updated Branches:
refs/heads/master 1a7f853f3 -> a0d3aca51
HIVE-16084 SHOW COMPACTIONS should display CompactionID (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a0d3aca5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a0d3aca5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a0d3aca5
Branch: refs/heads/master
Commit: a0d3aca51c7bf3d4d037fa0d99483f40f0491d1f
Parents: c396cd2
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 09:11:45 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java | 4 ++++
.../org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java | 5 +++--
ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out | 4 ++--
3 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 443be54..fd91974 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -2926,6 +2926,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
DataOutputStream os = getOutputStream(desc.getResFile());
try {
// Write a header
+ os.writeBytes("CompactionId");
+ os.write(separator);
os.writeBytes("Database");
os.write(separator);
os.writeBytes("Table");
@@ -2947,6 +2949,8 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
if (rsp.getCompacts() != null) {
for (ShowCompactResponseElement e : rsp.getCompacts()) {
+ os.writeBytes(Long.toString(e.getId()));
+ os.write(separator);
os.writeBytes(e.getDbname());
os.write(separator);
os.writeBytes(e.getTablename());
http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
index dc47a38..c524607 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ShowCompactionsDesc.java
@@ -27,8 +27,9 @@ import java.io.Serializable;
public class ShowCompactionsDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1L;
- private static final String schema = "dbname,tabname,partname,type,state,workerid," +
- "starttime,duration,hadoopjobid#string:string:string:string:string:string:string:string:string";
+ private static final String schema =
+ "compactionid,dbname,tabname,partname,type,state,workerid,starttime,duration,hadoopjobid#" +
+ "string:string:string:string:string:string:string:string:string:string";
private String resFile;
http://git-wip-us.apache.org/repos/asf/hive/blob/a0d3aca5/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
index a9814f3..90ab961 100644
--- a/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
+++ b/ql/src/test/results/clientpositive/dbtxnmgr_showlocks.q.out
@@ -67,8 +67,8 @@ PREHOOK: query: show compactions
PREHOOK: type: SHOW COMPACTIONS
POSTHOOK: query: show compactions
POSTHOOK: type: SHOW COMPACTIONS
-Database Table Partition Type State Worker Start Time Duration(ms) HadoopJobId
-default partitioned_acid_table p=abc MINOR initiated --- --- --- ---
+CompactionId Database Table Partition Type State Worker Start Time Duration(ms) HadoopJobId
+1 default partitioned_acid_table p=abc MINOR initiated --- --- --- ---
PREHOOK: query: drop table partitioned_acid_table
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@partitioned_acid_table
[2/3] hive git commit: HIVE-17110 BucketCodec should enforce value
ranges (Eugene Koifman, reviewed by Gopal V)
Posted by ek...@apache.org.
HIVE-17110 BucketCodec should enforce value ranges (Eugene Koifman, reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c396cd2b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c396cd2b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c396cd2b
Branch: refs/heads/master
Commit: c396cd2b8bc46e512ccdf54af8817bb82cf5340e
Parents: a3b4bf8
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 08:51:02 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/io/BucketCodec.java | 23 +++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c396cd2b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
index 343f41f..3af16e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/BucketCodec.java
@@ -86,11 +86,28 @@ public enum BucketCodec {
}
@Override
public int encode(AcidOutputFormat.Options options) {
- return this.version << 29 | options.getBucketId() << 16 |
- (options.getStatementId() >= 0 ? options.getStatementId() : 0);
+ int statementId = options.getStatementId() >= 0 ? options.getStatementId() : 0;
+
+ assert this.version >=0 && this.version <= MAX_VERSION
+ : "Version out of range: " + version;
+ if(!(options.getBucketId() >= 0 && options.getBucketId() <= MAX_BUCKET_ID)) {
+ throw new IllegalArgumentException("bucketId out of range: " + options.getBucketId());
+ }
+ if(!(statementId >= 0 && statementId <= MAX_STATEMENT_ID)) {
+ throw new IllegalArgumentException("statementId out of range: " + statementId);
+ }
+ return this.version << (1 + NUM_BUCKET_ID_BITS + 4 + NUM_STATEMENT_ID_BITS) |
+ options.getBucketId() << (4 + NUM_STATEMENT_ID_BITS) | statementId;
}
};
- private static int TOP3BITS_MASK = 0b1110_0000_0000_0000_0000_0000_0000_0000;
+ private static final int TOP3BITS_MASK = 0b1110_0000_0000_0000_0000_0000_0000_0000;
+ private static final int NUM_VERSION_BITS = 3;
+ private static final int NUM_BUCKET_ID_BITS = 12;
+ private static final int NUM_STATEMENT_ID_BITS = 12;
+ private static final int MAX_VERSION = (1 << NUM_VERSION_BITS) - 1;
+ private static final int MAX_BUCKET_ID = (1 << NUM_BUCKET_ID_BITS) - 1;
+ private static final int MAX_STATEMENT_ID = (1 << NUM_STATEMENT_ID_BITS) - 1;
+
public static BucketCodec determineVersion(int bucket) {
assert 7 << 29 == BucketCodec.TOP3BITS_MASK;
//look at top 3 bits and return appropriate enum
[3/3] hive git commit: HIVE-16360 Improve 'No delta files or original
files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)
Posted by ek...@apache.org.
HIVE-16360 Improve 'No delta files or original files found to compact in' message (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a3b4bf89
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a3b4bf89
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a3b4bf89
Branch: refs/heads/master
Commit: a3b4bf89c4200e0a6eb6de47ecb12940a0f8b885
Parents: 1a7f853
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Sep 26 08:48:05 2017 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Sep 26 09:16:22 2017 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/common/ValidCompactorTxnList.java | 8 ++++++--
.../org/apache/hadoop/hive/common/ValidReadTxnList.java | 7 ++++---
.../java/org/apache/hadoop/hive/common/ValidTxnList.java | 4 ++++
.../apache/hadoop/hive/ql/txn/compactor/TestCompactor.java | 4 +++-
.../java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java | 8 +++++++-
.../hive/metastore/txn/TestValidCompactorTxnList.java | 6 +++---
.../apache/hadoop/hive/ql/txn/compactor/CompactorMR.java | 9 +++++++--
7 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
index c022577..eaa0b34 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidCompactorTxnList.java
@@ -39,14 +39,18 @@ public class ValidCompactorTxnList extends ValidReadTxnList {
public ValidCompactorTxnList() {
super();
}
+ public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
+ this(abortedTxnList, abortedBits, highWatermark, Long.MAX_VALUE);
+ }
/**
* @param abortedTxnList list of all aborted transactions
* @param abortedBits bitset marking whether the corresponding transaction is aborted
* @param highWatermark highest committed transaction to be considered for compaction,
* equivalently (lowest_open_txn - 1).
*/
- public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark) {
- super(abortedTxnList, abortedBits, highWatermark); // abortedBits should be all true as everything in exceptions are aborted txns
+ public ValidCompactorTxnList(long[] abortedTxnList, BitSet abortedBits, long highWatermark, long minOpenTxnId) {
+ // abortedBits should be all true as everything in exceptions are aborted txns
+ super(abortedTxnList, abortedBits, highWatermark, minOpenTxnId);
if(this.exceptions.length <= 0) {
return;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
index 4e57772..002afd6 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidReadTxnList.java
@@ -193,9 +193,10 @@ public class ValidReadTxnList implements ValidTxnList {
public long[] getInvalidTransactions() {
return exceptions;
}
- @VisibleForTesting
- public long getMinOpenTxn() {
- return minOpenTxn;
+
+ @Override
+ public Long getMinOpenTxn() {
+ return minOpenTxn == Long.MAX_VALUE ? null : minOpenTxn;
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
index d4ac02c..108e5ca 100644
--- a/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
+++ b/common/src/java/org/apache/hadoop/hive/common/ValidTxnList.java
@@ -105,4 +105,8 @@ public interface ValidTxnList {
*/
public RangeResponse isTxnRangeAborted(long minTxnId, long maxTxnId);
+ /**
+ * Returns smallest Open transaction in this set, {@code null} if there is none.
+ */
+ Long getMinOpenTxn();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index 4bfc5be..aea1dfc 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -1150,7 +1150,6 @@ public class TestCompactor {
/**
* Users have the choice of specifying compaction related tblproperties either in CREATE TABLE
* statement or in ALTER TABLE .. COMPACT statement. This tests both cases.
- * @throws Exception
*/
@Test
public void testTableProperties() throws Exception {
@@ -1320,6 +1319,9 @@ public class TestCompactor {
}
@Override
+ public Long getMinOpenTxn() { return null; }
+
+ @Override
public long getHighWatermark() {
return Long.MAX_VALUE;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
index e6c62d3..30b155f 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java
@@ -81,6 +81,7 @@ public class TxnUtils {
* @return a valid txn list.
*/
public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
+ //highWater is the last txn id that has been allocated
long highWater = txns.getTxn_high_water_mark();
long minOpenTxn = Long.MAX_VALUE;
long[] exceptions = new long[txns.getOpen_txnsSize()];
@@ -100,7 +101,12 @@ public class TxnUtils {
highWater = minOpenTxn == Long.MAX_VALUE ? highWater : minOpenTxn - 1;
BitSet bitSet = new BitSet(exceptions.length);
bitSet.set(0, exceptions.length); // for ValidCompactorTxnList, everything in exceptions are aborted
- return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+ if(minOpenTxn == Long.MAX_VALUE) {
+ return new ValidCompactorTxnList(exceptions, bitSet, highWater);
+ }
+ else {
+ return new ValidCompactorTxnList(exceptions, bitSet, highWater, minOpenTxn);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
index 91d6215..eb88e32 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/txn/TestValidCompactorTxnList.java
@@ -103,11 +103,11 @@ public class TestValidCompactorTxnList {
public void readFromString() {
ValidCompactorTxnList txns = new ValidCompactorTxnList("37:" + Long.MAX_VALUE + "::7,9,10");
Assert.assertEquals(37L, txns.getHighWatermark());
- Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+ Assert.assertNull(txns.getMinOpenTxn());
Assert.assertArrayEquals(new long[]{7L, 9L, 10L}, txns.getInvalidTransactions());
txns = new ValidCompactorTxnList("21:" + Long.MAX_VALUE + ":");
Assert.assertEquals(21L, txns.getHighWatermark());
- Assert.assertEquals(Long.MAX_VALUE, txns.getMinOpenTxn());
+ Assert.assertNull(txns.getMinOpenTxn());
Assert.assertEquals(0, txns.getInvalidTransactions().length);
}
@@ -115,7 +115,7 @@ public class TestValidCompactorTxnList {
public void testAbortedTxn() throws Exception {
ValidCompactorTxnList txnList = new ValidCompactorTxnList("5:4::1,2,3");
Assert.assertEquals(5L, txnList.getHighWatermark());
- Assert.assertEquals(4, txnList.getMinOpenTxn());
+ Assert.assertEquals(4, txnList.getMinOpenTxn().longValue());
Assert.assertArrayEquals(new long[]{1L, 2L, 3L}, txnList.getInvalidTransactions());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a3b4bf89/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 04ef7fc..7f5c6d3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -265,10 +265,15 @@ public class CompactorMR {
dirsToSearch.add(baseDir);
}
}
-
if (parsedDeltas.size() == 0 && dir.getOriginalFiles().size() == 0) {
// Skip compaction if there's no delta files AND there's no original files
- LOG.error("No delta files or original files found to compact in " + sd.getLocation() + " for compactionId=" + ci.id);
+ String minOpenInfo = ".";
+ if(txns.getMinOpenTxn() != null) {
+ minOpenInfo = " with min Open " + JavaUtils.txnIdToString(txns.getMinOpenTxn()) +
+ ". Compaction cannot compact above this txnid";
+ }
+ LOG.error("No delta files or original files found to compact in " + sd.getLocation() +
+ " for compactionId=" + ci.id + minOpenInfo);
return;
}