You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2018/01/16 17:48:00 UTC
[1/2] hive git commit: HIVE-18429 - Compaction should handle a case
when it produces no output (Eugene Koifman, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master 87860fbca -> dfd7ea368
HIVE-18429 - Compaction should handle a case when it produces no output (Eugene Koifman, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe3190d1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe3190d1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe3190d1
Branch: refs/heads/master
Commit: fe3190d19fa1324a6835692e71f747e6cb37c7d1
Parents: 87860fb
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jan 16 09:30:09 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jan 16 09:31:55 2018 -0800
----------------------------------------------------------------------
.../hive/ql/txn/compactor/CompactorMR.java | 29 ++++++++++--
.../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 48 +++++++++++++++++++-
2 files changed, 71 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fe3190d1/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 26f5c7e..a64c9d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -295,7 +295,7 @@ public class CompactorMR {
}
/**
- * @param baseDir if not null, it's either table/partition root folder or base_xxxx.
+ * @param baseDir if not null, it's either table/partition root folder or base_xxxx.
* If it's base_xxxx, it's in dirsToSearch, else the actual original files
* (all leaves recursively) are in the dirsToSearch list
*/
@@ -915,13 +915,32 @@ public class CompactorMR {
FileSystem fs = tmpLocation.getFileSystem(conf);
LOG.debug("Moving contents of " + tmpLocation.toString() + " to " +
finalLocation.toString());
-
+ if(!fs.exists(tmpLocation)) {
+ /**
+ * No 'tmpLocation' may happen if job generated created 0 splits, which happens if all
+ * input delta and/or base files were empty or had
+ * only {@link org.apache.orc.impl.OrcAcidUtils#getSideFile(Path)} files.
+ * So make sure the new base/delta is created.
+ */
+ AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
+ .writingBase(conf.getBoolean(IS_MAJOR, false))
+ .isCompressed(conf.getBoolean(IS_COMPRESSED, false))
+ .minimumTransactionId(conf.getLong(MIN_TXN, Long.MAX_VALUE))
+ .maximumTransactionId(conf.getLong(MAX_TXN, Long.MIN_VALUE))
+ .bucket(0)
+ .statementId(-1);
+ Path newDeltaDir = AcidUtils.createFilename(finalLocation, options).getParent();
+ LOG.info(context.getJobID() + ": " + tmpLocation +
+ " not found. Assuming 0 splits. Creating " + newDeltaDir);
+ fs.mkdirs(newDeltaDir);
+ return;
+ }
FileStatus[] contents = fs.listStatus(tmpLocation);//expect 1 base or delta dir in this list
//we have MIN_TXN, MAX_TXN and IS_MAJOR in JobConf so we could figure out exactly what the dir
//name is that we want to rename; leave it for another day
- for (int i = 0; i < contents.length; i++) {
- Path newPath = new Path(finalLocation, contents[i].getPath().getName());
- fs.rename(contents[i].getPath(), newPath);
+ for (FileStatus fileStatus : contents) {
+ Path newPath = new Path(finalLocation, fileStatus.getPath().getName());
+ fs.rename(fileStatus.getPath(), newPath);
}
fs.delete(tmpLocation, true);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/fe3190d1/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index ac44779..bbf6c86 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.hive.ql;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hadoop.hive.metastore.txn.TxnStore;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -697,9 +700,9 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
map = hms.getPartitionColumnStatistics("default","T", partNames, colNames);
Assert.assertEquals("", 5, map.get(partNames.get(0)).get(0).getStatsData().getLongStats().getHighValue());
}
- @Ignore("enable after HIVE-18294")
@Test
public void testDefault() throws Exception {
+ hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true");
runStatementOnDriver("drop table if exists T");
runStatementOnDriver("create table T (a int, b int) stored as orc");
runStatementOnDriver("insert into T values(1,2),(3,4)");
@@ -711,6 +714,49 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
{"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000015_0000015_0000/bucket_00000"}
};
checkExpected(rs, expected, "insert data");
+ }
+ /**
+ * see HIVE-18429
+ */
+ @Test
+ public void testEmptyCompactionResult() throws Exception {
+ hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true");
+ runStatementOnDriver("drop table if exists T");
+ runStatementOnDriver("create table T (a int, b int) stored as orc");
+ int[][] data = {{1,2}, {3,4}};
+ runStatementOnDriver("insert into T" + makeValuesClause(data));
+ runStatementOnDriver("insert into T" + makeValuesClause(data));
+
+ //delete the bucket files so now we have empty delta dirs
+ List<String> rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T");
+ FileSystem fs = FileSystem.get(hiveConf);
+ for(String path : rs) {
+ fs.delete(new Path(path), true);
+ }
+ runStatementOnDriver("alter table T compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+
+ //check status of compaction job
+ TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+ ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
+ Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
+ Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
+ Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local"));
+
+ //now run another compaction make sure empty dirs don't cause issues
+ runStatementOnDriver("insert into T" + makeValuesClause(data));
+ runStatementOnDriver("alter table T compact 'major'");
+ TestTxnCommands2.runWorker(hiveConf);
+
+ //check status of compaction job
+ resp = txnHandler.showCompact(new ShowCompactRequest());
+ Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize());
+ for(int i = 0; i < 2; i++) {
+ Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(i).getState());
+ Assert.assertTrue(resp.getCompacts().get(i).getHadoopJobId().startsWith("job_local"));
+ }
+ rs = runStatementOnDriver("select a, b from T order by a, b");
+ Assert.assertEquals(stringifyValues(data), rs);
}
}
[2/2] hive git commit: HIVE-18419 - CliDriver loads different
hive-site.xml into HiveConf and MetastoreConf (Eugene Koifman,
reviewed by Alan Gates)
Posted by ek...@apache.org.
HIVE-18419 - CliDriver loads different hive-site.xml into HiveConf and MetastoreConf (Eugene Koifman, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfd7ea36
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfd7ea36
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfd7ea36
Branch: refs/heads/master
Commit: dfd7ea368dc8ea2ef071543c3d0cea9e05cd115a
Parents: fe3190d
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jan 16 09:33:48 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jan 16 09:33:48 2018 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 2 ++
.../hadoop/hive/metastore/conf/MetastoreConf.java | 14 +++++++++++++-
.../hadoop/hive/metastore/conf/TestMetastoreConf.java | 2 +-
3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f79dbac..9b102f2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
import org.apache.hadoop.hive.common.io.SortPrintStream;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
import org.apache.hive.druid.MiniDruidCluster;
import org.apache.hadoop.hive.llap.LlapItUtils;
import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
@@ -555,6 +556,7 @@ public class QTestUtil {
// HIVE-14443 move this fall-back logic to CliConfigs
if (confDir != null && !confDir.isEmpty()) {
HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
+ MetastoreConf.setHiveSiteLocation(HiveConf.getHiveSiteLocation());
System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 2e43dc8..00f3c8d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1032,6 +1032,10 @@ public class MetastoreConf {
throw new RuntimeException("You should never be creating one of these!");
}
+ public static void setHiveSiteLocation(URL location) {
+ hiveSiteURL = location;
+ }
+
public static Configuration newMetastoreConf() {
Configuration conf = new Configuration();
@@ -1047,7 +1051,15 @@ public class MetastoreConf {
// Add in hive-site.xml. We add this first so that it gets overridden by the new metastore
// specific files if they exist.
- hiveSiteURL = findConfigFile(classLoader, "hive-site.xml");
+ if(hiveSiteURL == null) {
+ /*
+ * this 'if' is pretty lame - QTestUtil.QTestUtil() uses hiveSiteURL to load a specific
+ * hive-site.xml from data/conf/<subdir> so this makes it follow the same logic - otherwise
+ * HiveConf and MetastoreConf may load different hive-site.xml ( For example,
+ * HiveConf uses data/conf/spark/hive-site.xml and MetastoreConf data/conf/hive-site.xml)
+ */
+ hiveSiteURL = findConfigFile(classLoader, "hive-site.xml");
+ }
if (hiveSiteURL != null) conf.addResource(hiveSiteURL);
// Now add hivemetastore-site.xml. Again we add this before our own config files so that the
http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
index e6e8fee..b8f53a0 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
@@ -25,7 +25,6 @@ import org.hamcrest.core.StringEndsWith;
import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,6 +47,7 @@ public class TestMetastoreConf {
@After
public void unsetProperties() {
+ MetastoreConf.setHiveSiteLocation(null);
for (MetastoreConf.ConfVars var : MetastoreConf.dataNucleusAndJdoConfs) {
System.getProperties().remove(var.getVarname());
}