You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2018/01/16 17:48:00 UTC

[1/2] hive git commit: HIVE-18429 - Compaction should handle a case when it produces no output (Eugene Koifman, reviewed by Prasanth Jayachandran)

Repository: hive
Updated Branches:
  refs/heads/master 87860fbca -> dfd7ea368


HIVE-18429 - Compaction should handle a case when it produces no output (Eugene Koifman, reviewed by Prasanth Jayachandran)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fe3190d1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fe3190d1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fe3190d1

Branch: refs/heads/master
Commit: fe3190d19fa1324a6835692e71f747e6cb37c7d1
Parents: 87860fb
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jan 16 09:30:09 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jan 16 09:31:55 2018 -0800

----------------------------------------------------------------------
 .../hive/ql/txn/compactor/CompactorMR.java      | 29 ++++++++++--
 .../apache/hadoop/hive/ql/TestTxnNoBuckets.java | 48 +++++++++++++++++++-
 2 files changed, 71 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/fe3190d1/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index 26f5c7e..a64c9d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -295,7 +295,7 @@ public class CompactorMR {
   }
 
   /**
-   * @param baseDir if not null, it's either table/partition root folder or base_xxxx.  
+   * @param baseDir if not null, it's either table/partition root folder or base_xxxx.
    *                If it's base_xxxx, it's in dirsToSearch, else the actual original files
    *                (all leaves recursively) are in the dirsToSearch list
    */
@@ -915,13 +915,32 @@ public class CompactorMR {
       FileSystem fs = tmpLocation.getFileSystem(conf);
       LOG.debug("Moving contents of " + tmpLocation.toString() + " to " +
           finalLocation.toString());
-
+      if(!fs.exists(tmpLocation)) {
+        /**
+         * No 'tmpLocation' may happen if job generated created 0 splits, which happens if all
+         * input delta and/or base files were empty or had
+         * only {@link org.apache.orc.impl.OrcAcidUtils#getSideFile(Path)} files.
+         * So make sure the new base/delta is created.
+         */
+        AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
+            .writingBase(conf.getBoolean(IS_MAJOR, false))
+            .isCompressed(conf.getBoolean(IS_COMPRESSED, false))
+            .minimumTransactionId(conf.getLong(MIN_TXN, Long.MAX_VALUE))
+            .maximumTransactionId(conf.getLong(MAX_TXN, Long.MIN_VALUE))
+            .bucket(0)
+            .statementId(-1);
+        Path newDeltaDir = AcidUtils.createFilename(finalLocation, options).getParent();
+        LOG.info(context.getJobID() + ": " + tmpLocation +
+            " not found.  Assuming 0 splits.  Creating " + newDeltaDir);
+        fs.mkdirs(newDeltaDir);
+        return;
+      }
       FileStatus[] contents = fs.listStatus(tmpLocation);//expect 1 base or delta dir in this list
       //we have MIN_TXN, MAX_TXN and IS_MAJOR in JobConf so we could figure out exactly what the dir
       //name is that we want to rename; leave it for another day
-      for (int i = 0; i < contents.length; i++) {
-        Path newPath = new Path(finalLocation, contents[i].getPath().getName());
-        fs.rename(contents[i].getPath(), newPath);
+      for (FileStatus fileStatus : contents) {
+        Path newPath = new Path(finalLocation, fileStatus.getPath().getName());
+        fs.rename(fileStatus.getPath(), newPath);
       }
       fs.delete(tmpLocation, true);
     }

http://git-wip-us.apache.org/repos/asf/hive/blob/fe3190d1/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
index ac44779..bbf6c86 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java
@@ -17,11 +17,14 @@
  */
 package org.apache.hadoop.hive.ql;
 
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
 import org.apache.hadoop.hive.metastore.api.ShowCompactRequest;
 import org.apache.hadoop.hive.metastore.api.ShowCompactResponse;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.txn.TxnStore;
 import org.apache.hadoop.hive.metastore.txn.TxnUtils;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
@@ -697,9 +700,9 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
     map = hms.getPartitionColumnStatistics("default","T", partNames, colNames);
     Assert.assertEquals("", 5, map.get(partNames.get(0)).get(0).getStatsData().getLongStats().getHighValue());
   }
-  @Ignore("enable after HIVE-18294")
   @Test
   public void testDefault() throws Exception {
+    hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true");
     runStatementOnDriver("drop table if exists T");
     runStatementOnDriver("create table T (a int, b int) stored as orc");
     runStatementOnDriver("insert into T values(1,2),(3,4)");
@@ -711,6 +714,49 @@ ekoifman:apache-hive-3.0.0-SNAPSHOT-bin ekoifman$ tree /Users/ekoifman/dev/hiver
       {"{\"transactionid\":15,\"bucketid\":536870912,\"rowid\":1}\t3\t4", "t/delta_0000015_0000015_0000/bucket_00000"}
     };
     checkExpected(rs, expected, "insert data");
+  }
+  /**
+   * see HIVE-18429
+   */
+  @Test
+  public void testEmptyCompactionResult() throws Exception {
+    hiveConf.set(MetastoreConf.ConfVars.CREATE_TABLES_AS_ACID.getVarname(), "true");
+    runStatementOnDriver("drop table if exists T");
+    runStatementOnDriver("create table T (a int, b int) stored as orc");
+    int[][] data = {{1,2}, {3,4}};
+    runStatementOnDriver("insert into T" + makeValuesClause(data));
+    runStatementOnDriver("insert into T" + makeValuesClause(data));
+
+    //delete the bucket files so now we have empty delta dirs
+    List<String> rs = runStatementOnDriver("select distinct INPUT__FILE__NAME from T");
+    FileSystem fs = FileSystem.get(hiveConf);
+    for(String path : rs) {
+      fs.delete(new Path(path), true);
+    }
+    runStatementOnDriver("alter table T compact 'major'");
+    TestTxnCommands2.runWorker(hiveConf);
+
+    //check status of compaction job
+    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+    ShowCompactResponse resp = txnHandler.showCompact(new ShowCompactRequest());
+    Assert.assertEquals("Unexpected number of compactions in history", 1, resp.getCompactsSize());
+    Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(0).getState());
+    Assert.assertTrue(resp.getCompacts().get(0).getHadoopJobId().startsWith("job_local"));
+
+    //now run another compaction make sure empty dirs don't cause issues
+    runStatementOnDriver("insert into T" + makeValuesClause(data));
+    runStatementOnDriver("alter table T compact 'major'");
+    TestTxnCommands2.runWorker(hiveConf);
+
+    //check status of compaction job
+    resp = txnHandler.showCompact(new ShowCompactRequest());
+    Assert.assertEquals("Unexpected number of compactions in history", 2, resp.getCompactsSize());
+    for(int i = 0; i < 2; i++) {
+      Assert.assertEquals("Unexpected 0 compaction state", TxnStore.CLEANING_RESPONSE, resp.getCompacts().get(i).getState());
+      Assert.assertTrue(resp.getCompacts().get(i).getHadoopJobId().startsWith("job_local"));
+    }
+    rs = runStatementOnDriver("select a, b from T order by a, b");
+    Assert.assertEquals(stringifyValues(data), rs);
 
   }
 }


[2/2] hive git commit: HIVE-18419 - CliDriver loads different hive-site.xml into HiveConf and MetastoreConf (Eugene Koifman, reviewed by Alan Gates)

Posted by ek...@apache.org.
HIVE-18419 - CliDriver loads different hive-site.xml into HiveConf and MetastoreConf (Eugene Koifman, reviewed by Alan Gates)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfd7ea36
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfd7ea36
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfd7ea36

Branch: refs/heads/master
Commit: dfd7ea368dc8ea2ef071543c3d0cea9e05cd115a
Parents: fe3190d
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Tue Jan 16 09:33:48 2018 -0800
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Tue Jan 16 09:33:48 2018 -0800

----------------------------------------------------------------------
 .../java/org/apache/hadoop/hive/ql/QTestUtil.java     |  2 ++
 .../hadoop/hive/metastore/conf/MetastoreConf.java     | 14 +++++++++++++-
 .../hadoop/hive/metastore/conf/TestMetastoreConf.java |  2 +-
 3 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f79dbac..9b102f2 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -96,6 +96,7 @@ import org.apache.hadoop.hive.common.io.SortAndDigestPrintStream;
 import org.apache.hadoop.hive.common.io.SortPrintStream;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hive.druid.MiniDruidCluster;
 import org.apache.hadoop.hive.llap.LlapItUtils;
 import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster;
@@ -555,6 +556,7 @@ public class QTestUtil {
     // HIVE-14443 move this fall-back logic to CliConfigs
     if (confDir != null && !confDir.isEmpty()) {
       HiveConf.setHiveSiteLocation(new URL("file://"+ new File(confDir).toURI().getPath() + "/hive-site.xml"));
+      MetastoreConf.setHiveSiteLocation(HiveConf.getHiveSiteLocation());
       System.out.println("Setting hive-site: "+HiveConf.getHiveSiteLocation());
     }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
index 2e43dc8..00f3c8d 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java
@@ -1032,6 +1032,10 @@ public class MetastoreConf {
     throw new RuntimeException("You should never be creating one of these!");
   }
 
+  public static void setHiveSiteLocation(URL location) {
+    hiveSiteURL = location;
+  }
+
   public static Configuration newMetastoreConf() {
 
     Configuration conf = new Configuration();
@@ -1047,7 +1051,15 @@ public class MetastoreConf {
 
     // Add in hive-site.xml.  We add this first so that it gets overridden by the new metastore
     // specific files if they exist.
-    hiveSiteURL = findConfigFile(classLoader, "hive-site.xml");
+    if(hiveSiteURL == null) {
+      /*
+       * this 'if' is pretty lame - QTestUtil.QTestUtil() uses hiveSiteURL to load a specific
+       * hive-site.xml from data/conf/<subdir> so this makes it follow the same logic - otherwise
+       * HiveConf and MetastoreConf may load different hive-site.xml  ( For example,
+       * HiveConf uses data/conf/spark/hive-site.xml and MetastoreConf data/conf/hive-site.xml)
+       */
+      hiveSiteURL = findConfigFile(classLoader, "hive-site.xml");
+    }
     if (hiveSiteURL != null) conf.addResource(hiveSiteURL);
 
     // Now add hivemetastore-site.xml.  Again we add this before our own config files so that the

http://git-wip-us.apache.org/repos/asf/hive/blob/dfd7ea36/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
index e6e8fee..b8f53a0 100644
--- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
+++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/conf/TestMetastoreConf.java
@@ -25,7 +25,6 @@ import org.hamcrest.core.StringEndsWith;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Assume;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,6 +47,7 @@ public class TestMetastoreConf {
 
   @After
   public void unsetProperties() {
+    MetastoreConf.setHiveSiteLocation(null);
     for (MetastoreConf.ConfVars var : MetastoreConf.dataNucleusAndJdoConfs) {
       System.getProperties().remove(var.getVarname());
     }