You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ek...@apache.org on 2018/12/14 00:07:19 UTC

[1/2] hive git commit: HIVE-20941: Compactor produces a delete_delta_x_y even if there are no input delete events (Igor Kryvenko via Eugene Koifman)

Repository: hive
Updated Branches:
  refs/heads/master b5b637127 -> e8e0396c1


HIVE-20941: Compactor produces a delete_delta_x_y even if there are no input delete events (Igor Kryvenko via Eugene Koifman)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7a3cac29
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7a3cac29
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7a3cac29

Branch: refs/heads/master
Commit: 7a3cac298b882e5a81838ba74074e0633294e479
Parents: b5b6371
Author: Igor Kryvenko <kr...@gmail.com>
Authored: Thu Dec 13 15:52:40 2018 -0800
Committer: Eugene Koifman <ek...@apache.org>
Committed: Thu Dec 13 15:52:40 2018 -0800

----------------------------------------------------------------------
 .../hive/ql/txn/compactor/TestCompactor.java    | 41 ++--------------
 .../hive/ql/txn/compactor/CompactorMR.java      | 49 ++++++++------------
 .../apache/hadoop/hive/ql/TestTxnCommands2.java | 26 +++++++++++
 .../apache/hadoop/hive/ql/TestTxnCommands3.java | 14 ++----
 4 files changed, 55 insertions(+), 75 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/7a3cac29/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
index beb36d7..5af047f 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java
@@ -17,11 +17,11 @@
  */
 package org.apache.hadoop.hive.ql.txn.compactor;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 
 import java.io.File;
-import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -80,7 +80,6 @@ import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.hive.common.util.Retry;
-import org.apache.hive.common.util.RetryTestRunner;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.streaming.DelimitedInputWriter;
 import org.apache.hive.hcatalog.streaming.HiveEndPoint;
@@ -1459,25 +1458,10 @@ public class TestCompactor {
     checkExpectedTxnsPresent(null, new Path[]{minorCompactedDelta}, columnNamesProperty, columnTypesProperty,
       0, 1L, 2L, 1);
 
-    // Verify that we have got correct set of delete_deltas.
+    //Assert that we have no delete deltas if there are no input delete events.
     FileStatus[] deleteDeltaStat =
       fs.listStatus(new Path(table.getSd().getLocation()), AcidUtils.deleteEventDeltaDirFilter);
-    String[] deleteDeltas = new String[deleteDeltaStat.length];
-    Path minorCompactedDeleteDelta = null;
-    for (int i = 0; i < deleteDeltas.length; i++) {
-      deleteDeltas[i] = deleteDeltaStat[i].getPath().getName();
-      if (deleteDeltas[i].equals("delete_delta_0000001_0000002_v0000005")) {
-        minorCompactedDeleteDelta = deleteDeltaStat[i].getPath();
-      }
-    }
-    Arrays.sort(deleteDeltas);
-    String[] expectedDeleteDeltas = new String[]{"delete_delta_0000001_0000002_v0000005"};
-    if (!Arrays.deepEquals(expectedDeleteDeltas, deleteDeltas)) {
-      Assert.fail("Expected: " + Arrays.toString(expectedDeleteDeltas) + ", found: " + Arrays.toString(deleteDeltas));
-    }
-    // There should be no rows in the delete_delta because there have been no delete events.
-    checkExpectedTxnsPresent(null, new Path[]{minorCompactedDeleteDelta}, columnNamesProperty, columnTypesProperty, 0,
-      0L, 0L, 1);
+    assertEquals(0, deleteDeltaStat.length);
   }
 
   @Test
@@ -1550,25 +1534,10 @@ public class TestCompactor {
     checkExpectedTxnsPresent(null, new Path[]{resultFile}, columnNamesProperty, columnTypesProperty,
       0, 1L, 4L, 1);
 
-    // Verify that we have got correct set of delete_deltas also
+    //Assert that we have no delete deltas if there are no input delete events.
     FileStatus[] deleteDeltaStat =
       fs.listStatus(new Path(table.getSd().getLocation()), AcidUtils.deleteEventDeltaDirFilter);
-    String[] deleteDeltas = new String[deleteDeltaStat.length];
-    Path minorCompactedDeleteDelta = null;
-    for (int i = 0; i < deleteDeltas.length; i++) {
-      deleteDeltas[i] = deleteDeltaStat[i].getPath().getName();
-      if (deleteDeltas[i].equals("delete_delta_0000001_0000004_v0000009")) {
-        minorCompactedDeleteDelta = deleteDeltaStat[i].getPath();
-      }
-    }
-    Arrays.sort(deleteDeltas);
-    String[] expectedDeleteDeltas = new String[]{"delete_delta_0000001_0000004_v0000009"};
-    if (!Arrays.deepEquals(expectedDeleteDeltas, deleteDeltas)) {
-      Assert.fail("Expected: " + Arrays.toString(expectedDeleteDeltas) + ", found: " + Arrays.toString(deleteDeltas));
-    }
-    // There should be no rows in the delete_delta because there have been no delete events.
-    checkExpectedTxnsPresent(null, new Path[]{minorCompactedDeleteDelta}, columnNamesProperty, columnTypesProperty, 0,
-      0L, 0L, 1);
+    assertEquals(0, deleteDeltaStat.length);
 
     if (connection1 != null) {
       connection1.close();

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3cac29/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
index c6cb7c5..42ce174 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
@@ -962,23 +962,17 @@ public class CompactorMR {
       V value = reader.createValue();
       getWriter(reporter, reader.getObjectInspector(), split.getBucket());
 
-      AcidUtils.AcidOperationalProperties acidOperationalProperties
-          = AcidUtils.getAcidOperationalProperties(jobConf);
-
-      if (!isMajor && acidOperationalProperties.isSplitUpdate()) {
-        // When split-update is enabled for ACID, we initialize a separate deleteEventWriter
-        // that is used to write all the delete events (in case of minor compaction only). For major
-        // compaction, history is not required to be maintained hence the delete events are processed
-        // but not re-written separately.
-        getDeleteEventWriter(reporter, reader.getObjectInspector(), split.getBucket());
-      }
+      AcidUtils.AcidOperationalProperties acidOperationalProperties = AcidUtils.getAcidOperationalProperties(jobConf);
 
       while (reader.next(identifier, value)) {
         boolean sawDeleteRecord = reader.isDelete(value);
-        if (isMajor && sawDeleteRecord) continue;
-        if (sawDeleteRecord && deleteEventWriter != null) {
-          // When minor compacting, write delete events to a separate file when split-update is
-          // turned on.
+        if (isMajor && sawDeleteRecord) {
+          continue;
+        }
+        if (sawDeleteRecord && acidOperationalProperties.isSplitUpdate()) {
+          if (deleteEventWriter == null) {
+            getDeleteEventWriter(reporter, reader.getObjectInspector(), split.getBucket());
+          }
           deleteEventWriter.write(value);
           reporter.progress();
         } else {
@@ -1027,7 +1021,7 @@ public class CompactorMR {
             .bucket(bucket)
             .statementId(-1)//setting statementId == -1 makes compacted delta files use
             .visibilityTxnId(getCompactorTxnId());
-        //delta_xxxx_yyyy format
+      //delta_xxxx_yyyy format
 
         // Instantiate the underlying output format
         @SuppressWarnings("unchecked")//since there is no way to parametrize instance of Class
@@ -1040,28 +1034,25 @@ public class CompactorMR {
 
     private void getDeleteEventWriter(Reporter reporter, ObjectInspector inspector,
         int bucket) throws IOException {
-      if (deleteEventWriter == null) {
-        AcidOutputFormat.Options options = new AcidOutputFormat.Options(jobConf);
-        options.inspector(inspector)
-          .writingBase(false)
+
+      AcidOutputFormat.Options options = new AcidOutputFormat.Options(jobConf);
+      options.inspector(inspector).writingBase(false)
           .writingDeleteDelta(true)   // this is the option which will make it a delete writer
           .isCompressed(jobConf.getBoolean(IS_COMPRESSED, false))
-          .tableProperties(new StringableMap(jobConf.get(TABLE_PROPS)).toProperties())
-          .reporter(reporter)
+          .tableProperties(new StringableMap(jobConf.get(TABLE_PROPS)).toProperties()).reporter(reporter)
           .minimumWriteId(jobConf.getLong(MIN_TXN, Long.MAX_VALUE))
-          .maximumWriteId(jobConf.getLong(MAX_TXN, Long.MIN_VALUE))
-          .bucket(bucket)
+          .maximumWriteId(jobConf.getLong(MAX_TXN, Long.MIN_VALUE)).bucket(bucket)
           .statementId(-1)//setting statementId == -1 makes compacted delta files use
-            // delta_xxxx_yyyy format
+          // delta_xxxx_yyyy format
           .visibilityTxnId(getCompactorTxnId());
 
-        // Instantiate the underlying output format
-        @SuppressWarnings("unchecked")//since there is no way to parametrize instance of Class
-        AcidOutputFormat<WritableComparable, V> aof =
+      // Instantiate the underlying output format
+      @SuppressWarnings("unchecked")//since there is no way to parametrize instance of Class
+          AcidOutputFormat<WritableComparable, V> aof =
           instantiate(AcidOutputFormat.class, jobConf.get(OUTPUT_FORMAT_CLASS_NAME));
 
-        deleteEventWriter = aof.getRawRecordWriter(new Path(jobConf.get(TMP_LOCATION)), options);
-      }
+      deleteEventWriter = aof.getRawRecordWriter(new Path(jobConf.get(TMP_LOCATION)), options);
+
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3cac29/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index dc39f5e..546ff95 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -2249,6 +2249,32 @@ public class TestTxnCommands2 {
   }
 
   /**
+   * This tests that delete_delta_x_y dirs will be not produced during minor compaction if no input delete events.
+   * See HIVE-20941.
+   * @throws Exception
+   */
+  @Test
+  public void testDeleteEventsCompaction() throws Exception {
+    int[][] tableData1 = {{1, 2}};
+    int[][] tableData2 = {{2, 3}};
+    int[][] tableData3 = {{3, 4}};
+    runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData1));
+    runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData2));
+    runStatementOnDriver("insert into " + Table.ACIDTBL + "(a,b) " + makeValuesClause(tableData3));
+
+    TxnStore txnHandler = TxnUtils.getTxnStore(hiveConf);
+    txnHandler.compact(new CompactionRequest("default", Table.ACIDTBL.name().toLowerCase(), CompactionType.MINOR));
+    runWorker(hiveConf);
+    runCleaner(hiveConf);
+
+    FileSystem fs = FileSystem.get(hiveConf);
+    FileStatus[] fileStatuses = fs.globStatus(new Path(TEST_WAREHOUSE_DIR + "/" + Table.ACIDTBL.name().toLowerCase() + "/*"));
+    for(FileStatus fileStatus : fileStatuses) {
+      Assert.assertFalse(fileStatus.getPath().getName().startsWith(AcidUtils.DELETE_DELTA_PREFIX));
+    }
+  }
+
+  /**
    * takes raw data and turns it into a string as if from Driver.getResults()
    * sorts rows in dictionary order
    */

http://git-wip-us.apache.org/repos/asf/hive/blob/7a3cac29/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
index 7535f84..1138f11 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands3.java
@@ -329,12 +329,9 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests {
     txnMgr2 = swapTxnManager(txnMgr1);
     driver2 = swapDrivers(driver1);
     runStatementOnDriver("alter table T compact 'minor'");//T4
-    TestTxnCommands2.runWorker(hiveConf);//makes delta_1_2 & delete_delta_1_2
+    TestTxnCommands2.runWorker(hiveConf);//makes delta_1_2
          /* Now we should have
      target/warehouse/t/
-     ├── delete_delta_0000001_0000002_v0000019
-     │   ├── _orc_acid_version
-     │   └── bucket_00000
      ├── delta_0000001_0000001_0000
      │   ├── _orc_acid_version
      │   └── bucket_00000
@@ -350,7 +347,6 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests {
         FileUtils.HIDDEN_FILES_PATH_FILTER);
 
     String[] expectedList = new String[] {
-        "/t/delete_delta_0000001_0000002_v0000019",
         "/t/delta_0000001_0000002_v0000019",
         "/t/delta_0000001_0000001_0000",
         "/t/delta_0000002_0000002_0000",
@@ -370,8 +366,7 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests {
     txnMgr1 = swapTxnManager(txnMgr2);
     driver1 = swapDrivers(driver2);
     runStatementOnDriver("commit");//commits T3
-    //so now cleaner should be able to delete delta_0000001_0000001_0000
-    // & delta_0000002_0000002_0000
+    //so now cleaner should be able to delete delta_0000002_0000002_0000
 
     //insert a row so that compactor makes a new delta (due to HIVE-20901)
     runStatementOnDriver("insert into T values(2,5)");//makes delta_3_3 in T5
@@ -379,13 +374,12 @@ public class TestTxnCommands3 extends TxnCommandsBaseForTests {
     runStatementOnDriver("alter table T compact 'minor'");
     TestTxnCommands2.runWorker(hiveConf);
     /*
-    at this point delete|delta_0000001_0000003_v0000022 are visible to everyone
-    so cleaner removes all files shadowed by them (which is everything in this case)
+    at this point delta_0000001_0000003_v0000022 is visible to everyone
+    so cleaner removes all files shadowed by it (which is everything in this case)
     */
     TestTxnCommands2.runCleaner(hiveConf);
 
     expectedList = new String[] {
-        "/t/delete_delta_0000001_0000003_v0000022",
         "/t/delta_0000001_0000003_v0000022"
     };
     actualList = fs.listStatus(new Path(warehousePath + "/t"),


[2/2] hive git commit: HIVE-21020: log which table/partition is being processed by a txn in Worker (Eugene Koifman, reviewed by Jason Dere, Igor Kryvenko)

Posted by ek...@apache.org.
HIVE-21020: log which table/partition is being processed by a txn in Worker (Eugene Koifman, reviewed by Jason Dere, Igor Kryvenko)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e8e0396c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e8e0396c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e8e0396c

Branch: refs/heads/master
Commit: e8e0396c17bd126e116d0326279ed6da91e9ea37
Parents: 7a3cac2
Author: Eugene Koifman <ek...@apache.org>
Authored: Thu Dec 13 15:54:53 2018 -0800
Committer: Eugene Koifman <ek...@apache.org>
Committed: Thu Dec 13 15:54:53 2018 -0800

----------------------------------------------------------------------
 ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/e8e0396c/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
index 4a1cac1..2104341 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.ql.txn.compactor;
 
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hive.common.JavaUtils;
 import org.apache.hadoop.hive.common.ValidCompactorWriteIdList;
 import org.apache.hadoop.hive.common.ValidTxnList;
 import org.apache.hadoop.hive.metastore.api.AbortTxnRequest;
@@ -198,7 +199,7 @@ public class Worker extends CompactorThread {
         jobName.append("-compactor-");
         jobName.append(ci.getFullPartitionName());
 
-        LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName());
+        LOG.info("Starting " + ci.type.toString() + " compaction for " + ci.getFullPartitionName() + " in " + JavaUtils.txnIdToString(compactorTxnId));
         final StatsUpdater su = StatsUpdater.init(ci, txnHandler.findColumnsWithStats(ci), conf,
           runJobAsSelf(runAs) ? runAs : t.getOwner());
         final CompactorMR mr = new CompactorMR();