[incubator-hudi] branch master updated: HUDI-131 Zero FIle Listing in Compactor run

[vi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 99b0c72  HUDI-131 Zero FIle Listing in Compactor run
99b0c72 is described below

commit 99b0c72aa6e02a86517bc674bdd6b18f7842c7d8
Author: Balaji Varadarajan <va...@uber.com>
AuthorDate: Fri May 24 16:51:09 2019 -0700

    HUDI-131 Zero FIle Listing in Compactor run
---
 .../com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java     | 5 ++---
 .../main/java/com/uber/hoodie/common/model/CompactionOperation.java  | 5 +++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
index 024bd3a..42c4755 100644
--- a/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
+++ b/hoodie-client/src/main/java/com/uber/hoodie/io/compact/HoodieRealtimeTableCompactor.java
@@ -126,14 +126,13 @@ public class HoodieRealtimeTableCompactor implements HoodieCompactor {
       return Lists.<WriteStatus>newArrayList();
     }
 
-    Option<HoodieDataFile> oldDataFileOpt = hoodieCopyOnWriteTable.getROFileSystemView()
-        .getDataFileOn(operation.getPartitionPath(), operation.getBaseInstantTime(), operation.getFileId());
+    Option<HoodieDataFile> oldDataFileOpt = operation.getBaseFile();
 
     // Compacting is very similar to applying updates to existing file
     Iterator<List<WriteStatus>> result;
     // If the dataFile is present, there is a base parquet file present, perform updates else perform inserts into a
     // new base parquet file.
-    if (operation.getDataFilePath().isPresent()) {
+    if (oldDataFileOpt.isPresent()) {
       result = hoodieCopyOnWriteTable
           .handleUpdate(commitTime, operation.getFileId(), scanner.getRecords(), oldDataFileOpt.get());
     } else {
diff --git a/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java b/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java
index e369f71..35ebea9 100644
--- a/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java
+++ b/hoodie-common/src/main/java/com/uber/hoodie/common/model/CompactionOperation.java
@@ -110,6 +110,11 @@ public class CompactionOperation implements Serializable {
     return id;
   }
 
+  public Option<HoodieDataFile> getBaseFile() {
+    //TODO: HUDI-130 - Paths return in compaction plan needs to be relative to base-path
+    return dataFilePath.map(df -> new HoodieDataFile(df));
+  }
+
   /**
    * Convert Avro generated Compaction operation to POJO for Spark RDD operation
    * @param operation Hoodie Compaction Operation