[hudi] branch master updated: [HUDI-1637] Avoid to rename for bucket update when there is only one flush action during a checkpoint (#2599)

[vi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 97864a4  [HUDI-1637] Avoid to rename for bucket update when there is only one flush action during a checkpoint (#2599)
97864a4 is described below

commit 97864a48c1979ca2b3f0579cf26bba81fba7e46c
Author: Danny Chan <yu...@gmail.com>
AuthorDate: Thu Feb 25 10:21:27 2021 +0800

    [HUDI-1637] Avoid to rename for bucket update when there is only one flush action during a checkpoint (#2599)
    
    Some of the object storages do not have strong read-after-write
    consistency, we should promote to remove the rename operations in the
    future.
---
 .../src/main/java/org/apache/hudi/io/FlinkMergeHandle.java       | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
index cfd1729..3d33b2e 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/FlinkMergeHandle.java
@@ -94,7 +94,8 @@ public class FlinkMergeHandle<T extends HoodieRecordPayload, I, K, O>
    * Use the fileId + "-" + rollNumber as the new fileId of a mini-batch write.
    */
   protected String generatesDataFileName() {
-    return FSUtils.makeDataFileName(instantTime, writeToken, fileId + "-" + rollNumber, hoodieTable.getBaseFileExtension());
+    final String fileID = this.needBootStrap ? fileId : fileId + "-" + rollNumber;
+    return FSUtils.makeDataFileName(instantTime, writeToken, fileID, hoodieTable.getBaseFileExtension());
   }
 
   public boolean isNeedBootStrap() {
@@ -178,6 +179,12 @@ public class FlinkMergeHandle<T extends HoodieRecordPayload, I, K, O>
   }
 
   public void finishWrite() {
+    // The file visibility should be kept by the configured ConsistencyGuard instance.
+    if (rolloverPaths.size() == 1) {
+      // only one flush action, no need to roll over
+      return;
+    }
+
     for (int i = 0; i < rolloverPaths.size() - 1; i++) {
       Path path = rolloverPaths.get(i);
       try {