You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2025/11/27 10:10:59 UTC
(hudi) branch master updated: fix: fix get empty completion time in corner case (#14379)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new aff607b1c975 fix: fix get empty completion time in corner case (#14379)
aff607b1c975 is described below
commit aff607b1c9756c10715afa98c8fa0c599225924a
Author: TheR1sing3un <ch...@apache.org>
AuthorDate: Thu Nov 27 18:10:51 2025 +0800
fix: fix get empty completion time in corner case (#14379)
the corner case: the load instant range is contained within one of the archived file instant range.
---------
Signed-off-by: TheR1sing3un <ch...@apache.org>
Co-authored-by: danny0405 <yu...@gmail.com>
---
.../timeline/TestCompletionTimeQueryView.java | 47 ++++++++++++++++++++--
.../table/timeline/HoodieArchivedTimeline.java | 13 ++++++
.../hudi/common/table/timeline/LSMTimeline.java | 2 +-
3 files changed, 57 insertions(+), 5 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
index 5e5434644c16..8b77c1459fc2 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/timeline/TestCompletionTimeQueryView.java
@@ -53,6 +53,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import static org.apache.hudi.common.testutils.HoodieTestUtils.INSTANT_FILE_NAME_GENERATOR;
@@ -105,6 +106,27 @@ public class TestCompletionTimeQueryView {
}
}
+ @Test
+ void testReadCompletionTimeWithCornerCase() throws Exception {
+ String tableName = "testTable";
+ String tablePath = tempFile.getAbsolutePath() + StoragePath.SEPARATOR + tableName;
+ HoodieTableMetaClient metaClient = HoodieTestUtils.init(
+ HoodieTestUtils.getDefaultStorageConf(), tablePath, HoodieTableType.COPY_ON_WRITE, tableName);
+ prepareTimeline(tablePath, metaClient, (lsmTimelineWriter, activeActions) -> {
+ // archive [1, 2], [3, 6] specifically to create corner cases
+ lsmTimelineWriter.write(activeActions.subList(0, 2), Option.empty(), Option.empty());
+ lsmTimelineWriter.write(activeActions.subList(2, 6), Option.empty(), Option.empty());
+ });
+ try (CompletionTimeQueryView view =
+ metaClient.getTableFormat().getTimelineFactory().createCompletionTimeQueryView(metaClient)) {
+ // 1. first time, we try to load archived instant from 5, so the cursor will move forward from 7 to 5
+ assertThat(view.getCompletionTime(String.format("%08d", 5)).orElse(""), is(String.format("%08d", 1005)));
+
+ // 2. then we try to load archived instant from 4, it should get the completion time correctly
+ assertThat(view.getCompletionTime(String.format("%08d", 4)).orElse(""), is(String.format("%08d", 1004)));
+ }
+ }
+
@Test
void testReadStartTime() throws Exception {
String tableName = "testTable";
@@ -180,6 +202,16 @@ public class TestCompletionTimeQueryView {
}
private void prepareTimeline(String tablePath, HoodieTableMetaClient metaClient, List<Pair<String, String>> instantRequestedAndCompletionTime, String requestedInstantTime) throws Exception {
+ prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime, requestedInstantTime, (writer, activeActions) -> {
+ // archive [1,2], [3,4], [5,6] separately
+ writer.write(activeActions.subList(0, 2), Option.empty(), Option.empty());
+ writer.write(activeActions.subList(2, 4), Option.empty(), Option.empty());
+ writer.write(activeActions.subList(4, 6), Option.empty(), Option.empty());
+ });
+ }
+
+ private void prepareTimeline(String tablePath, HoodieTableMetaClient metaClient, List<Pair<String, String>> instantRequestedAndCompletionTime, String requestedInstantTime,
+ BiConsumer<LSMTimelineWriter, List<ActiveAction>> timelineWriterListBiConsumer) throws Exception {
HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder().withPath(tablePath)
.withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.INMEMORY).build())
.withMarkersType("DIRECT")
@@ -199,10 +231,7 @@ public class TestCompletionTimeQueryView {
testTable.addRequestedCommit(requestedInstantTime);
List<HoodieInstant> instants = TIMELINE_FACTORY.createActiveTimeline(metaClient, false).getInstantsAsStream().sorted().collect(Collectors.toList());
LSMTimelineWriter writer = LSMTimelineWriter.getInstance(writeConfig, getMockHoodieTable(metaClient));
- // archive [1,2], [3,4], [5,6] separately
- writer.write(activeActions.subList(0, 2), Option.empty(), Option.empty());
- writer.write(activeActions.subList(2, 4), Option.empty(), Option.empty());
- writer.write(activeActions.subList(4, 6), Option.empty(), Option.empty());
+ timelineWriterListBiConsumer.accept(writer, activeActions);
// reconcile the active timeline
instants.subList(0, 3 * 6).forEach(
instant -> TimelineUtils.deleteInstantFile(metaClient.getStorage(),
@@ -222,6 +251,16 @@ public class TestCompletionTimeQueryView {
prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime, String.format("%08d", 11));
}
+ private void prepareTimeline(String tablePath, HoodieTableMetaClient metaClient, BiConsumer<LSMTimelineWriter, List<ActiveAction>> timelineWriterListBiConsumer) throws Exception {
+ List<Pair<String, String>> instantRequestedAndCompletionTime = new ArrayList<>();
+ for (int i = 1; i < 11; i++) {
+ String instantTime = String.format("%08d", i);
+ String completionTime = String.format("%08d", i + 1000);
+ instantRequestedAndCompletionTime.add(Pair.of(instantTime, completionTime));
+ }
+ prepareTimeline(tablePath, metaClient, instantRequestedAndCompletionTime, String.format("%08d", 11), timelineWriterListBiConsumer);
+ }
+
@SuppressWarnings("rawtypes")
private HoodieTable getMockHoodieTable(HoodieTableMetaClient metaClient) {
HoodieTable hoodieTable = mock(HoodieTable.class);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
index 5c79864fe5ae..6393f417b22e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/HoodieArchivedTimeline.java
@@ -82,6 +82,19 @@ public interface HoodieArchivedTimeline extends HoodieTimeline {
public boolean isInRange(String instantTime) {
return InstantComparison.isInRange(instantTime, this.startTs, this.endTs);
}
+
+ /**
+ * Returns whether the given instant time range has overlapping with the current range.
+ */
+ public boolean hasOverlappingInRange(String startInstant, String endInstant) {
+ return isInRange(startInstant) || isInRange(endInstant) || isContained(startInstant, endInstant);
+ }
+
+ private boolean isContained(String startInstant, String endInstant) {
+ // the given range is finite and expected to be covered by the current range which naturally cannot be infinite.
+ return startTs != null && InstantComparison.compareTimestamps(startTs, InstantComparison.GREATER_THAN_OR_EQUALS, startInstant)
+ && endTs != null && InstantComparison.compareTimestamps(endTs, InstantComparison.LESSER_THAN_OR_EQUALS, endInstant);
+ }
}
/**
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
index 2c06bdcd010d..c99dcb67f0bd 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/timeline/LSMTimeline.java
@@ -142,7 +142,7 @@ public class LSMTimeline {
public static boolean isFileInRange(HoodieArchivedTimeline.TimeRangeFilter filter, String fileName) {
String minInstant = getMinInstantTime(fileName);
String maxInstant = getMaxInstantTime(fileName);
- return filter.isInRange(minInstant) || filter.isInRange(maxInstant);
+ return filter.hasOverlappingInRange(minInstant, maxInstant);
}
/**