You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vh...@apache.org on 2026/08/05 13:26:46 UTC
(hudi) branch release-1.2.1 updated (9b701ca1e27e -> 5223271b194d)
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 9b701ca1e27e feat(flink): Support writing out-of-line BLOB columns (#18958)
new a64cefc09293 fix(flink): reuse the preceeding avg size if there is no eligible estimation (#19022)
new 15dd0838f1a8 test(flink): de-flake testStreamReadMorTableWithCompactionFromEarliest (#19019)
new 783a9935c80f test(flink): retry short CollectSink reads to de-flake stream-read ITs (#19030)
new 38f524b48bdc test(trino): de-flake TestHudi*FileOperations by asserting only synchronous reads (#19004)
new 5223271b194d fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats (#18904)
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../client/heartbeat/HoodieHeartbeatClient.java | 150 ++++++++++++++++-----
.../org/apache/hudi/config/HoodieWriteConfig.java | 6 +-
.../heartbeat/TestHoodieHeartbeatClient.java | 102 ++++++++++++++
.../partitioner/profile/DeltaWriteProfile.java | 6 +-
.../sink/partitioner/profile/WriteProfile.java | 8 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 85 ++++++++++++
.../apache/hudi/table/ITTestHoodieDataSource.java | 114 +++++++++++++++-
.../hudi/TestHudiAlluxioCacheFileOperations.java | 79 +++++++----
.../hudi/TestHudiMemoryCacheFileOperations.java | 26 ++--
.../plugin/hudi/TestHudiNoCacheFileOperations.java | 26 ++--
10 files changed, 492 insertions(+), 110 deletions(-)
(hudi) 02/05: test(flink): de-flake testStreamReadMorTableWithCompactionFromEarliest (#19019)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 15dd0838f1a8ecd56deaf1d990caa0c191bef43b
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Wed Jun 17 10:42:08 2026 +0700
test(flink): de-flake testStreamReadMorTableWithCompactionFromEarliest (#19019)
(cherry picked from commit d671c276261a7ac8fdda88f6169df571624637c9)
---
.../apache/hudi/table/ITTestHoodieDataSource.java | 42 ++++++++++++++++++++++
1 file changed, 42 insertions(+)
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index 32b5b9476f14..301450a2602f 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -3892,6 +3892,15 @@ public class ITTestHoodieDataSource {
// fail the job on stream-closed-mid-read (the right behavior for real I/O
// failures), so this tolerance is scoped to the SuccessException-based test
// pattern below and is NOT mirrored in production code.
+ // 3. NullPointerException from ParquetColumnarRowSplitReader#readNextRowGroup: the
+ // same benign teardown race as (2), observed with different timing. When the
+ // SplitFetcher's close() fully completes first, ParquetColumnarRowSplitReader#close
+ // nulls out its `reader` field, so the in-flight row-group read on the task thread
+ // surfaces as a NullPointerException (reader.readNextRowGroup() on a null reader)
+ // instead of an IOException("Stream is closed!"). Same functional outcome - the
+ // sink has already collected the expected rows - only the error symptom differs.
+ // Tolerated narrowly (an NPE originating from that exact frame) for the same
+ // reason as (2), and likewise NOT mirrored in production code.
if (!isAcceptableTerminalFailure(e)) {
throw new AssertionError("Unexpected job failure", e);
}
@@ -3917,8 +3926,41 @@ public class ITTestHoodieDataSource {
if (msg != null && msg.contains("Stream is closed")) {
return true;
}
+ // The NPE twin of the "Stream is closed!" teardown race (cause #3 at the call site):
+ // a NullPointerException whose own stack trace originates from
+ // ParquetColumnarRowSplitReader#readNextRowGroup, i.e. reader.readNextRowGroup() ran on a
+ // null `reader` that ParquetColumnarRowSplitReader#close had just nulled out. Scoped to
+ // that exact frame so genuine NPEs - and the legitimate IOException("expecting more
+ // rows...") thrown from the same method - still fail the test.
+ if (isNullPointerException(cur) && containsReadNextRowGroupFrame(cur)) {
+ return true;
+ }
cur = cur.getCause();
}
return false;
}
+
+ /**
+ * True for a real {@link NullPointerException} as well as one wrapped in Flink's
+ * {@code SerializedThrowable} when the failure is propagated back from the cluster (its
+ * {@code toString()} preserves the original {@code java.lang.NullPointerException} prefix).
+ */
+ private static boolean isNullPointerException(Throwable t) {
+ return t instanceof NullPointerException
+ || t.toString().startsWith(NullPointerException.class.getName());
+ }
+
+ /**
+ * Whether {@code t}'s stack trace (preserved even through {@code SerializedThrowable})
+ * contains a {@code ParquetColumnarRowSplitReader#readNextRowGroup} frame.
+ */
+ private static boolean containsReadNextRowGroupFrame(Throwable t) {
+ for (StackTraceElement frame : t.getStackTrace()) {
+ if (frame.getClassName().endsWith("ParquetColumnarRowSplitReader")
+ && "readNextRowGroup".equals(frame.getMethodName())) {
+ return true;
+ }
+ }
+ return false;
+ }
}
(hudi) 01/05: fix(flink): reuse the preceeding avg size if there is no eligible estimation (#19022)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a64cefc09293a0fb92e37455f9c484c05f816fcb
Author: Danny Chan <yu...@gmail.com>
AuthorDate: Wed Jun 17 11:25:41 2026 +0800
fix(flink): reuse the preceeding avg size if there is no eligible estimation (#19022)
* fix(flink): reuse the preceeding avg size if there is no eligible estimation
(cherry picked from commit 5579715ef7d926fc68c9b0dc1918ff835c798e89)
---
.../partitioner/profile/DeltaWriteProfile.java | 6 +-
.../sink/partitioner/profile/WriteProfile.java | 8 +-
.../hudi/sink/partitioner/TestBucketAssigner.java | 85 ++++++++++++++++++++++
3 files changed, 90 insertions(+), 9 deletions(-)
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java
index a79c78e6d363..2cdee3453a46 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/DeltaWriteProfile.java
@@ -29,8 +29,6 @@ import org.apache.hudi.common.table.view.SyncableFileSystemView;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.table.action.commit.SmallFile;
-import lombok.extern.slf4j.Slf4j;
-
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -41,7 +39,6 @@ import java.util.stream.Collectors;
*
* <p>Note: assumes the index can always index log files for Flink write.
*/
-@Slf4j
public class DeltaWriteProfile extends WriteProfile {
public DeltaWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) {
@@ -93,7 +90,7 @@ public class DeltaWriteProfile extends WriteProfile {
@Override
protected long averageBytesPerRecord() {
- long avgSize = config.getCopyOnWriteRecordSizeEstimate();
+ long avgSize = this.avgSize > 0 ? this.avgSize : config.getCopyOnWriteRecordSizeEstimate();
HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
if (!commitTimeline.empty()) {
long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D);
@@ -109,7 +106,6 @@ public class DeltaWriteProfile extends WriteProfile {
}
}
}
- log.info("Refresh average bytes per record => " + avgSize);
return avgSize;
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java
index c3819bf08116..304d8f5d7452 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/partitioner/profile/WriteProfile.java
@@ -78,7 +78,7 @@ public class WriteProfile {
* The average record size.
*/
@Getter
- private long avgSize = -1L;
+ protected long avgSize = -1L;
/**
* Total records to write for each bucket based on
@@ -134,7 +134,7 @@ public class WriteProfile {
* records pack into one file.
*/
protected long averageBytesPerRecord() {
- long avgSize = config.getCopyOnWriteRecordSizeEstimate();
+ long avgSize = this.avgSize > 0 ? this.avgSize : config.getCopyOnWriteRecordSizeEstimate();
HoodieTimeline commitTimeline = metaClient.getCommitTimeline().filterCompletedInstants();
if (!commitTimeline.empty()) {
long sizeFromCommitMetadata = calculateRecordSizeThroughCommitMetadata(commitTimeline, 1.0D);
@@ -142,7 +142,6 @@ public class WriteProfile {
avgSize = sizeFromCommitMetadata;
}
}
- log.info("Refresh average bytes per record => " + avgSize);
return avgSize;
}
@@ -238,8 +237,9 @@ public class WriteProfile {
private void recordProfile() {
this.avgSize = averageBytesPerRecord();
+ log.info("Refresh average bytes per record => {}", avgSize);
this.recordsPerBucket = config.getParquetMaxFileSize() / avgSize;
- log.info("Refresh insert records per bucket => " + recordsPerBucket);
+ log.info("Refresh insert records per bucket => {}", recordsPerBucket);
}
/**
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java
index 8281ebbd86e4..3b5c265f7c91 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/partitioner/TestBucketAssigner.java
@@ -23,6 +23,7 @@ import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieRecordLocation;
import org.apache.hudi.common.model.HoodieTableType;
+import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
@@ -46,11 +47,13 @@ import org.junit.jupiter.api.io.TempDir;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
@@ -444,6 +447,26 @@ public class TestBucketAssigner {
writeProfile.getRecordsPerBucket(), is(writeConfig.getParquetMaxFileSize() / expectedAvgSize));
}
+ @Test
+ public void testWriteProfileReusesPreviousAvgSizeWhenNoEligibleCommitOnReload() throws Exception {
+ conf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1);
+ conf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024");
+ TestData.writeData(TestData.DATA_SET_INSERT, conf);
+
+ writeConfig = FlinkWriteClients.getHoodieClientConfig(conf);
+ setScriptedRecordSizes(512L, -1L);
+ WriteProfile writeProfile = new ScriptedRecordSizeWriteProfile(writeConfig, context);
+ assertThat("Average record size should use the profiled commit metadata",
+ writeProfile.getAvgSize(), is(512L));
+
+ writeProfile.reload(1);
+
+ assertThat("Average record size should reuse the previous estimate when no eligible commit metadata is found",
+ writeProfile.getAvgSize(), is(512L));
+ assertThat("Records per bucket should continue to use the previous estimate",
+ writeProfile.getRecordsPerBucket(), is(writeConfig.getParquetMaxFileSize() / 512L));
+ }
+
@Test
public void testDeltaWriteProfileRecordsPerBucketUsesCompressionRatio() throws Exception {
File morPath = new File(tempFile, "mor");
@@ -505,6 +528,34 @@ public class TestBucketAssigner {
writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / expectedAvgSize));
}
+ @Test
+ public void testDeltaWriteProfileReusesPreviousAvgSizeWhenNoEligibleDeltaCommitOnReload() throws Exception {
+ File morPath = new File(tempFile, "mor_reuse_previous_avg");
+ Configuration morConf = TestConfigurations.getDefaultConf(morPath.getAbsolutePath());
+ morConf.set(FlinkOptions.TABLE_TYPE, HoodieTableType.MERGE_ON_READ.name());
+ morConf.set(FlinkOptions.WRITE_PARQUET_MAX_FILE_SIZE, 1);
+ morConf.setString(HoodieCompactionConfig.COPY_ON_WRITE_RECORD_SIZE_ESTIMATE.key(), "1024");
+ StreamerUtil.initTableIfNotExists(morConf);
+ TestData.writeData(TestData.DATA_SET_INSERT, morConf);
+
+ HoodieWriteConfig morWriteConfig = FlinkWriteClients.getHoodieClientConfig(morConf);
+ HoodieFlinkEngineContext morContext = new HoodieFlinkEngineContext(
+ HadoopFSUtils.getStorageConf(HadoopConfigurations.getHadoopConf(morConf)),
+ new FlinkTaskContextSupplier(null));
+
+ setScriptedRecordSizes(256L, -1L);
+ DeltaWriteProfile writeProfile = new ScriptedRecordSizeDeltaWriteProfile(morWriteConfig, morContext);
+ assertThat("Average record size should use the profiled delta commit metadata",
+ writeProfile.getAvgSize(), is(256L));
+
+ writeProfile.reload(1);
+
+ assertThat("Average record size should reuse the previous estimate when no eligible delta commit metadata is found",
+ writeProfile.getAvgSize(), is(256L));
+ assertThat("Records per bucket should continue to use the previous estimate",
+ writeProfile.getRecordsPerBucket(), is(morWriteConfig.getParquetMaxFileSize() / 256L));
+ }
+
private static String getLastCompleteInstant(WriteProfile profile) {
return StreamerUtil.getLastCompletedInstant(profile.getMetaClient());
}
@@ -526,6 +577,40 @@ public class TestBucketAssigner {
assertThat(bucketInfo.getBucketType(), is(bucketType));
}
+ private static Queue<Long> scriptedRecordSizes = new ArrayDeque<>();
+
+ private static void setScriptedRecordSizes(Long... recordSizes) {
+ scriptedRecordSizes = new ArrayDeque<>(Arrays.asList(recordSizes));
+ }
+
+ /**
+ * WriteProfile with scripted record size estimates.
+ */
+ static class ScriptedRecordSizeWriteProfile extends WriteProfile {
+ ScriptedRecordSizeWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) {
+ super(config, context);
+ }
+
+ @Override
+ protected long calculateRecordSizeThroughCommitMetadata(HoodieTimeline commitTimeline, double fileSizeCalibrationRatio) {
+ return scriptedRecordSizes.remove();
+ }
+ }
+
+ /**
+ * DeltaWriteProfile with scripted record size estimates.
+ */
+ static class ScriptedRecordSizeDeltaWriteProfile extends DeltaWriteProfile {
+ ScriptedRecordSizeDeltaWriteProfile(HoodieWriteConfig config, HoodieFlinkEngineContext context) {
+ super(config, context);
+ }
+
+ @Override
+ protected long calculateRecordSizeThroughCommitMetadata(HoodieTimeline commitTimeline, double fileSizeCalibrationRatio) {
+ return scriptedRecordSizes.remove();
+ }
+ }
+
/**
* Mock BucketAssigner that can specify small files explicitly.
*/
(hudi) 04/05: test(trino): de-flake TestHudi*FileOperations by asserting only synchronous reads (#19004)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 38f524b48bdc6713a77a957a97ea858923ce7f87
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Thu Jun 18 14:41:52 2026 +0700
test(trino): de-flake TestHudi*FileOperations by asserting only synchronous reads (#19004)
(cherry picked from commit 4bafe5a6075ef27691ef126cb412759d83e4a611)
---
.../hudi/TestHudiAlluxioCacheFileOperations.java | 79 ++++++++++++++--------
.../hudi/TestHudiMemoryCacheFileOperations.java | 26 +++----
.../plugin/hudi/TestHudiNoCacheFileOperations.java | 26 +++----
3 files changed, 70 insertions(+), 61 deletions(-)
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java
index 89507721afa5..911c5d4c3e7d 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiAlluxioCacheFileOperations.java
@@ -17,6 +17,7 @@ import com.google.common.collect.HashMultiset;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMultiset;
import com.google.common.collect.Multiset;
+import io.airlift.units.Duration;
import io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer;
import io.trino.plugin.hudi.util.FileOperationUtils.FileOperation;
import io.trino.testing.AbstractTestQueryFramework;
@@ -36,7 +37,6 @@ import static com.google.common.io.MoreFiles.deleteRecursively;
import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
import static io.trino.filesystem.tracing.CacheFileSystemTraceUtils.getCacheOperationSpans;
import static io.trino.plugin.hudi.testing.ResourceHudiTablesInitializer.TestingTable.HUDI_MULTI_FG_PT_V8_MOR;
-import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.DATA;
import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.INDEX_DEFINITION;
import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.LOG;
import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TABLE;
@@ -44,7 +44,9 @@ import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.METADATA_TAB
import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TABLE_PROPERTIES;
import static io.trino.plugin.hudi.util.FileOperationUtils.FileType.TIMELINE;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
+import static io.trino.testing.assertions.Assert.assertEventually;
import static java.util.stream.Collectors.toCollection;
+import static org.assertj.core.api.Assertions.assertThat;
@ResourceLock("HUDI_CACHE_SYSTEM")
@Execution(ExecutionMode.SAME_THREAD)
@@ -66,10 +68,11 @@ public class TestHudiAlluxioCacheFileOperations
.put("fs.cache.directories", cacheDirectory.toAbsolutePath().toString())
.put("fs.cache.max-sizes", "100MB")
.put("hudi.metadata.cache.enabled", "false")
- // Disable async table-statistics refresh: it reads the metadata table on a
- // background executor whose spans can outlive the query and leak into the next
- // test's measurement (the symmetric off-by-N flake). Disabling it makes the
- // file-operation counts deterministic right after the query returns.
+ // Disable the async table-statistics refresh: on the first query it reads the index
+ // definitions and table-property files (and the metadata table) on a background
+ // executor. Those non-metadata-table reads land in the asserted set and their timing
+ // is non-deterministic, so we turn the refresh off and assert only the synchronous
+ // planning-path reads.
.put("hudi.table-statistics-enabled", "false")
.buildOrThrow();
@@ -87,12 +90,6 @@ public class TestHudiAlluxioCacheFileOperations
assertFileSystemAccesses(
query,
ImmutableMultiset.<FileOperation>builder()
- .addCopies(new FileOperation("Alluxio.readCached", DATA), 2)
- .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 27)
- .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 4)
- .addCopies(new FileOperation("Alluxio.readCached", LOG), 15)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
- .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 10)
.addCopies(new FileOperation("InputFile.length", TIMELINE), 2)
.addCopies(new FileOperation("InputFile.length", LOG), 1)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
@@ -103,12 +100,6 @@ public class TestHudiAlluxioCacheFileOperations
assertFileSystemAccesses(
query,
ImmutableMultiset.<FileOperation>builder()
- .addCopies(new FileOperation("Alluxio.readCached", DATA), 2)
- .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 27)
- .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 4)
- .addCopies(new FileOperation("Alluxio.readCached", LOG), 15)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
- .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 10)
.addCopies(new FileOperation("InputFile.length", TIMELINE), 2)
.addCopies(new FileOperation("InputFile.length", LOG), 1)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
@@ -127,12 +118,6 @@ public class TestHudiAlluxioCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperation>builder()
- .addCopies(new FileOperation("Alluxio.readCached", DATA), 6)
- .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 215)
- .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 8)
- .addCopies(new FileOperation("Alluxio.readCached", LOG), 30)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
- .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 69)
.addCopies(new FileOperation("InputFile.length", TIMELINE), 4)
.addCopies(new FileOperation("InputFile.length", LOG), 2)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
@@ -142,12 +127,6 @@ public class TestHudiAlluxioCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperation>builder()
- .addCopies(new FileOperation("Alluxio.readCached", DATA), 6)
- .addCopies(new FileOperation("Alluxio.readCached", METADATA_TABLE), 215)
- .addCopies(new FileOperation("Alluxio.readCached", TIMELINE), 8)
- .addCopies(new FileOperation("Alluxio.readCached", LOG), 30)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
- .addCopies(new FileOperation("InputFile.length", METADATA_TABLE), 69)
.addCopies(new FileOperation("InputFile.length", TIMELINE), 4)
.addCopies(new FileOperation("InputFile.length", LOG), 2)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
@@ -156,6 +135,40 @@ public class TestHudiAlluxioCacheFileOperations
.build());
}
+ @Test
+ public void testReadsServedFromAlluxioCache()
+ {
+ // The tests above intentionally do not assert exact Alluxio cache hit/miss counts: the cache
+ // write is asynchronous, so the per-query counts flake (a write from one query can still be in
+ // flight when the next query runs). This test instead gives count-independent coverage that the
+ // Alluxio cache is actually engaged: once the cache is warmed, at least one read is served from
+ // it (an "Alluxio.readCached" span). assertEventually re-runs the query until the asynchronous
+ // cache write has landed and a genuine hit is observed, and fails loudly at the deadline if the
+ // cache never serves a read.
+ @Language("SQL") String query = "SELECT * FROM " + HUDI_MULTI_FG_PT_V8_MOR;
+ DistributedQueryRunner queryRunner = getDistributedQueryRunner();
+
+ // Warm the cache; the page write into Alluxio happens on a background thread.
+ queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query);
+
+ assertEventually(
+ Duration.valueOf("30s"),
+ Duration.valueOf("500ms"),
+ () -> {
+ queryRunner.executeWithPlan(queryRunner.getDefaultSession(), query);
+ assertThat(countCachedReads(queryRunner))
+ .as("Alluxio.readCached spans (cache hits)")
+ .isGreaterThanOrEqualTo(1);
+ });
+ }
+
+ private static long countCachedReads(QueryRunner queryRunner)
+ {
+ return getCacheOperationSpans(queryRunner).stream()
+ .filter(span -> span.getName().equals("Alluxio.readCached"))
+ .count();
+ }
+
private void assertFileSystemAccesses(@Language("SQL") String query, Multiset<FileOperation> expectedCacheAccesses)
{
DistributedQueryRunner queryRunner = getDistributedQueryRunner();
@@ -169,6 +182,14 @@ public class TestHudiAlluxioCacheFileOperations
.stream()
.filter(span -> !span.getName().startsWith("InputFile.exists"))
.map(FileOperation::create)
+ // Metadata-table reads are issued from Hudi background pools (split loading, partition
+ // listing, table-statistics refresh) whose spans can outlive the synchronous query and
+ // land in the next query's measurement window, so their per-query counts are not
+ // deterministic. Alluxio cache hits/misses (Alluxio.*) depend on whether an earlier
+ // asynchronous cache write had already completed, so their counts are not deterministic
+ // either. Both are excluded; only synchronous foreground reads are asserted.
+ .filter(operation -> operation.fileType() != METADATA_TABLE)
+ .filter(operation -> !operation.operationType().startsWith("Alluxio."))
.collect(toCollection(HashMultiset::create));
}
}
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java
index 61867b1cde7d..8e1b2fe81911 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiMemoryCacheFileOperations.java
@@ -56,10 +56,11 @@ public class TestHudiMemoryCacheFileOperations
.put("hudi.metadata-enabled", "true")
.put("hudi.metadata.cache.enabled", "true")
.put("fs.cache.enabled", "false")
- // Disable async table-statistics refresh: it reads the metadata table on a
- // background executor whose spans can outlive the query and leak into the next
- // test's measurement (the symmetric off-by-N flake). Disabling it makes the
- // file-operation counts deterministic right after the query returns.
+ // Disable the async table-statistics refresh: on the first query it reads the index
+ // definitions and table-property files (and the metadata table) on a background
+ // executor. Those non-metadata-table reads land in the asserted set and their timing
+ // is non-deterministic, so we turn the refresh off and assert only the synchronous
+ // planning-path reads.
.put("hudi.table-statistics-enabled", "false")
.buildOrThrow();
@@ -78,11 +79,8 @@ public class TestHudiMemoryCacheFileOperations
query,
ImmutableMultiset.<FileOperation>builder()
.addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2)
- .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 4)
- .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 6)
.addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2)
.addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
.add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES))
.addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2)
@@ -92,11 +90,8 @@ public class TestHudiMemoryCacheFileOperations
query,
ImmutableMultiset.<FileOperation>builder()
.addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 2)
- .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 4)
- .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 6)
.addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 2)
.addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 1)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
.add(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES))
.addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 2)
@@ -114,11 +109,8 @@ public class TestHudiMemoryCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperation>builder()
.addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6)
- .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 29)
- .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 40)
.addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4)
.addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
.addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2)
.addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4)
@@ -127,11 +119,8 @@ public class TestHudiMemoryCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperation>builder()
.addCopies(new FileOperation("FileSystemCache.cacheInput", DATA), 6)
- .addCopies(new FileOperation("FileSystemCache.cacheLength", METADATA_TABLE), 29)
- .addCopies(new FileOperation("FileSystemCache.cacheStream", METADATA_TABLE), 40)
.addCopies(new FileOperation("FileSystemCache.cacheStream", TIMELINE), 4)
.addCopies(new FileOperation("FileSystemCache.cacheStream", LOG), 2)
- .addCopies(new FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
.addCopies(new FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
.addCopies(new FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2)
.addCopies(new FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4)
@@ -153,6 +142,11 @@ public class TestHudiMemoryCacheFileOperations
.filter(span -> !span.getName().startsWith("InputFile.exists"))
.filter(span -> !isTrinoSchemaOrPermissions(getFileLocation(span)))
.map(FileOperation::create)
+ // Metadata-table reads are issued from Hudi background pools (split loading, partition
+ // listing, table-statistics refresh) whose spans can outlive the synchronous query and
+ // land in the next query's measurement window. Their per-query counts are therefore
+ // non-deterministic, so they are excluded; only synchronous foreground reads are asserted.
+ .filter(operation -> operation.fileType() != METADATA_TABLE)
.collect(toCollection(HashMultiset::create));
}
}
diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java
index 71518b9fc67b..55071241fff7 100644
--- a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java
+++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/TestHudiNoCacheFileOperations.java
@@ -56,10 +56,11 @@ public class TestHudiNoCacheFileOperations
.put("hudi.metadata-enabled", "true")
.put("hudi.metadata.cache.enabled", "false")
.put("fs.cache.enabled", "false")
- // Disable async table-statistics refresh: it reads the metadata table on a
- // background executor whose spans can outlive the query and leak into the next
- // test's measurement (the symmetric off-by-N flake). Disabling it makes the
- // file-operation counts deterministic right after the query returns.
+ // Disable the async table-statistics refresh: on the first query it reads the index
+ // definitions and table-property files (and the metadata table) on a background
+ // executor. Those non-metadata-table reads land in the asserted set and their timing
+ // is non-deterministic, so we turn the refresh off and assert only the synchronous
+ // planning-path reads.
.put("hudi.table-statistics-enabled", "false")
.buildOrThrow();
@@ -78,9 +79,6 @@ public class TestHudiNoCacheFileOperations
query,
ImmutableMultiset.<FileOperationUtils.FileOperation>builder()
.addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 4)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 6)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1)
@@ -92,10 +90,7 @@ public class TestHudiNoCacheFileOperations
query,
ImmutableMultiset.<FileOperationUtils.FileOperation>builder()
.addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 2)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 4)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 4)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 2)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 6)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 2)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", LOG), 1)
.add(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES))
@@ -114,10 +109,7 @@ public class TestHudiNoCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperationUtils.FileOperation>builder()
.addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 6)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 29)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 40)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4)
@@ -127,10 +119,7 @@ public class TestHudiNoCacheFileOperations
assertFileSystemAccesses(query,
ImmutableMultiset.<FileOperationUtils.FileOperation>builder()
.addCopies(new FileOperationUtils.FileOperation("Input.readTail", DATA), 6)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.lastModified", METADATA_TABLE), 29)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.length", METADATA_TABLE), 29)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", INDEX_DEFINITION), 4)
- .addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE), 40)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", METADATA_TABLE_PROPERTIES), 2)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TABLE_PROPERTIES), 4)
.addCopies(new FileOperationUtils.FileOperation("InputFile.newStream", TIMELINE), 4)
@@ -153,6 +142,11 @@ public class TestHudiNoCacheFileOperations
.filter(span -> !span.getName().startsWith("InputFile.exists"))
.filter(span -> !isTrinoSchemaOrPermissions(getFileLocation(span)))
.map(FileOperationUtils.FileOperation::create)
+ // Metadata-table reads are issued from Hudi background pools (split loading, partition
+ // listing, table-statistics refresh) whose spans can outlive the synchronous query and
+ // land in the next query's measurement window. Their per-query counts are therefore
+ // non-deterministic, so they are excluded; only synchronous foreground reads are asserted.
+ .filter(operation -> operation.fileType() != METADATA_TABLE)
.collect(toCollection(HashMultiset::create));
}
}
(hudi) 05/05: fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats (#18904)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5223271b194d2fea8250020ad3c23ac622a548ca
Author: Prashant Wason <pw...@uber.com>
AuthorDate: Mon Jun 22 01:48:47 2026 -0700
fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats (#18904)
* fix: prevent heartbeat timer from being permanently killed by slow or delayed heartbeats
HoodieHeartbeatClient could permanently stop generating heartbeats for an
instant, causing later commits to abort with "Heartbeat for instant ... has
expired" even though the writer was still alive:
- The heartbeat file is written synchronously on the Timer thread. Because the
timer uses scheduleAtFixedRate, a slow or hung storage write blocks the thread
and freezes all subsequent heartbeats for that instant.
- When a heartbeat refresh is delayed past the tolerable interval,
updateHeartbeat() called Thread.currentThread().interrupt(), permanently
killing the timer thread and turning a transient delay into a permanent
blackout.
Fix:
- Perform the heartbeat file write on a bounded daemon executor (Future.get with
a per-interval timeout) so a slow or hung storage call cannot block the timer
thread; a timed-out write is retried on the next tick.
- Remove the self-interrupt; log a warning and continue. The commit-time check
HeartbeatUtils.abortIfHeartbeatExpired() remains the sole enforcement point.
Add TestHoodieHeartbeatClient.testTimerSurvivesHungHeartbeatWrite.
* fix: address review comments on heartbeat resilience
- isHeartbeatExpired: handle a null last-heartbeat-time (the very first write
can time out, leaving it unset) by falling back to the DFS read, avoiding an
NPE on the unboxing comparison.
- On a detected lapse, stop refreshing the heartbeat (cancel the timer) and do
not advance the last-heartbeat time, so a lapsed writer still aborts at commit
via HeartbeatUtils.abortIfHeartbeatExpired() and cannot resurrect a heartbeat
that a concurrent cleaner (LAZY failed-writes policy) may already have acted
on. The timer is cancelled cleanly rather than via Thread.interrupt().
- close(): make idempotent via a closed flag and guard executor creation after
close.
- Use boxed Long for heartbeatWriteTimeoutMs to match the sibling duration
fields.
- Raise the default hoodie.client.heartbeat.tolerable.misses from 2 to 10 so
transient driver pauses (e.g. GC) or storage-latency spikes do not abort a
still-healthy writer.
* fix: keep heartbeat client reusable after close
A write client reuses its HoodieHeartbeatClient across operations: after
close(), startCommit() and acquireRollbackHeartbeatIfMultiWriter() call
heartbeatClient.start() again. The previously added "already closed" guard in
the executor accessor turned this valid reuse into a failure (observed in
TestJavaHoodieBackedMetadata and TestHoodieJavaClientOnCopyOnWriteStorage).
Remove the guard and the closed flag; close() remains idempotent via
synchronization plus the null-check on the executor, so repeated/concurrent
close() is still safe while reuse-after-close works (the executor is lazily
re-created on the next heartbeat).
* switch timer into a executor service
---------
Co-authored-by: danny0405 <yu...@gmail.com>
(cherry picked from commit c6f38c6b6e84212f6f9aec5c88ae271ff424e741)
---
.../client/heartbeat/HoodieHeartbeatClient.java | 150 ++++++++++++++++-----
.../org/apache/hudi/config/HoodieWriteConfig.java | 6 +-
.../heartbeat/TestHoodieHeartbeatClient.java | 102 ++++++++++++++
3 files changed, 224 insertions(+), 34 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
index a043f73e632c..b8f2f15fdf0a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/heartbeat/HoodieHeartbeatClient.java
@@ -19,6 +19,7 @@
package org.apache.hudi.client.heartbeat;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.CustomizedThreadFactory;
import org.apache.hudi.common.util.ValidationUtils;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieHeartbeatException;
@@ -35,9 +36,15 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.Serializable;
import java.util.Map;
-import java.util.Timer;
-import java.util.TimerTask;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import static org.apache.hudi.common.heartbeat.HoodieHeartbeatUtils.getLastHeartbeatTime;
@@ -58,7 +65,16 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
// heartbeat interval in millis
private final Long heartbeatIntervalInMs;
private final Long maxAllowableHeartbeatIntervalInMs;
+ // Maximum time the scheduler thread will wait for a single heartbeat file write to complete before
+ // abandoning it and letting the next tick retry. Bounded to one interval so that a slow/hung
+ // storage write cannot block the scheduler thread (and thus freeze all subsequent heartbeats).
+ private final Long heartbeatWriteTimeoutMs;
private final Map<String, Heartbeat> instantToHeartbeatMap;
+ // Daemon executor used to perform the (potentially slow) storage write off the scheduler thread so the
+ // write can be time-bounded. A cached pool is intentional: if one write hangs, that thread is left
+ // parked while the next tick proceeds on a fresh thread. Lazily created and marked transient since
+ // this client is Serializable with a transient storage handle.
+ private transient ExecutorService heartbeatWriteExecutor;
public HoodieHeartbeatClient(HoodieStorage storage, String basePath, Long heartbeatIntervalInMs,
Integer numTolerableHeartbeatMisses) {
@@ -68,9 +84,18 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
this.heartbeatFolderPath = HoodieTableMetaClient.getHeartbeatFolderPath(basePath);
this.heartbeatIntervalInMs = heartbeatIntervalInMs;
this.maxAllowableHeartbeatIntervalInMs = this.heartbeatIntervalInMs * numTolerableHeartbeatMisses;
+ this.heartbeatWriteTimeoutMs = this.heartbeatIntervalInMs;
this.instantToHeartbeatMap = new ConcurrentHashMap<>();
}
+ private synchronized ExecutorService getHeartbeatWriteExecutor() {
+ if (heartbeatWriteExecutor == null) {
+ heartbeatWriteExecutor =
+ Executors.newCachedThreadPool(new CustomizedThreadFactory("heartbeat_write", true));
+ }
+ return heartbeatWriteExecutor;
+ }
+
@Data
static class Heartbeat {
@@ -79,10 +104,12 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
private boolean isHeartbeatStopped = false;
private Long lastHeartbeatTime;
private Integer numHeartbeats = 0;
- private Timer timer = new Timer(true);
+ private ScheduledExecutorService heartbeatScheduler =
+ Executors.newSingleThreadScheduledExecutor(new CustomizedThreadFactory("heartbeat_scheduler", true));
+ private ScheduledFuture<?> scheduledFuture;
}
- class HeartbeatTask extends TimerTask {
+ class HeartbeatTask implements Runnable {
private final String instantTime;
@@ -92,7 +119,11 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
@Override
public void run() {
- updateHeartbeat(instantTime);
+ try {
+ updateHeartbeat(instantTime);
+ } catch (Exception e) {
+ log.error("Failed to update heartbeat for instant {}; will retry on next tick", instantTime, e);
+ }
}
}
@@ -114,11 +145,11 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
newHeartbeat.setHeartbeatStarted(true);
instantToHeartbeatMap.put(instantTime, newHeartbeat);
// Ensure heartbeat is generated for the first time with this blocking call.
- // Since timer submits the task to a thread, no guarantee when that thread will get CPU
+ // Since scheduler submits the task to a thread, no guarantee when that thread will get CPU
// cycles to generate the first heartbeat.
updateHeartbeat(instantTime);
- newHeartbeat.getTimer().scheduleAtFixedRate(new HeartbeatTask(instantTime), this.heartbeatIntervalInMs,
- this.heartbeatIntervalInMs);
+ newHeartbeat.setScheduledFuture(newHeartbeat.getHeartbeatScheduler().scheduleAtFixedRate(
+ new HeartbeatTask(instantTime), this.heartbeatIntervalInMs, this.heartbeatIntervalInMs, TimeUnit.MILLISECONDS));
}
/**
@@ -130,7 +161,7 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
public Heartbeat stop(String instantTime) throws HoodieException {
Heartbeat heartbeat = instantToHeartbeatMap.remove(instantTime);
if (isHeartbeatStarted(heartbeat)) {
- stopHeartbeatTimer(heartbeat);
+ stopHeartbeatScheduler(heartbeat);
HeartbeatUtils.deleteHeartbeatFile(storage, basePath, instantTime);
log.info("Deleted heartbeat file for instant {}", instantTime);
}
@@ -138,12 +169,12 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
}
/**
- * Stops all timers of heartbeats started via this instance of the client.
+ * Stops all heartbeat schedulers started via this instance of the client.
*
* @throws HoodieException
*/
public void stopHeartbeatTimers() throws HoodieException {
- instantToHeartbeatMap.values().stream().filter(this::isHeartbeatStarted).forEach(this::stopHeartbeatTimer);
+ instantToHeartbeatMap.values().stream().filter(this::isHeartbeatStarted).forEach(this::stopHeartbeatScheduler);
}
/**
@@ -158,17 +189,24 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
}
/**
- * Stops the timer of the given heartbeat.
+ * Stops the scheduler of the given heartbeat.
*
* @param heartbeat The heartbeat to stop.
*/
- private void stopHeartbeatTimer(Heartbeat heartbeat) {
+ private void stopHeartbeatScheduler(Heartbeat heartbeat) {
log.info("Stopping heartbeat for instant {}", heartbeat.getInstantTime());
- heartbeat.getTimer().cancel();
+ shutdownHeartbeatScheduler(heartbeat);
heartbeat.setHeartbeatStopped(true);
log.info("Stopped heartbeat for instant {}", heartbeat.getInstantTime());
}
+ private void shutdownHeartbeatScheduler(Heartbeat heartbeat) {
+ if (heartbeat.getScheduledFuture() != null) {
+ heartbeat.getScheduledFuture().cancel(false);
+ }
+ heartbeat.getHeartbeatScheduler().shutdownNow();
+ }
+
public static Boolean heartbeatExists(HoodieStorage storage, String basePath, String instantTime) throws IOException {
StoragePath heartbeatFilePath = new StoragePath(
HoodieTableMetaClient.getHeartbeatFolderPath(basePath), instantTime);
@@ -178,17 +216,18 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
public boolean isHeartbeatExpired(String instantTime) throws IOException {
Long currentTime = System.currentTimeMillis();
Heartbeat lastHeartbeatForWriter = instantToHeartbeatMap.get(instantTime);
- if (lastHeartbeatForWriter == null) {
- log.info("Heartbeat not found in internal map, falling back to reading from DFS");
- long lastHeartbeatForWriterTime = getLastHeartbeatTime(this.storage, basePath, instantTime);
- lastHeartbeatForWriter = new Heartbeat();
- lastHeartbeatForWriter.setLastHeartbeatTime(lastHeartbeatForWriterTime);
- lastHeartbeatForWriter.setInstantTime(instantTime);
- lastHeartbeatForWriter.getTimer().cancel();
+ Long lastHeartbeatTime = lastHeartbeatForWriter == null ? null : lastHeartbeatForWriter.getLastHeartbeatTime();
+ // lastHeartbeatTime can be null when the heartbeat is not in the internal map, or when it is in the
+ // map but no heartbeat has been generated yet (e.g. the first write timed out). In both cases fall
+ // back to reading the last heartbeat time from DFS (returns 0 if no heartbeat file exists, which is
+ // correctly treated as expired).
+ if (lastHeartbeatTime == null) {
+ log.info("Heartbeat time not available in internal map, falling back to reading from DFS");
+ lastHeartbeatTime = getLastHeartbeatTime(this.storage, basePath, instantTime);
}
- if (currentTime - lastHeartbeatForWriter.getLastHeartbeatTime() > this.maxAllowableHeartbeatIntervalInMs) {
+ if (currentTime - lastHeartbeatTime > this.maxAllowableHeartbeatIntervalInMs) {
log.warn("Heartbeat expired, currentTime = {}, last heartbeat = {}, heartbeat interval = {}", currentTime,
- lastHeartbeatForWriter, this.heartbeatIntervalInMs);
+ lastHeartbeatTime, this.heartbeatIntervalInMs);
return true;
}
return false;
@@ -197,20 +236,31 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
private void updateHeartbeat(String instantTime) throws HoodieHeartbeatException {
try {
Long newHeartbeatTime = System.currentTimeMillis();
- OutputStream outputStream =
- this.storage.create(
- new StoragePath(heartbeatFolderPath, instantTime), true);
- outputStream.close();
+ writeHeartbeatFile(instantTime);
Heartbeat heartbeat = instantToHeartbeatMap.get(instantTime);
if (heartbeat.getLastHeartbeatTime() != null && isHeartbeatExpired(instantTime)) {
- log.error("Aborting, missed generating heartbeat within allowable interval {} ms", this.maxAllowableHeartbeatIntervalInMs);
- // Since TimerTask allows only java.lang.Runnable, cannot throw an exception and bubble to the caller thread, hence
- // explicitly interrupting the timer thread.
- Thread.currentThread().interrupt();
+ // A previous refresh was delayed past the tolerable interval. Stop refreshing this heartbeat
+ // (cancel the scheduler) and do NOT advance the last heartbeat time, so the heartbeat stays expired
+ // and the writer aborts at commit time via HeartbeatUtils.abortIfHeartbeatExpired(). We must not
+ // keep refreshing here: a concurrent process (e.g. an async cleaner under LAZY failed-writes
+ // policy) may already have started rolling back this instant once it observed the expiry, and
+ // resurrecting the heartbeat could let this writer commit on top of rolled-back files.
+ // The scheduler is cancelled cleanly rather than via Thread.interrupt(), which would permanently
+ // kill the scheduler thread (turning a transient delay into a permanent blackout on the first miss).
+ log.error("Missed generating heartbeat for instant {} within allowable interval {} ms; stopping heartbeat refresh",
+ instantTime, this.maxAllowableHeartbeatIntervalInMs);
+ shutdownHeartbeatScheduler(heartbeat);
+ return;
}
heartbeat.setInstantTime(instantTime);
heartbeat.setLastHeartbeatTime(newHeartbeatTime);
heartbeat.setNumHeartbeats(heartbeat.getNumHeartbeats() + 1);
+ } catch (TimeoutException te) {
+ // The storage write did not complete within the bounded window. Do not advance the last heartbeat
+ // time (the write is unconfirmed); the next scheduled tick will retry on a fresh executor thread.
+ // Crucially, the scheduler thread is freed instead of being blocked by a hung storage call.
+ log.warn("Heartbeat file write for instant {} did not complete within {} ms; will retry on next tick",
+ instantTime, this.heartbeatWriteTimeoutMs);
} catch (IOException io) {
boolean isHeartbeatStopped = instantToHeartbeatMap.get(instantTime).isHeartbeatStopped();
if (isHeartbeatStopped) {
@@ -221,13 +271,49 @@ public class HoodieHeartbeatClient implements AutoCloseable, Serializable {
}
}
+ /**
+ * Writes the heartbeat file for the given instant on a dedicated daemon executor, bounded by
+ * {@link #heartbeatWriteTimeoutMs}. Performing the storage write off the scheduler thread (and with a
+ * timeout) ensures that a slow or hung storage call cannot block the scheduler thread and freeze all
+ * subsequent heartbeats for this instant.
+ */
+ private void writeHeartbeatFile(String instantTime) throws IOException, TimeoutException {
+ Future<Void> future = getHeartbeatWriteExecutor().submit(() -> {
+ try (OutputStream outputStream =
+ this.storage.create(new StoragePath(heartbeatFolderPath, instantTime), true)) {
+ // create + close confirms the heartbeat file write landed on storage.
+ }
+ return null;
+ });
+ try {
+ future.get(heartbeatWriteTimeoutMs, TimeUnit.MILLISECONDS);
+ } catch (TimeoutException te) {
+ future.cancel(true);
+ throw te;
+ } catch (InterruptedException ie) {
+ future.cancel(true);
+ Thread.currentThread().interrupt();
+ throw new HoodieHeartbeatException("Interrupted while writing heartbeat for instant " + instantTime, ie);
+ } catch (ExecutionException ee) {
+ Throwable cause = ee.getCause();
+ if (cause instanceof IOException) {
+ throw (IOException) cause;
+ }
+ throw new HoodieHeartbeatException("Failed to write heartbeat for instant " + instantTime, cause);
+ }
+ }
+
public Heartbeat getHeartbeat(String instantTime) {
return this.instantToHeartbeatMap.get(instantTime);
}
@Override
- public void close() {
+ public synchronized void close() {
this.stopHeartbeatTimers();
this.instantToHeartbeatMap.clear();
+ if (heartbeatWriteExecutor != null) {
+ heartbeatWriteExecutor.shutdownNow();
+ heartbeatWriteExecutor = null;
+ }
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index a663c63dba78..1ba577faabf1 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -680,9 +680,11 @@ public class HoodieWriteConfig extends HoodieConfig {
public static final ConfigProperty<Integer> CLIENT_HEARTBEAT_NUM_TOLERABLE_MISSES = ConfigProperty
.key("hoodie.client.heartbeat.tolerable.misses")
- .defaultValue(2)
+ .defaultValue(10)
.markAdvanced()
- .withDocumentation("Number of heartbeat misses, before a writer is deemed not alive and all pending writes are aborted.");
+ .withDocumentation("Number of heartbeat misses, before a writer is deemed not alive and all pending writes are aborted. "
+ + "A higher value tolerates transient driver pauses (e.g. GC) or storage-latency spikes that would otherwise "
+ + "delay a heartbeat and cause a still-healthy writer's commit to be aborted.");
public static final ConfigProperty<Boolean> CLUSTERING_BLOCK_FOR_PENDING_INGESTION = ConfigProperty
.key("hoodie.clustering.fail.on.pending.ingestion.during.conflict.resolution")
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
index c7ea5fa87bbd..5feb79ee83bc 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/heartbeat/TestHoodieHeartbeatClient.java
@@ -21,12 +21,18 @@ package org.apache.hudi.client.heartbeat;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
+import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.IOException;
+import java.io.OutputStream;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.awaitility.Awaitility.await;
@@ -113,4 +119,100 @@ public class TestHoodieHeartbeatClient extends HoodieCommonTestHarness {
assertFalse(hoodieHeartbeatClient.isHeartbeatExpired(instantTime1));
assertTrue(hoodieHeartbeatClient.getHeartbeat(instantTime1).isHeartbeatStopped());
}
+
+ /**
+ * Regression test for the heartbeat-expiry incident: a single slow/hung storage write must not
+ * block (freeze) the heartbeat scheduler thread. The first heartbeat write blocks (simulating a hung
+ * cloud-storage call); we assert the scheduler keeps producing heartbeats on fresh threads once that
+ * write times out, proving the scheduler thread was not blocked by the synchronous storage call (#1).
+ * A high tolerable-misses is used so that recovery after the blocked write does not itself trip the
+ * expiry path (which intentionally stops refresh on a genuine lapse).
+ */
+ @Test
+ public void testSlowHeartbeatWriteDoesNotBlockScheduler() {
+ CountDownLatch releaseFirstWrite = new CountDownLatch(1);
+ SlowCreateStorage slowStorage =
+ new SlowCreateStorage((FileSystem) metaClient.getStorage().getFileSystem(), releaseFirstWrite);
+ // interval 1s, write timeout = 1s; high tolerable-misses so the ~1s recovery gap stays well within
+ // the allowable window and the scheduler keeps beating rather than treating it as a lapse.
+ HoodieHeartbeatClient hoodieHeartbeatClient =
+ new HoodieHeartbeatClient(slowStorage, metaClient.getBasePath().toString(),
+ heartBeatInterval, 10);
+ try {
+ hoodieHeartbeatClient.start(instantTime1);
+ // Despite the first write hanging, the scheduler must keep generating heartbeats on fresh threads.
+ await().atMost(15, SECONDS)
+ .until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() >= 2);
+ } finally {
+ releaseFirstWrite.countDown();
+ hoodieHeartbeatClient.close();
+ }
+ }
+
+ @Test
+ public void testScheduledHeartbeatRetriesAfterWriteFailure() {
+ FailOnceAfterInitialCreateStorage storage =
+ new FailOnceAfterInitialCreateStorage((FileSystem) metaClient.getStorage().getFileSystem());
+ HoodieHeartbeatClient hoodieHeartbeatClient =
+ new HoodieHeartbeatClient(storage, metaClient.getBasePath().toString(), heartBeatInterval, 10);
+ try {
+ hoodieHeartbeatClient.start(instantTime1);
+ await().atMost(10, SECONDS).until(storage::hasInjectedFailure);
+ await().atMost(10, SECONDS)
+ .until(() -> hoodieHeartbeatClient.getHeartbeat(instantTime1).getNumHeartbeats() >= 2);
+ } finally {
+ hoodieHeartbeatClient.close();
+ }
+ }
+
+ /**
+ * A storage wrapper whose first {@code create()} call blocks until released, simulating a hung
+ * storage write. All subsequent calls delegate normally.
+ */
+ private static class SlowCreateStorage extends HoodieHadoopStorage {
+
+ private final AtomicBoolean firstCall = new AtomicBoolean(true);
+ private final CountDownLatch releaseFirstWrite;
+
+ SlowCreateStorage(FileSystem fs, CountDownLatch releaseFirstWrite) {
+ super(fs);
+ this.releaseFirstWrite = releaseFirstWrite;
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean overwrite) throws IOException {
+ if (firstCall.getAndSet(false)) {
+ try {
+ releaseFirstWrite.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IOException("Interrupted while simulating a hung heartbeat write", e);
+ }
+ }
+ return super.create(path, overwrite);
+ }
+ }
+
+ private static class FailOnceAfterInitialCreateStorage extends HoodieHadoopStorage {
+
+ private final AtomicInteger createCalls = new AtomicInteger(0);
+ private final AtomicBoolean injectedFailure = new AtomicBoolean(false);
+
+ FailOnceAfterInitialCreateStorage(FileSystem fs) {
+ super(fs);
+ }
+
+ @Override
+ public OutputStream create(StoragePath path, boolean overwrite) throws IOException {
+ int currentCall = createCalls.incrementAndGet();
+ if (currentCall == 2 && injectedFailure.compareAndSet(false, true)) {
+ throw new IOException("Injected scheduled heartbeat write failure");
+ }
+ return super.create(path, overwrite);
+ }
+
+ private boolean hasInjectedFailure() {
+ return injectedFailure.get();
+ }
+ }
}
(hudi) 03/05: test(flink): retry short CollectSink reads to de-flake stream-read ITs (#19030)
Posted by vh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 783a9935c80f703c1334fe3f7cd215c699d542cc
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Thu Jun 18 09:33:12 2026 +0700
test(flink): retry short CollectSink reads to de-flake stream-read ITs (#19030)
* test(flink): retry short CollectSink reads to de-flake stream-read ITs
ITTestHoodieDataSource streaming reads are collected with CollectTableSink and terminated by a forced SuccessException at the expected row count. A tolerated teardown race (Stream-is-closed / readNextRowGroup NPE, broadened in #19019) can fire before the read completes, so the job ends with fewer rows than expected and the test asserts on an incomplete result.
Wrap submit-and-collect in submitAndFetchWithRetry, which re-reads (up to 3 times) when the collected count is below the expectation; re-reading the already committed table is idempotent. Log the swallowed non-SuccessException cause for diagnosability. Test-only change, no production code touched.
* addressed review comments: fix stale comment reference and shorten terminal-cause log
---------
Co-authored-by: Vova Kolmakov <wo...@apache.org>
(cherry picked from commit c4426265e1f4f349dca53671a0826e7e68cb6513)
---
.../apache/hudi/table/ITTestHoodieDataSource.java | 72 ++++++++++++++++++++--
1 file changed, 66 insertions(+), 6 deletions(-)
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index 301450a2602f..399829f94189 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -79,6 +79,8 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -126,6 +128,15 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
*/
@ExtendWith(FlinkMiniCluster.class)
public class ITTestHoodieDataSource {
+ private static final Logger LOG = LoggerFactory.getLogger(ITTestHoodieDataSource.class);
+
+ // A streaming read collected via CollectTableSink is terminated by a forced SuccessException once it
+ // reaches its expected row count. A benign teardown race (see isAcceptableTerminalFailure) can instead
+ // close the source stream mid-read and terminate the job before all rows are emitted, leaving an
+ // incomplete result. Re-reading from the same (already committed) table is idempotent, so retry a few
+ // times before giving up. See submitAndFetchWithRetry.
+ private static final int MAX_STREAM_READ_ATTEMPTS = 3;
+
private TableEnvironment streamTableEnv;
private TableEnvironment batchTableEnv;
@@ -560,7 +571,7 @@ public class ITTestHoodieDataSource {
+ " 'connector' = '" + CollectSinkTableFactory.FACTORY_ID + "',\n"
+ " 'sink-expected-row-num' = '2'"
+ ")";
- List<Row> result = execSelectSqlWithExpectedNum(streamTableEnv, "select name, sum(age) from t1 group by name", sinkDDL);
+ List<Row> result = submitAndFetchWithRetry(streamTableEnv, "select name, sum(age) from t1 group by name", sinkDDL, 2);
final String expected = "[+I(+I[Danny, 24]), +I(+I[Stephen, 34])]";
assertRowsEquals(result, expected, true);
}
@@ -3839,15 +3850,32 @@ public class ITTestHoodieDataSource {
} else {
sinkDDL = TestConfigurations.getCollectSinkDDLWithExpectedNum("sink", expectedNum);
}
- return execSelectSqlWithExpectedNum(tEnv, select, sinkDDL);
+ return submitAndFetchWithRetry(tEnv, select, sinkDDL, expectedNum);
}
/**
- * Use CollectTableSink to collect results with expected row number.
+ * Submits a streaming select that collects into the {@link CollectSinkTableFactory} sink and returns
+ * the collected rows.
+ *
+ * <p>The streaming job is terminated by a forced {@link CollectSinkTableFactory.SuccessException} once
+ * {@code expectedNum} rows are collected. A benign teardown race (see {@link #isAcceptableTerminalFailure})
+ * can instead end the job before all rows are emitted, leaving a short result. Re-reading the already
+ * committed table is idempotent, so retry up to {@link #MAX_STREAM_READ_ATTEMPTS} times when the result
+ * is short; this keeps the race from surfacing as a confusing row-count assertion failure.
*/
- private List<Row> execSelectSqlWithExpectedNum(TableEnvironment tEnv, String select, String sinkDDL) {
- TableResult tableResult = submitSelectSql(tEnv, select, sinkDDL);
- return fetchResultWithExpectedNum(tEnv, tableResult);
+ private List<Row> submitAndFetchWithRetry(TableEnvironment tEnv, String select, String sinkDDL, int expectedNum) {
+ List<Row> rows = Collections.emptyList();
+ for (int attempt = 1; attempt <= MAX_STREAM_READ_ATTEMPTS; attempt++) {
+ TableResult tableResult = submitSelectSql(tEnv, select, sinkDDL);
+ rows = fetchResultWithExpectedNum(tEnv, tableResult);
+ if (expectedNum <= 0 || rows.size() >= expectedNum) {
+ return rows;
+ }
+ LOG.warn("Streaming read collected {} of {} expected rows on attempt {}/{}; a tolerated teardown "
+ + "race ended the job before the read completed. Retrying. select=[{}]",
+ rows.size(), expectedNum, attempt, MAX_STREAM_READ_ATTEMPTS, select);
+ }
+ return rows;
}
private TableResult submitSelectSql(TableEnvironment tEnv, String select, String sinkDDL) {
@@ -3904,6 +3932,14 @@ public class ITTestHoodieDataSource {
if (!isAcceptableTerminalFailure(e)) {
throw new AssertionError("Unexpected job failure", e);
}
+ // The races (2)/(3) usually fire after the sink has collected its expected rows, but can also fire
+ // before - ending the read with a short result. Log the tolerated cause so an incomplete read is
+ // diagnosable; submitAndFetchWithRetry re-reads when the collected count is below the expectation.
+ if (!isSuccessException(e)) {
+ LOG.warn("Streaming read terminated by a tolerated teardown race ({}); collected {} rows so far.",
+ describeTerminalCause(e),
+ CollectSinkTableFactory.RESULT.values().stream().mapToInt(List::size).sum());
+ }
}
tEnv.executeSql("DROP TABLE IF EXISTS sink");
return CollectSinkTableFactory.RESULT.values().stream()
@@ -3963,4 +3999,28 @@ public class ITTestHoodieDataSource {
}
return false;
}
+
+ /**
+ * Whether {@code e} (or any of its causes) is the normal {@link CollectSinkTableFactory.SuccessException}
+ * terminator (the happy path), as opposed to one of the tolerated teardown-race symptoms.
+ */
+ private static boolean isSuccessException(Throwable e) {
+ for (Throwable cur = e; cur != null; cur = cur.getCause()) {
+ if (cur instanceof CollectSinkTableFactory.SuccessException) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Short description of {@code e}'s root cause, for logging which tolerated terminal failure fired.
+ */
+ private static String describeTerminalCause(Throwable e) {
+ Throwable root = e;
+ while (root.getCause() != null) {
+ root = root.getCause();
+ }
+ return root.getClass().getSimpleName() + ": " + root.getMessage();
+ }
}