You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2026/10/10 08:04:18 UTC
(hudi) branch release-1.1.2 updated (fa4f78a5b3a4 -> 38087e2aeebc)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a change to branch release-1.1.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
from fa4f78a5b3a4 fix(ci): isolate Spark merge contexts and repair Trino test infrastructure
new 5d739da79510 fix(ci): avoid Java 8 cgroup metrics crash in Hadoop daemons
new 38087e2aeebc test(metadata): backport clustering retry test fixes from master
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../hudi/client/TestJavaHoodieBackedMetadata.java | 35 +++++++++++++++++-----
.../docker_java17/docker_java17_test.sh | 5 ++++
2 files changed, 33 insertions(+), 7 deletions(-)
(hudi) 02/02: test(metadata): backport clustering retry test fixes from master
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch release-1.1.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 38087e2aeebc2ccca3ae4caf2f2919cb93728f62
Author: danny0405 <yu...@gmail.com>
AuthorDate: Sat Oct 10 16:04:08 2026 +0800
test(metadata): backport clustering retry test fixes from master
Backport the Java-client test changes from #19120 and #19434: use monotonic instant times and simulate the failed clustering commit before any reader can cache its completed state.
---
.../hudi/client/TestJavaHoodieBackedMetadata.java | 35 +++++++++++++++++-----
1 file changed, 28 insertions(+), 7 deletions(-)
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index d439189a0a63..9df7d4b8ae32 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -1818,7 +1818,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
HoodieJavaWriteClient client = getHoodieWriteClient(config);
// Write 1 (Bulk insert)
- String newCommitTime = "0000001";
+ String newCommitTime = WriteClientTestUtils.createNewInstantTime();
List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 20);
WriteClientTestUtils.startCommitWithTime(client, newCommitTime);
List<WriteStatus> writeStatuses = client.insert(records, newCommitTime);
@@ -1827,7 +1827,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
validateMetadata(client);
// Write 2 (inserts)
- newCommitTime = "0000002";
+ newCommitTime = WriteClientTestUtils.createNewInstantTime();
WriteClientTestUtils.startCommitWithTime(client, newCommitTime);
records = dataGen.generateInserts(newCommitTime, 20);
writeStatuses = client.insert(records, newCommitTime);
@@ -1843,7 +1843,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
.withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName())
.build();
- HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER)
+ HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.LAZY)
.withClusteringConfig(clusteringConfig)
.withRollbackUsingMarkers(false)
.build();
@@ -1859,17 +1859,38 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
partitionFiles.getValue().stream().forEach(file ->
replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file))));
+ // manually remove clustering completed instant from .hoodie folder to mimic clustering
+ // succeeded in metadata table, but failed before committing to data table. This must happen
+ // before any subsequent metadata-table read so that no reader ever sees the first attempt's
+ // instant as committed, matching the crash scenario this test simulates.
+ FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime);
+
+ // preconditions: clustering is pending on the data table while its deltacommit is completed
+ // in the metadata table.
+ metaClient.reloadActiveTimeline();
+ assertFalse(metaClient.getActiveTimeline().filterCompletedInstants().containsInstant(clusteringCommitTime));
+ assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime));
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(
+ metaClient.getStorageConf(), getMetadataTableBasePath(basePath));
+ assertTrue(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline()
+ .filterCompletedInstants().containsInstant(clusteringCommitTime));
+
// trigger new write to mimic other writes succeeding before re-attempt.
- newCommitTime = "0000003";
+ newCommitTime = WriteClientTestUtils.createNewInstantTime();
WriteClientTestUtils.startCommitWithTime(client, newCommitTime);
records = dataGen.generateInserts(newCommitTime, 20);
writeStatuses = client.insert(records, newCommitTime);
client.commit(newCommitTime, writeStatuses);
assertNoWriteErrors(writeStatuses);
- validateMetadata(client);
+ // ignore the first clustering attempt's uncommitted files when comparing listings.
+ validateMetadata(client, Option.of(clusteringCommitTime));
+
+ // the new write must leave the pending clustering and its metadata-table deltacommit intact.
+ metaClient.reloadActiveTimeline();
+ assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime));
+ assertTrue(metadataMetaClient.reloadActiveTimeline().getDeltaCommitTimeline()
+ .filterCompletedInstants().containsInstant(clusteringCommitTime));
- // manually remove clustering completed instant from .hoodie folder and to mimic succeeded clustering in metadata table, but failed in data table.
- FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime);
HoodieWriteMetadata<List<WriteStatus>> updatedClusterMetadata = newClient.cluster(clusteringCommitTime, true);
metaClient.reloadActiveTimeline();
(hudi) 01/02: fix(ci): avoid Java 8 cgroup metrics crash in Hadoop daemons
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch release-1.1.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5d739da79510ff2ce90708a3cc0bdcd04cf327c1
Author: danny0405 <yu...@gmail.com>
AuthorDate: Sat Oct 10 15:54:22 2026 +0800
fix(ci): avoid Java 8 cgroup metrics crash in Hadoop daemons
---
packaging/bundle-validation/docker_java17/docker_java17_test.sh | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/packaging/bundle-validation/docker_java17/docker_java17_test.sh b/packaging/bundle-validation/docker_java17/docker_java17_test.sh
index 9d14ceb321e4..4de72e5e2176 100755
--- a/packaging/bundle-validation/docker_java17/docker_java17_test.sh
+++ b/packaging/bundle-validation/docker_java17/docker_java17_test.sh
@@ -61,6 +61,11 @@ start_datanode () {
}
setup_hdfs () {
+ # The base image's Java 8 cgroup metrics can crash on newer CI hosts.
+ # Disable container detection only for Hadoop and bound each daemon's heap explicitly.
+ export HADOOP_OPTS="${HADOOP_OPTS:-} -XX:-UseContainerSupport"
+ export HADOOP_HEAPSIZE_MAX=512
+
echo "::warning::docker_test_java17.sh copying hadoop conf"
cp "$WORKDIR/tmp-conf-dir/hdfs-site.xml" "$HADOOP_HOME/etc/hadoop/hdfs-site.xml" || return 1
cp "$WORKDIR/tmp-conf-dir/core-site.xml" "$HADOOP_HOME/etc/hadoop/core-site.xml" || return 1