You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by pw...@apache.org on 2026/03/19 23:42:41 UTC
(hudi) branch master updated: feat(metadata-table): add config to disable automatic deletion of MDT partitions (#18181)
This is an automated email from the ASF dual-hosted git repository.
pwason pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new b634262f060a feat(metadata-table): add config to disable automatic deletion of MDT partitions (#18181)
b634262f060a is described below
commit b634262f060a47fc0efbe179d9bbf0840e262f53
Author: Prashant Wason <pw...@uber.com>
AuthorDate: Thu Mar 19 16:42:36 2026 -0700
feat(metadata-table): add config to disable automatic deletion of MDT partitions (#18181)
* feat(metadata-table): add config to disable automatic deletion of MDT partitions
When the write config does not have correct enable flags for MDT partitions,
they are automatically deleted. This does not work well for production datasets
with multiple writers where accidental config changes can cause large MDT to
be deleted.
Added new config `hoodie.metadata.auto.delete.partitions` (default: true) to
control this behavior. When disabled, users must explicitly drop metadata
partitions using the hudi-cli or DROP INDEX command.
Changes:
- Added AUTO_DELETE_PARTITIONS config in HoodieMetadataConfig
- Added isAutoDeleteMdtPartitionsEnabled() in HoodieWriteConfig
- Updated maybeDeleteMetadataTable() to check config before deleting
- Updated deleteMetadataIndexIfNecessary() to check config before deleting
- Updated HoodieSparkTable to check isMetadataTableAvailable() for writer init
- Replaced storage.exists() with metadataWriter.isInitialized()
* Address review comments: add Javadoc, clarify MDT behavior, add tests
- Remove redundant comment from AUTO_DELETE_PARTITIONS config
- Update Javadoc in HoodieTable.java to mention auto.delete.partitions config
- Add clarifying comment in HoodieSparkTable.java explaining that MDT
writers are created when isMetadataTableAvailable() to keep MDT in sync
even when config.isMetadataTableEnabled() is false, preventing stale data
- Add withAutoDeletePartitions() builder method in HoodieMetadataConfig
- Add tests for maybeDeleteMetadataTable() and deleteMetadataIndexIfNecessary()
to verify they are no-ops when auto-delete is disabled
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
* Fix MDT writer creation to preserve turn-off behavior when auto-delete is enabled
Only create a metadata writer for disabled MDT when auto-delete is also disabled.
When auto-delete is enabled (default), preserve the existing behavior of falling
through to maybeDeleteMetadataTable() to clean up the MDT.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
* Add e2e functional test for auto-delete partitions disabled with column stats
Adds testAutoDeletePartitionsDisabledPreservesColumnStats which validates
that when auto.delete.partitions=false and column stats is disabled in
write config, the column_stats MDT partition is not deleted and continues
to receive log file updates from subsequent commits.
Co-Authored-By: Claude Opus 4.6 <no...@anthropic.com>
---------
Co-authored-by: Claude Opus 4.6 <no...@anthropic.com>
---
.../org/apache/hudi/config/HoodieWriteConfig.java | 4 +
.../java/org/apache/hudi/table/HoodieTable.java | 10 ++-
.../org/apache/hudi/table/TestHoodieTable.java | 52 +++++++++++++
.../org/apache/hudi/table/HoodieSparkTable.java | 25 +++---
.../hudi/common/config/HoodieMetadataConfig.java | 20 +++++
.../hudi/functional/TestHoodieBackedMetadata.java | 90 ++++++++++++++++++++++
6 files changed, 187 insertions(+), 14 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 2486a398292a..d7b5f3d938a9 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2710,6 +2710,10 @@ public class HoodieWriteConfig extends HoodieConfig {
return getBoolean(HoodieMetadataConfig.ENABLE_LOG_COMPACTION_ON_METADATA_TABLE);
}
+ public boolean isAutoDeleteMdtPartitionsEnabled() {
+ return metadataConfig.isAutoDeletePartitionsEnabled();
+ }
+
public int getGlobalRecordLevelIndexMinFileGroupCount() {
return metadataConfig.getGlobalRecordLevelIndexMinFileGroupCount();
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index 91ef89f3ec56..a857ef4417d6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -1118,9 +1118,13 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
}
/**
- * Deletes the metadata table if the writer disables metadata table with hoodie.metadata.enable=false
+ * Deletes the metadata table if the writer disables metadata table with hoodie.metadata.enable=false.
+ * This operation is a no-op if hoodie.metadata.auto.delete.partitions is set to false.
*/
public void maybeDeleteMetadataTable() {
+ if (!config.isAutoDeleteMdtPartitionsEnabled()) {
+ return;
+ }
if (shouldExecuteMetadataTableDeletion()) {
try {
log.info("Deleting metadata table because it is disabled in writer.");
@@ -1133,8 +1137,12 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
/**
* Deletes the metadata partition if the writer disables any metadata index.
+ * This operation is a no-op if hoodie.metadata.auto.delete.partitions is set to false.
*/
public void deleteMetadataIndexIfNecessary() {
+ if (!config.isAutoDeleteMdtPartitionsEnabled()) {
+ return;
+ }
Stream.of(MetadataPartitionType.getValidValues()).forEach(partitionType -> {
if (shouldDeleteMetadataPartition(partitionType)) {
try {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java
index 2996377477f4..67c1c0dc57f0 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/TestHoodieTable.java
@@ -30,9 +30,11 @@ import org.apache.hudi.common.table.timeline.versioning.v1.InstantComparatorV1;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.SerializationUtils;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
+import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.table.storage.HoodieStorageLayout;
import org.junit.jupiter.api.Test;
@@ -136,4 +138,54 @@ class TestHoodieTable extends HoodieCommonTestHarness {
// Validate that function scheduleRollback is called.
assertEquals(1, ((TestBaseHoodieTable) hoodieTable).getCountOfScheduleRollbackFunctionCalls());
}
+
+ @Test
+ void testMaybeDeleteMetadataTableSkipsWhenAutoDeleteDisabled() throws IOException {
+ initMetaClient();
+ // Create metadata table directory to simulate existing MDT
+ metaClient.getStorage().createDirectory(
+ HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()));
+
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
+ .enable(false) // MDT disabled in config
+ .withAutoDeletePartitions(false) // Auto-delete disabled
+ .build();
+
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withMetadataConfig(metadataConfig)
+ .build();
+
+ HoodieEngineContext context = mock(HoodieEngineContext.class);
+ HoodieTable hoodieTable = new TestBaseHoodieTable(writeConfig, context, metaClient);
+
+ // Call maybeDeleteMetadataTable - should be a no-op when auto-delete is disabled
+ hoodieTable.maybeDeleteMetadataTable();
+
+ // Verify MDT directory still exists since auto-delete is disabled
+ boolean mdtExists = metaClient.getStorage().exists(
+ HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()));
+ assertTrue(mdtExists, "Metadata table should NOT be deleted when auto-delete is disabled");
+ }
+
+ @Test
+ void testDeleteMetadataIndexIfNecessarySkipsWhenAutoDeleteDisabled() throws IOException {
+ initMetaClient();
+ HoodieMetadataConfig metadataConfig = HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .withAutoDeletePartitions(false) // Auto-delete disabled
+ .build();
+
+ HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
+ .withPath(basePath)
+ .withMetadataConfig(metadataConfig)
+ .build();
+
+ HoodieEngineContext context = mock(HoodieEngineContext.class);
+ HoodieTable hoodieTable = new TestBaseHoodieTable(writeConfig, context, metaClient);
+
+ // This should not throw and should be a no-op since auto-delete is disabled
+ hoodieTable.deleteMetadataIndexIfNecessary();
+ // If we reach here without exception, the test passes
+ }
}
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
index fa66af392713..95e76b863659 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
@@ -28,18 +28,14 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieMetadataException;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.index.SparkHoodieIndexFactory;
-import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.SparkMetadataWriterFactory;
import org.apache.spark.TaskContext;
import org.apache.spark.TaskContext$;
-import java.io.IOException;
-
public abstract class HoodieSparkTable<T>
extends HoodieTable<T, HoodieData<HoodieRecord<T>>, HoodieData<HoodieKey>, HoodieData<WriteStatus>> {
@@ -101,7 +97,15 @@ public abstract class HoodieSparkTable<T>
if (isMetadataTable()) {
return Option.empty();
}
- if (config.isMetadataTableEnabled()) {
+ // We create a metadata writer if either:
+ // 1. isMetadataTableEnabled() - MDT is explicitly enabled in write config (normal flow).
+ // 2. isMetadataTableAvailable() AND auto-delete is disabled - MDT exists on disk (based on table config)
+ // and auto-delete is disabled (hoodie.metadata.auto.delete.partitions=false). This ensures writers
+ // keep the MDT in sync even when config.isMetadataTableEnabled() is false, preventing stale data.
+ // When auto-delete is enabled (default), we fall through to maybeDeleteMetadataTable() which
+ // handles cleanup of the MDT when metadata is disabled.
+ if (config.isMetadataTableEnabled()
+ || (getMetaClient().getTableConfig().isMetadataTableAvailable() && !config.isAutoDeleteMdtPartitionsEnabled())) {
// if any partition is deleted, we need to reload the metadata table writer so that new table configs are picked up
// to reflect the delete mdt partitions.
if (autoDetectAndDeleteMetadataPartitions) {
@@ -114,14 +118,9 @@ public abstract class HoodieSparkTable<T>
HoodieTableMetadataWriter metadataWriter = streamingWrites
? SparkMetadataWriterFactory.createWithStreamingWrites(getContext().getStorageConf(), config, failedWritesCleaningPolicy, getContext(), Option.of(triggeringInstantTimestamp))
: SparkMetadataWriterFactory.create(getContext().getStorageConf(), config, failedWritesCleaningPolicy, getContext(), Option.of(triggeringInstantTimestamp), metaClient.getTableConfig());
- try {
- if (isMetadataTableExists || metaClient.getStorage().exists(
- HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()))) {
- isMetadataTableExists = true;
- return Option.of(metadataWriter);
- }
- } catch (IOException e) {
- throw new HoodieMetadataException("Checking existence of metadata table failed", e);
+ if (isMetadataTableExists || metadataWriter.isInitialized()) {
+ isMetadataTableExists = true;
+ return Option.of(metadataWriter);
}
} else {
// if metadata is not enabled in the write config, we should try and delete it (if present)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index 0b1567c3ced3..7b5b27c1a6a5 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -511,6 +511,17 @@ public final class HoodieMetadataConfig extends HoodieConfig {
+ "The index name either starts with or matches exactly can be one of the following: "
+ StringUtils.join(Arrays.stream(MetadataPartitionType.values()).map(MetadataPartitionType::getPartitionPath).collect(Collectors.toList()), ", "));
+ public static final ConfigProperty<Boolean> AUTO_DELETE_PARTITIONS = ConfigProperty
+ .key(METADATA_PREFIX + ".auto.delete.partitions")
+ .defaultValue(true)
+ .markAdvanced()
+ .sinceVersion("1.2.0")
+ .withDocumentation("When enabled (default), metadata table partitions (indexes) that are disabled in the write config "
+ + "will be automatically deleted. When disabled, users must explicitly drop metadata partitions "
+ + "using the hudi-cli or DROP INDEX command. Disabling automatic deletion is recommended for production "
+ + "datasets with multiple writers to avoid accidental deletion of large metadata indexes due to "
+ + "misconfiguration.");
+
// Range-based repartitioning configuration for metadata table lookups
public static final ConfigProperty<Double> RANGE_REPARTITION_SAMPLING_FRACTION = ConfigProperty
.key(METADATA_PREFIX + ".range.repartition.sampling.fraction")
@@ -925,6 +936,10 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return getBooleanOrDefault(FAIL_ON_TABLE_SERVICE_FAILURES);
}
+ public boolean isAutoDeletePartitionsEnabled() {
+ return getBooleanOrDefault(AUTO_DELETE_PARTITIONS);
+ }
+
/**
* Checks if a specific metadata index is marked for dropping based on the metadata configuration.
* NOTE: Only applicable for secondary indexes (SI) or expression indexes (EI).
@@ -1247,6 +1262,11 @@ public final class HoodieMetadataConfig extends HoodieConfig {
return this;
}
+ public Builder withAutoDeletePartitions(boolean autoDeletePartitions) {
+ metadataConfig.setValue(AUTO_DELETE_PARTITIONS, String.valueOf(autoDeletePartitions));
+ return this;
+ }
+
public HoodieMetadataConfig build() {
metadataConfig.setDefaultValue(ENABLE, getDefaultMetadataEnable(engineType));
metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_COLUMN_STATS, getDefaultColStatsEnable(engineType));
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
index 30aaab80264f..a9e98f8898ac 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
@@ -417,6 +417,96 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(tableConfig.getMetadataPartitions().isEmpty());
}
+ @Test
+ public void testAutoDeletePartitionsDisabledPreservesColumnStats() throws Exception {
+ initPath();
+ HoodieWriteConfig cfg = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER)
+ .withParallelism(1, 1).withBulkInsertParallelism(1).withFinalizeWriteParallelism(1).withDeleteParallelism(1)
+ .withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder().withConsistencyCheckEnabled(true).build())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .enable(true)
+ .withMetadataIndexColumnStats(true)
+ .build())
+ .build();
+ init(COPY_ON_WRITE, Option.of(cfg), true, false, false);
+ HoodieSparkEngineContext engineContext = new HoodieSparkEngineContext(jsc);
+
+ // Step 1: Insert and upsert with column stats enabled
+ try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, cfg)) {
+ String commitTime = "0000001";
+ List<HoodieRecord> records = dataGen.generateInserts(commitTime, 20);
+ WriteClientTestUtils.startCommitWithTime(client, commitTime);
+ List<WriteStatus> statusList = client.insert(jsc.parallelize(records, 1), commitTime).collect();
+ assertTrue(client.commit(commitTime, jsc.parallelize(statusList)));
+ assertNoWriteErrors(statusList);
+
+ commitTime = "0000002";
+ WriteClientTestUtils.startCommitWithTime(client, commitTime);
+ records = dataGen.generateUniqueUpdates(commitTime, 10);
+ statusList = client.upsert(jsc.parallelize(records, 1), commitTime).collect();
+ assertTrue(client.commit(commitTime, jsc.parallelize(statusList)));
+ assertNoWriteErrors(statusList);
+ validateMetadata(client);
+ }
+
+ // Verify column_stats is present in table config
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ HoodieTableConfig tableConfig = metaClient.getTableConfig();
+ assertTrue(tableConfig.getMetadataPartitions().contains(COLUMN_STATS.getPartitionPath()));
+
+ // Step 2: Disable column stats but set auto.delete.partitions=false, then do an upsert
+ HoodieWriteConfig cfgWithAutoDeleteDisabled = HoodieWriteConfig.newBuilder()
+ .withProperties(cfg.getProps())
+ .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+ .withProperties(cfg.getMetadataConfig().getProps())
+ .withMetadataIndexColumnStats(false)
+ .withAutoDeletePartitions(false)
+ .build())
+ .build();
+
+ // Record the number of log files in column_stats partition before the upsert
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(
+ storageConf, metadataTableBasePath);
+ HoodieTableFileSystemView fsViewBefore = HoodieTableFileSystemView.fileListingBasedFileSystemView(
+ engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ List<FileSlice> colStatsSlicesBefore = fsViewBefore
+ .getLatestFileSlices(COLUMN_STATS.getPartitionPath()).collect(Collectors.toList());
+ assertFalse(colStatsSlicesBefore.isEmpty(), "Column stats partition should have file slices before upsert");
+ long logFileCountBefore = colStatsSlicesBefore.get(0).getLogFiles().count();
+
+ try (SparkRDDWriteClient client = new SparkRDDWriteClient(engineContext, cfgWithAutoDeleteDisabled)) {
+ String commitTime = "0000003";
+ WriteClientTestUtils.startCommitWithTime(client, commitTime);
+ List<HoodieRecord> records = dataGen.generateUniqueUpdates(commitTime, 10);
+ List<WriteStatus> writeStatuses = client.upsert(jsc.parallelize(records, 1), commitTime).collect();
+ assertTrue(client.commit(commitTime, jsc.parallelize(writeStatuses)));
+ assertNoWriteErrors(writeStatuses);
+ }
+
+ // Step 3: Verify column_stats partition is NOT deleted from table config
+ metaClient = HoodieTableMetaClient.reload(metaClient);
+ tableConfig = metaClient.getTableConfig();
+ assertTrue(tableConfig.getMetadataPartitions().contains(COLUMN_STATS.getPartitionPath()),
+ "Column stats partition should NOT be deleted when auto-delete is disabled");
+
+ // Step 4: Verify column_stats partition received new log files (from HoodieCommitMetadata)
+ metadataMetaClient = HoodieTableMetaClient.reload(metadataMetaClient);
+ HoodieTableFileSystemView fsViewAfter = HoodieTableFileSystemView.fileListingBasedFileSystemView(
+ engineContext, metadataMetaClient, metadataMetaClient.getActiveTimeline());
+ List<FileSlice> colStatsSlicesAfter = fsViewAfter
+ .getLatestFileSlices(COLUMN_STATS.getPartitionPath()).collect(Collectors.toList());
+ assertFalse(colStatsSlicesAfter.isEmpty(), "Column stats partition should still have file slices after upsert");
+ long logFileCountAfter = colStatsSlicesAfter.get(0).getLogFiles().count();
+ assertTrue(logFileCountAfter > logFileCountBefore,
+ "Column stats partition should have received new log files after upsert. Before: "
+ + logFileCountBefore + ", After: " + logFileCountAfter);
+
+ // Step 5: Verify column stats log files contain valid column stats records
+ List<HoodieLogFile> colStatsLogFiles = colStatsSlicesAfter.get(0)
+ .getLogFiles().collect(Collectors.toList());
+ verifyMetadataColumnStatsRecords(storage, colStatsLogFiles);
+ }
+
@Test
public void testTurnOffMetadataTableAfterEnable() throws Exception {
init(COPY_ON_WRITE, true);