You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2025/10/28 17:12:35 UTC

(hudi) branch release-1.1.0 updated (eeb07995b3ac -> bfc236b0398f)

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a change to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git


    from eeb07995b3ac fix: Skip payload class validation when merge mode is not custom with v9 tables (#14116)
     new 68707164cb3b fix: Fixing error msg when table is not found with MergeIntoCommand (#14118)
     new b634cfc47ee0 fix: Handle deletes and updates properly in secondary index (#14090)
     new d13d8ae9c82e fix(core): add table level validation for decimal evolution (#14089)
     new d424808c6802 fix: Handle missing valueType column after upgrade   (#14105)
     new ae90b950f20d perf: Reduce memory usage of writing HFile log block (#14078)
     new 3468a173c263 fix: Fix cleaning of historical internal schema files (#14126)
     new 0568b0a90df8 fix: Upgrade parquet-avro version to 1.15.1 in trino bundle and plugin (#14140)
     new 3b1dc608ef61 fix: Upgrade Java xmlbuilder version to fix CVE-2014-125087 (#14144)
     new 83eac32573a8 fix: fix partition stats delete properly for downgrade from V9 to V8 (#14138)
     new 90a886a153ad fix: updating error messages thrown to end users (#14115)
     new dec9c8ed2adc fix: Fix the instant time issue for row writer bulk insert hoodie streamer (#14153)
     new 256523affc3e fix: Fixed the recovering method for the older versions where checksum is not present (#14148)
     new c2fd476a8bb2 refactor: Add required setter methods for Flink-CDC dev (#14150)
     new 63ec30032ff9 fi: Fixing secondary index read for V1 (#14149)
     new 7dcccd607393 fix: Give proper error message for multi-writer scenarios without lock provider set (#14119)
     new 9510647bfa9b feat: Partition predicate fix for Databricks runtime support (#14059)
     new dfd26b46ffb6 fix: Partition stats should be controlled using column stats config (#14165)
     new 72f4452981eb fix: Fix upgrade handling for MySqlDebeziumAvroPayload with deltastreamer (#14159)
     new bfc236b0398f fix: fix downgrade to not delete unintended partitions in MDT (#14162)

The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../apache/hudi/client/BaseHoodieWriteClient.java  |   7 +-
 .../org/apache/hudi/config/HoodieWriteConfig.java  |   5 +-
 .../org/apache/hudi/io/HoodieAppendHandle.java     |   2 +
 .../metadata/HoodieBackedTableMetadataWriter.java  |  37 +++--
 .../hudi/metadata/MetadataIndexGenerator.java      | 132 ------------------
 .../apache/hudi/metadata/MetadataIndexMapper.java  |  66 +++++++++
 .../apache/hudi/metadata/RecordIndexMapper.java    |  84 ++++++++++++
 .../apache/hudi/metadata/SecondaryIndexMapper.java |  68 ++++++++++
 .../SecondaryIndexRecordGenerationUtils.java       |  13 +-
 .../java/org/apache/hudi/table/HoodieTable.java    |  12 +-
 .../table/upgrade/EightToNineUpgradeHandler.java   |  44 +++---
 .../hudi/table/upgrade/UpgradeDowngrade.java       |   2 +-
 .../hudi/table/upgrade/UpgradeDowngradeUtils.java  |  14 +-
 ...Generator.java => TestMetadataIndexMapper.java} |  13 +-
 .../upgrade/TestEightToNineUpgradeHandler.java     | 107 ++++++---------
 .../upgrade/TestNineToEightDowngradeHandler.java   | 120 +++++++++++++++++
 .../FlinkHoodieBackedTableMetadataWriter.java      |   5 -
 .../org/apache/hudi/table/HoodieFlinkTable.java    |   3 +-
 .../JavaHoodieBackedTableMetadataWriter.java       |   5 -
 .../org/apache/hudi/table/HoodieJavaTable.java     |   9 +-
 .../hudi/client/StreamingMetadataWriteHandler.java |   2 +-
 .../apache/hudi/index/HoodieSparkIndexClient.java  |  25 +---
 .../SparkHoodieBackedTableMetadataWriter.java      |   4 -
 ...ieBackedTableMetadataWriterTableVersionSix.java |   5 -
 .../org/apache/hudi/table/HoodieSparkTable.java    |   7 +-
 .../spark/sql/HoodieCatalystPlansUtils.scala       |   5 +
 .../table/functional/TestCleanPlanExecutor.java    |   3 +-
 .../apache/hudi/avro/AvroSchemaCompatibility.java  |  40 +++++-
 .../hudi/common/config/HoodieMetadataConfig.java   |  30 ++---
 .../hudi/common/model/HoodieAvroIndexedRecord.java |   2 +-
 .../hudi/common/model/HoodieRecordPayload.java     |  21 ++-
 .../model/debezium/MySqlDebeziumAvroPayload.java   |   5 +
 .../hudi/common/table/HoodieTableConfig.java       |   7 +
 .../hudi/common/table/HoodieTableVersion.java      |   9 +-
 .../common/table/log/block/HoodieDataBlock.java    |   2 +-
 .../org/apache/hudi/common/util/CommitUtils.java   |   2 +-
 .../org/apache/hudi/common/util/ConfigUtils.java   |  15 ++-
 .../org/apache/hudi/common/util/HFileUtils.java    |  92 ++++++-------
 .../io/FileBasedInternalSchemaStorageManager.java  |   4 +-
 .../hudi/metadata/HoodieBackedTableMetadata.java   |   8 +-
 .../hudi/metadata/HoodieTableMetadataUtil.java     |  40 ++++--
 .../java/org/apache/hudi/stats/ValueMetadata.java  |   5 +
 .../org/apache/hudi/avro/TestAvroSchemaUtils.java  |  70 ++++++++++
 .../apache/hudi/common/util/TestConfigUtils.java   |  27 ++++
 .../hudi/sink/StreamWriteOperatorCoordinator.java  |   6 +-
 .../sink/common/AbstractStreamWriteFunction.java   |  10 +-
 .../org/apache/hudi/sink/event/Correspondent.java  |   8 ++
 .../org/apache/hudi/sink/TestWriteCopyOnWrite.java |   2 -
 .../hudi/sink/TestWriteMergeOnReadWithCompact.java |   3 -
 .../java/org/apache/hudi/source/TestFileIndex.java |   1 -
 .../hudi/source/TestIncrementalInputSplits.java    |   1 -
 .../apache/hudi/table/ITTestHoodieDataSource.java  |   2 -
 .../apache/hudi/table/ITTestSchemaEvolution.java   |   3 +-
 .../apache/hudi/table/TestHoodieTableSource.java   |   1 -
 .../hudi/io/hadoop/HoodieBaseParquetWriter.java    |   4 +
 .../hudi/common/table/TestHoodieTableConfig.java   |  13 +-
 .../apache/hudi/common/util/TestCommitUtils.java   |  35 +++++
 .../TestFileBasedInternalSchemaStorageManager.java |   2 +
 .../hudi/metadata/TestHoodieTableMetadataUtil.java |   2 -
 .../main/java/org/apache/hudi/DataSourceUtils.java |  15 ++-
 .../BaseDatasetBulkInsertCommitActionExecutor.java |   4 +-
 .../DatasetBucketRescaleCommitActionExecutor.java  |   4 +-
 .../DatasetBulkInsertCommitActionExecutor.java     |   4 +-
 ...setBulkInsertOverwriteCommitActionExecutor.java |   4 +-
 ...lkInsertOverwriteTableCommitActionExecutor.java |   4 +-
 ...eamerDatasetBulkInsertCommitActionExecutor.java |   4 +-
 .../org/apache/hudi/HoodieSparkSqlWriter.scala     |  12 +-
 .../apache/hudi/SparkHoodieTableFileIndex.scala    |  20 ++-
 .../hudi/analysis/HoodieSparkBaseAnalysis.scala    |  10 +-
 .../TestDataSkippingWithMORColstats.java           |   3 -
 .../hudi/functional/TestHoodieBackedMetadata.java  |   2 -
 .../apache/hudi/io/TestMetadataWriterCommit.java   |   6 +-
 .../hudi/table/upgrade/TestUpgradeDowngrade.java   | 108 ++++++++++++---
 .../cow-tables/hudi-v8-table-cow.zip               | Bin 0 -> 380008 bytes
 .../generate-fixtures.sh                           |   3 +
 ...ixture-mor.scala => generate-fixture-cow.scala} |  28 ++--
 .../org/apache/hudi/TestHoodieSparkSqlWriter.scala |   2 +-
 .../functional/PartitionStatsIndexTestBase.scala   |   3 +-
 .../hudi/functional/TestBasicSchemaEvolution.scala |   3 +-
 .../hudi/functional/TestPartitionStatsIndex.scala  |  22 ---
 .../TestPartitionStatsIndexWithSql.scala           |   6 +-
 .../functional/TestPartitionStatsPruning.scala     |  19 +--
 .../functional/TestRecordLevelIndexWithSQL.scala   |   2 +
 .../functional/TestSecondaryIndexDataTypes.scala   |   5 +
 .../functional/TestSecondaryIndexPruning.scala     | 150 ++++++++++++++++++++-
 .../sql/hudi/dml/others/TestMergeIntoTable.scala   |  33 +++++
 .../hudi/feature/index/TestExpressionIndex.scala   |   5 +-
 .../TestHoodieBackedTableMetadataIndexLookup.scala |   1 +
 .../sql/hudi/feature/index/TestIndexSyntax.scala   |   4 +
 .../hudi/feature/index/TestSecondaryIndex.scala    |   4 +
 .../spark/sql/HoodieSpark33CatalystPlanUtils.scala |   4 +
 .../spark/sql/HoodieSpark34CatalystPlanUtils.scala |   6 +
 .../spark/sql/HoodieSpark35CatalystPlanUtils.scala |   6 +
 .../spark/sql/HoodieSpark40CatalystPlanUtils.scala |   6 +
 .../hudi/hive/util/IMetaStoreClientUtil.java       |  23 +++-
 .../apache/hudi/sync/common/HoodieSyncClient.java  |  19 ++-
 hudi-trino-plugin/pom.xml                          |   2 +
 .../apache/hudi/utilities/sources/InputBatch.java  |   7 +-
 .../hudi/utilities/streamer/HoodieStreamer.java    |   1 -
 .../utilities/streamer/HoodieStreamerUtils.java    |   8 +-
 .../apache/hudi/utilities/streamer/StreamSync.java |  30 ++++-
 .../TestHoodieMetadataTableValidator.java          |   5 +-
 .../deltastreamer/HoodieDeltaStreamerTestBase.java |  26 +++-
 .../deltastreamer/TestHoodieDeltaStreamer.java     |  54 ++++++--
 ...TestHoodieDeltaStreamerSchemaEvolutionBase.java |   2 +-
 .../hudi/utilities/sources/TestInputBatch.java     |   6 +-
 .../streamer/TestHoodieStreamerUtils.java          |   5 +-
 packaging/hudi-trino-bundle/pom.xml                |   2 +-
 pom.xml                                            |   6 +
 109 files changed, 1397 insertions(+), 582 deletions(-)
 delete mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexGenerator.java
 create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexMapper.java
 create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java
 create mode 100644 hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexMapper.java
 rename hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/{TestMetadataIndexGenerator.java => TestMetadataIndexMapper.java} (85%)
 create mode 100644 hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/cow-tables/hudi-v8-table-cow.zip
 copy hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/scala-templates/{generate-fixture-mor.scala => generate-fixture-cow.scala} (89%)


(hudi) 19/19: fix: fix downgrade to not delete unintended partitions in MDT (#14162)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit bfc236b0398fc6f4bb732cc23785c9a988ad4dfe
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Tue Oct 28 10:28:51 2025 +0530

    fix: fix downgrade to not delete unintended partitions in MDT (#14162)
    
    
    ---------
    
    Co-authored-by: Vamsi <va...@onehouse.ai>
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../apache/hudi/client/BaseHoodieWriteClient.java  |  2 +-
 .../java/org/apache/hudi/table/HoodieTable.java    | 12 +--
 .../hudi/table/upgrade/UpgradeDowngrade.java       |  2 +-
 .../org/apache/hudi/table/HoodieFlinkTable.java    |  3 +-
 .../org/apache/hudi/table/HoodieJavaTable.java     |  9 ++-
 .../hudi/client/StreamingMetadataWriteHandler.java |  2 +-
 .../org/apache/hudi/table/HoodieSparkTable.java    |  7 +-
 .../hudi/table/upgrade/TestUpgradeDowngrade.java   | 91 ++++++++++++++++++++--
 8 files changed, 106 insertions(+), 22 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index e5880bd29a54..cc19970fb60c 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -1119,7 +1119,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
         });
       });
 
-      Option<HoodieTableMetadataWriter> metadataWriterOpt = table.getMetadataWriter(dropInstant);
+      Option<HoodieTableMetadataWriter> metadataWriterOpt = table.getMetadataWriter(dropInstant, false, false);
       // first update table config. Metadata writer initializes the inflight metadata
       // partitions so we need to first remove the metadata before creating the writer
       // Also the partitions need to be removed after creating the metadata writer since the writer
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
index 1438b7e59712..4ae51ac75298 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/HoodieTable.java
@@ -1085,7 +1085,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
    * @return instance of {@link HoodieTableMetadataWriter}
    */
   public final Option<HoodieTableMetadataWriter> getMetadataWriter(String triggeringInstantTimestamp) {
-    return getMetadataWriter(triggeringInstantTimestamp, false);
+    return getMetadataWriter(triggeringInstantTimestamp, false, true);
   }
 
   /**
@@ -1094,8 +1094,8 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
    * @param triggeringInstantTimestamp - The instant that is triggering this metadata write
    * @return instance of {@link HoodieTableMetadataWriter}
    */
-  public final Option<HoodieTableMetadataWriter> getMetadataWriter(String triggeringInstantTimestamp, boolean streamingWrites) {
-    return getMetadataWriter(triggeringInstantTimestamp, EAGER, streamingWrites);
+  public final Option<HoodieTableMetadataWriter> getMetadataWriter(String triggeringInstantTimestamp, boolean streamingWrites, boolean autoDetectAndDeleteMetadataPartitions) {
+    return getMetadataWriter(triggeringInstantTimestamp, EAGER, streamingWrites, autoDetectAndDeleteMetadataPartitions);
   }
 
   /**
@@ -1105,7 +1105,7 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
    * @return An instance of {@link HoodieTableMetadataWriter}.
    */
   public Option<HoodieTableMetadataWriter> getIndexingMetadataWriter(String triggeringInstantTimestamp) {
-    return getMetadataWriter(triggeringInstantTimestamp, LAZY, false);
+    return getMetadataWriter(triggeringInstantTimestamp, LAZY, false, false);
   }
 
   /**
@@ -1121,12 +1121,14 @@ public abstract class HoodieTable<T, I, K, O> implements Serializable {
    * @param triggeringInstantTimestamp The instant that is triggering this metadata write
    * @param failedWritesCleaningPolicy Cleaning policy on failed writes
    * @param streamingWrites            Whether streaming write is enabled
+   * @param autoDetectAndDeleteMetadataPartitions true when metadata partitions could be deleted based on incoming write config properties.
    * @return instance of {@link HoodieTableMetadataWriter}
    */
   protected Option<HoodieTableMetadataWriter> getMetadataWriter(
       String triggeringInstantTimestamp,
       HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy,
-      boolean streamingWrites) {
+      boolean streamingWrites,
+      boolean autoDetectAndDeleteMetadataPartitions) {
     // Each engine is expected to override this and
     // provide the actual metadata writer, if enabled.
     return Option.empty();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
index bc643313b99a..581452afa747 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
@@ -269,7 +269,7 @@ public class UpgradeDowngrade {
 
       HoodieTable table = upgradeDowngradeHelper.getTable(updatedConfig, context);
       String newInstant = table.getMetaClient().createNewInstantTime(false);
-      Option<HoodieTableMetadataWriter> mdtWriterOpt = table.getMetadataWriter(newInstant);
+      Option<HoodieTableMetadataWriter> mdtWriterOpt = table.getMetadataWriter(newInstant, false, false);
       mdtWriterOpt.ifPresent(mdtWriter -> {
         HoodieCommitMetadata commitMetadata = new HoodieCommitMetadata();
         commitMetadata.setOperationType(WriteOperationType.UPSERT);
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkTable.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkTable.java
index 403526d23aba..910a66a2c3f0 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkTable.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/table/HoodieFlinkTable.java
@@ -100,7 +100,8 @@ public abstract class HoodieFlinkTable<T>
   protected Option<HoodieTableMetadataWriter> getMetadataWriter(
       String triggeringInstantTimestamp,
       HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy,
-      boolean streamingWrites) {
+      boolean streamingWrites,
+      boolean autoDetectAndDeleteMetadataPartitions) {
     if (isMetadataTable()) {
       return Option.empty();
     }
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
index 77200c69cf5e..32d6e7d281ec 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/table/HoodieJavaTable.java
@@ -81,7 +81,8 @@ public abstract class HoodieJavaTable<T>
   @Override
   protected Option<HoodieTableMetadataWriter> getMetadataWriter(String triggeringInstantTimestamp,
                                                                 HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy,
-                                                                boolean streamingWrites) {
+                                                                boolean streamingWrites,
+                                                                boolean autoDetectAndDeleteMetadataPartitions) {
     if (isMetadataTable()) {
       return Option.empty();
     }
@@ -93,8 +94,10 @@ public abstract class HoodieJavaTable<T>
           getContext().getStorageConf(), config, failedWritesCleaningPolicy, getContext(),
           Option.of(triggeringInstantTimestamp));
       // even with metadata enabled, some index could have been disabled
-      // delete metadata partitions corresponding to such indexes
-      deleteMetadataIndexIfNecessary();
+      // delete metadata partitions corresponding to such indexes if autoDetectAndDeleteMdtPartitions is enabled
+      if (autoDetectAndDeleteMetadataPartitions) {
+        deleteMetadataIndexIfNecessary();
+      }
       try {
         if (isMetadataTableExists || metaClient.getStorage().exists(
             HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()))) {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/StreamingMetadataWriteHandler.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/StreamingMetadataWriteHandler.java
index 93cf377cb6be..470f10013ccb 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/StreamingMetadataWriteHandler.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/StreamingMetadataWriteHandler.java
@@ -137,7 +137,7 @@ public class StreamingMetadataWriteHandler {
       return this.metadataWriterMap.get(triggeringInstant);
     }
 
-    Option<HoodieTableMetadataWriter> metadataWriterOpt = table.getMetadataWriter(triggeringInstant, true);
+    Option<HoodieTableMetadataWriter> metadataWriterOpt = table.getMetadataWriter(triggeringInstant, true, true);
     metadataWriterMap.put(triggeringInstant, metadataWriterOpt); // populate this for every new instant time.
     // if metadata table does not exist, the map will contain an entry, with value Option.empty.
     // if not, it will contain the metadata writer instance.
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
index a9276fd37902..fa66af392713 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkTable.java
@@ -96,14 +96,17 @@ public abstract class HoodieSparkTable<T>
   protected Option<HoodieTableMetadataWriter> getMetadataWriter(
       String triggeringInstantTimestamp,
       HoodieFailedWritesCleaningPolicy failedWritesCleaningPolicy,
-      boolean streamingWrites) {
+      boolean streamingWrites,
+      boolean autoDetectAndDeleteMetadataPartitions) {
     if (isMetadataTable()) {
       return Option.empty();
     }
     if (config.isMetadataTableEnabled()) {
       // if any partition is deleted, we need to reload the metadata table writer so that new table configs are picked up
       // to reflect the delete mdt partitions.
-      deleteMetadataIndexIfNecessary();
+      if (autoDetectAndDeleteMetadataPartitions) {
+        deleteMetadataIndexIfNecessary();
+      }
 
       // Create the metadata table writer. First time after the upgrade this creation might trigger
       // metadata table bootstrapping. Bootstrapping process could fail and checking the table
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 5ae4fc222a1c..032eda704eb7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -18,8 +18,13 @@
 
 package org.apache.hudi.table.upgrade;
 
+import org.apache.hudi.client.SparkRDDWriteClient;
+import org.apache.hudi.client.WriteClientTestUtils;
+import org.apache.hudi.common.config.HoodieMetadataConfig;
 import org.apache.hudi.common.config.RecordMergeMode;
 import org.apache.hudi.common.model.HoodieIndexMetadata;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.HoodieTableVersion;
@@ -27,15 +32,18 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.timeline.HoodieTimeline;
 import org.apache.hudi.common.table.timeline.InstantFileNameGenerator;
 import org.apache.hudi.common.table.timeline.versioning.TimelineLayoutVersion;
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
 import org.apache.hudi.common.testutils.HoodieTestUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieUpgradeDowngradeException;
 import org.apache.hudi.keygen.constant.KeyGeneratorType;
 import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.MetadataPartitionType;
 import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
 
+import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Row;
 import org.junit.jupiter.api.Disabled;
@@ -49,6 +57,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
+import java.net.URI;
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
@@ -57,6 +66,7 @@ import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.getCommitTimeAtUTC;
 import static org.apache.hudi.keygen.KeyGenUtils.getComplexKeygenErrorMessage;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -98,31 +108,31 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
     boolean isUpgrade = fromVersion.lesserThan(toVersion);
     String operation = isUpgrade ? "upgrade" : "downgrade";
     LOG.info("Testing {} from version {} to {}", operation, fromVersion, toVersion);
-    
+
     HoodieTableMetaClient originalMetaClient = loadFixtureTable(fromVersion, suffix);
     assertEquals(fromVersion, originalMetaClient.getTableConfig().getTableVersion(),
         "Fixture table should be at expected version");
-    
+
     HoodieWriteConfig config = createWriteConfig(originalMetaClient, true);
-    
+
     int initialPendingCommits = originalMetaClient.getCommitsTimeline().filterPendingExcludingCompaction().countInstants();
     int initialCompletedCommits = originalMetaClient.getCommitsTimeline().filterCompletedInstants().countInstants();
-    
+
     Dataset<Row> originalData = readTableData(originalMetaClient, "before " + operation);
-    
+
     // Confirm that there are log files before rollback and compaction operations
     if (isRollbackAndCompactTransition(fromVersion, toVersion)) {
       validateLogFilesCount(originalMetaClient, operation, suffix.equals("-mor"));
     }
-    
+
     new UpgradeDowngrade(originalMetaClient, config, context(), SparkUpgradeDowngradeHelper.getInstance())
         .run(toVersion, null);
-    
+
     HoodieTableMetaClient resultMetaClient = HoodieTableMetaClient.builder()
         .setConf(storageConf().newInstance())
         .setBasePath(originalMetaClient.getBasePath())
         .build();
-    
+
     assertTableVersionOnDataAndMetadataTable(resultMetaClient, toVersion);
     validateVersionSpecificProperties(resultMetaClient, toVersion);
     validateDataConsistency(originalData, resultMetaClient, "after " + operation);
@@ -376,6 +386,62 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
     }
   }
 
+  @ParameterizedTest
+  @MethodSource("testMdtValidationDowngrade")
+  public void testMdtPartitionNotDroppedWhenDowngradedFromTableVersionNine(HoodieTableType tableType, boolean mdtEnabled) throws Exception {
+    HoodieTableVersion fromVersion = HoodieTableVersion.NINE;
+    HoodieTableVersion toVersion = HoodieTableVersion.EIGHT;
+
+    Properties props = new Properties();
+    props.put(HoodieTableConfig.TYPE.key(), tableType.name());
+    HoodieTableMetaClient metaClient =
+        getHoodieMetaClient(storageConf(), URI.create(basePath()).getPath(), props);
+
+    HoodieWriteConfig writeConfig = getConfigBuilder(true)
+        .withPath(metaClient.getBasePath())
+        .withWriteTableVersion(fromVersion.versionCode())
+        .withMetadataConfig(HoodieMetadataConfig.newBuilder()
+                .withEnableRecordIndex(true).build())
+        .withProps(props)
+        .build();
+
+    SparkRDDWriteClient writeClient = new SparkRDDWriteClient(context(), writeConfig);
+    String partitionPath = "2021/09/11";
+    HoodieTestDataGenerator dataGenerator = new HoodieTestDataGenerator(new String[]{partitionPath});
+
+    String instant1 = getCommitTimeAtUTC(1);
+    List<HoodieRecord> records = dataGenerator.generateInserts(instant1, 100);
+    JavaRDD<HoodieRecord> dataset = jsc().parallelize(records, 2);
+
+    WriteClientTestUtils.startCommitWithTime(writeClient, instant1);
+    writeClient.commit(instant1, writeClient.insert(dataset, instant1));
+    metaClient.reloadTableConfig();
+
+    // verify record index partition exists before downgrade
+    assertTrue(metaClient.getTableConfig().getMetadataPartitions().contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath()));
+
+    HoodieWriteConfig.Builder upgradeWriteConfig = HoodieWriteConfig.newBuilder()
+        .withPath(metaClient.getBasePath())
+        .withProps(props);
+    if (mdtEnabled) {
+      upgradeWriteConfig.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(true).withEnableRecordIndex(false).build());
+    } else {
+      upgradeWriteConfig.withMetadataConfig(HoodieMetadataConfig.newBuilder().enable(false).build());
+    }
+
+    new UpgradeDowngrade(metaClient, upgradeWriteConfig.build(), context(), SparkUpgradeDowngradeHelper.getInstance())
+        .run(toVersion, null);
+
+    HoodieTableMetaClient resultMetaClient = HoodieTableMetaClient.builder()
+        .setConf(storageConf().newInstance())
+        .setBasePath(metaClient.getBasePath())
+        .build();
+
+    resultMetaClient.reloadTableConfig();
+    // verify record index partition exists after downgrade
+    assertTrue(resultMetaClient.getTableConfig().getMetadataPartitions().contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath()));
+  }
+
   /**
    * Load a fixture table from resources and copy it to a temporary location for testing.
    */
@@ -536,6 +602,15 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
     );
   }
 
+  private static Stream<Arguments> testMdtValidationDowngrade() {
+    return Stream.of(
+        Arguments.of(HoodieTableType.COPY_ON_WRITE, true),
+        Arguments.of(HoodieTableType.COPY_ON_WRITE, false),
+        Arguments.of(HoodieTableType.MERGE_ON_READ, true),
+        Arguments.of(HoodieTableType.MERGE_ON_READ, false)
+    );
+  }
+
   private static Stream<Arguments> testArgsPayloadUpgradeDowngrade() {
     String[] payloadTypes = {
         "default", "overwrite", "partial", "postgres", "mysql",


(hudi) 02/19: fix: Handle deletes and updates properly in secondary index (#14090)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit b634cfc47ee0dadf5da8ae9bca42a739daccca39
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed Oct 22 23:00:38 2025 +0800

    fix: Handle deletes and updates properly in secondary index (#14090)
---
 .../metadata/HoodieBackedTableMetadataWriter.java  |  34 +++--
 .../hudi/metadata/MetadataIndexGenerator.java      | 132 -------------------
 .../apache/hudi/metadata/MetadataIndexMapper.java  |  66 ++++++++++
 .../apache/hudi/metadata/RecordIndexMapper.java    |  84 ++++++++++++
 .../apache/hudi/metadata/SecondaryIndexMapper.java |  68 ++++++++++
 .../SecondaryIndexRecordGenerationUtils.java       |  13 +-
 ...Generator.java => TestMetadataIndexMapper.java} |  13 +-
 .../FlinkHoodieBackedTableMetadataWriter.java      |   5 -
 .../JavaHoodieBackedTableMetadataWriter.java       |   5 -
 .../SparkHoodieBackedTableMetadataWriter.java      |   4 -
 ...ieBackedTableMetadataWriterTableVersionSix.java |   5 -
 .../hudi/metadata/HoodieTableMetadataUtil.java     |  40 ++++--
 .../functional/TestSecondaryIndexPruning.scala     | 142 ++++++++++++++++++++-
 13 files changed, 424 insertions(+), 187 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 5d172095affd..51d5a9016081 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -180,7 +180,6 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
   boolean initialized = false;
   private HoodieTableFileSystemView metadataView;
   private final boolean streamingWritesEnabled;
-  private final Option<MetadataIndexGenerator> metadataIndexGenerator;
 
   protected HoodieBackedTableMetadataWriter(StorageConfiguration<?> storageConf,
                                             HoodieWriteConfig writeConfig,
@@ -225,18 +224,12 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
     }
     ValidationUtils.checkArgument(!initialized || this.metadata != null, "MDT Reader should have been opened post initialization");
     this.streamingWritesEnabled = streamingWritesEnabled;
-    this.metadataIndexGenerator = streamingWritesEnabled ? Option.of(initializeMetadataIndexGenerator()) : Option.empty();
   }
 
   List<MetadataPartitionType> getEnabledPartitions(HoodieMetadataConfig metadataConfig, HoodieTableMetaClient metaClient) {
     return MetadataPartitionType.getEnabledPartitions(metadataConfig, metaClient);
   }
 
-  /**
-   * Returns the utilities for metadata index generation.
-   */
-  abstract MetadataIndexGenerator initializeMetadataIndexGenerator();
-
   private void mayBeReinitMetadataReader() {
     if (metadata == null || metadataMetaClient == null || metadata.getMetadataFileSystemView() == null) {
       initMetadataReader();
@@ -1239,12 +1232,31 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
     }
 
     maybeInitializeNewFileGroupsForPartitionedRLI(writeStatus, instantTime);
-    HoodieData<HoodieRecord> untaggedRecords = writeStatus.flatMap(
-        new MetadataIndexGenerator.WriteStatusBasedMetadataIndexMapper(mdtPartitionsToTag, dataWriteConfig));
+
+    Map<MetadataPartitionType, MetadataIndexMapper> indexMapperMap = mdtPartitionsToTag.stream()
+        .filter(e -> e.equals(RECORD_INDEX) || e.equals(SECONDARY_INDEX))
+        .collect(Collectors.toMap(
+            key -> key,
+            key -> {
+              if (RECORD_INDEX.equals(key)) {
+                return new RecordIndexMapper(dataWriteConfig);
+              }
+              return new SecondaryIndexMapper(dataWriteConfig);
+            }
+        ));
+
+    if (indexMapperMap.isEmpty()) {
+      return engineContext.emptyHoodieData();
+    }
+
+    HoodieData<HoodieRecord> processedRecords = indexMapperMap.values().stream()
+        .map(indexMapper -> indexMapper.postProcess(writeStatus.flatMap(indexMapper)))
+        .reduce(HoodieData::union)
+        .get();
 
     // tag records w/ location
-    Pair<List<HoodieFileGroupId>, HoodieData<HoodieRecord>> hoodieFileGroupsToUpdateAndTaggedMdtRecords = tagRecordsWithLocationForStreamingWrites(untaggedRecords,
-        mdtPartitionPathsToTag);
+    Pair<List<HoodieFileGroupId>, HoodieData<HoodieRecord>> hoodieFileGroupsToUpdateAndTaggedMdtRecords =
+        tagRecordsWithLocationForStreamingWrites(processedRecords, mdtPartitionPathsToTag);
 
     // write partial writes to MDT table (for those partitions where streaming writes are enabled)
     HoodieData<WriteStatus> writeStatusCollection = convertEngineSpecificDataToHoodieData(streamWriteToMetadataTable(hoodieFileGroupsToUpdateAndTaggedMdtRecords, instantTime));
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexGenerator.java
deleted file mode 100644
index 87fbbec05a45..000000000000
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexGenerator.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.metadata;
-
-import org.apache.hudi.client.SecondaryIndexStats;
-import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.common.function.SerializableFunction;
-import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordDelegate;
-import org.apache.hudi.common.model.HoodieRecordLocation;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieMetadataException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hudi.metadata.MetadataPartitionType.RECORD_INDEX;
-import static org.apache.hudi.metadata.MetadataPartitionType.SECONDARY_INDEX;
-
-/**
- * For now this is a placeholder to generate all MDT records in one place.
- * Once <a href="https://github.com/apache/hudi/pull/13226">Refactor MDT update logic with Indexer</a> is landed,
- * we will leverage the new abstraction to generate MDT records.
- */
-public class MetadataIndexGenerator implements Serializable {
-
-  private static final Logger LOG = LoggerFactory.getLogger(MetadataIndexGenerator.class);
-
-  /**
-   * MDT record transformation utility. This function is expected to be invoked from a map Partition call, where one spark task will receive
-   * one WriteStatus as input and the output contains prepared MDT table records for all eligible partitions that can operate on one
-   * WriteStatus instance only.
-   */
-  static class WriteStatusBasedMetadataIndexMapper implements SerializableFunction<WriteStatus, Iterator<HoodieRecord>> {
-    private final List<MetadataPartitionType> enabledPartitionTypes;
-    private final HoodieWriteConfig dataWriteConfig;
-
-    public WriteStatusBasedMetadataIndexMapper(List<MetadataPartitionType> enabledPartitionTypes, HoodieWriteConfig dataWriteConfig) {
-      this.enabledPartitionTypes = enabledPartitionTypes;
-      this.dataWriteConfig = dataWriteConfig;
-    }
-
-    @Override
-    public Iterator<HoodieRecord> apply(WriteStatus writeStatus) throws Exception {
-      List<HoodieRecord> allRecords = new ArrayList<>();
-      if (enabledPartitionTypes.contains(RECORD_INDEX)) {
-        allRecords.addAll(processWriteStatusForRLI(writeStatus, dataWriteConfig));
-      }
-      if (enabledPartitionTypes.contains(SECONDARY_INDEX)) {
-        allRecords.addAll(processWriteStatusForSecondaryIndex(writeStatus));
-      }
-      // yet to add support for more partitions.
-      // bloom filter
-      // secondary index
-      // expression index.
-      return allRecords.iterator();
-    }
-  }
-
-  protected static List<HoodieRecord> processWriteStatusForRLI(WriteStatus writeStatus, HoodieWriteConfig dataWriteConfig) {
-    List<HoodieRecord> allRecords = new ArrayList<>();
-    for (HoodieRecordDelegate recordDelegate : writeStatus.getIndexStats().getWrittenRecordDelegates()) {
-      if (!writeStatus.isErrored(recordDelegate.getHoodieKey())) {
-        if (recordDelegate.getIgnoreIndexUpdate()) {
-          continue;
-        }
-        HoodieRecord hoodieRecord;
-        Option<HoodieRecordLocation> newLocation = recordDelegate.getNewLocation();
-        if (newLocation.isPresent()) {
-          if (recordDelegate.getCurrentLocation().isPresent()) {
-            // This is an update, no need to update index if the location has not changed
-            // newLocation should have the same fileID as currentLocation. The instantTimes differ as newLocation's
-            // instantTime refers to the current commit which was completed.
-            if (!recordDelegate.getCurrentLocation().get().getFileId().equals(newLocation.get().getFileId())) {
-              final String msg = String.format("Detected update in location of record with key %s from %s to %s. The fileID should not change.",
-                  recordDelegate, recordDelegate.getCurrentLocation().get(), newLocation.get());
-              LOG.error(msg);
-              throw new HoodieMetadataException(msg);
-            }
-            // for updates, we can skip updating RLI partition in MDT
-          } else {
-            // Insert new record case
-            hoodieRecord = HoodieMetadataPayload.createRecordIndexUpdate(
-                recordDelegate.getRecordKey(), recordDelegate.getPartitionPath(),
-                newLocation.get().getFileId(), newLocation.get().getInstantTime(), dataWriteConfig.getWritesFileIdEncoding());
-            allRecords.add(hoodieRecord);
-          }
-        } else {
-          // Delete existing index for a deleted record
-          hoodieRecord = HoodieMetadataPayload.createRecordIndexDelete(recordDelegate.getRecordKey(), recordDelegate.getPartitionPath(), dataWriteConfig.isPartitionedRecordIndexEnabled());
-          allRecords.add(hoodieRecord);
-        }
-      }
-    }
-    return allRecords;
-  }
-
-  protected static List<HoodieRecord> processWriteStatusForSecondaryIndex(WriteStatus writeStatus) {
-    List<HoodieRecord> secondaryIndexRecords = new ArrayList<>(writeStatus.getIndexStats().getSecondaryIndexStats().size());
-    for (Map.Entry<String, List<SecondaryIndexStats>> entry : writeStatus.getIndexStats().getSecondaryIndexStats().entrySet()) {
-      String indexPartitionName = entry.getKey();
-      List<SecondaryIndexStats> secondaryIndexStats = entry.getValue();
-      for (SecondaryIndexStats stat : secondaryIndexStats) {
-        secondaryIndexRecords.add(HoodieMetadataPayload.createSecondaryIndexRecord(stat.getRecordKey(), stat.getSecondaryKeyValue(), indexPartitionName, stat.isDeleted()));
-      }
-    }
-    return secondaryIndexRecords;
-  }
-}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexMapper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexMapper.java
new file mode 100644
index 000000000000..0b453de2b475
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/MetadataIndexMapper.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.function.SerializableFunction;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.io.Serializable;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Abstract base class for metadata index mappers. Each index type can extend this to implement
+ * its own record generation logic.
+ */
+public abstract class MetadataIndexMapper implements SerializableFunction<WriteStatus, Iterator<HoodieRecord>>, Serializable {
+  protected final HoodieWriteConfig dataWriteConfig;
+
+  public MetadataIndexMapper(HoodieWriteConfig dataWriteConfig) {
+    this.dataWriteConfig = dataWriteConfig;
+  }
+
+  /**
+   * Generates metadata index records from a WriteStatus.
+   *
+   * @param writeStatus the write status to process
+   * @return list of metadata records
+   */
+  protected abstract List<HoodieRecord> generateRecords(WriteStatus writeStatus);
+
+  /**
+   * Post-processes the generated records. Default implementation returns records as-is.
+   * Subclasses can override to add deduplication, validation, or other transformations.
+   *
+   * @param records the generated records
+   * @return post-processed records
+   */
+  public HoodieData<HoodieRecord> postProcess(HoodieData<HoodieRecord> records) {
+    return records;
+  }
+
+  @Override
+  public final Iterator<HoodieRecord> apply(WriteStatus writeStatus) throws Exception {
+    return generateRecords(writeStatus).iterator();
+  }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java
new file mode 100644
index 000000000000..13029bca3df9
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/RecordIndexMapper.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordDelegate;
+import org.apache.hudi.common.model.HoodieRecordLocation;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieMetadataException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Mapper for Record Level Index (RLI).
+ */
+public class RecordIndexMapper extends MetadataIndexMapper {
+  private static final Logger LOG = LoggerFactory.getLogger(RecordIndexMapper.class);
+
+  public RecordIndexMapper(HoodieWriteConfig dataWriteConfig) {
+    super(dataWriteConfig);
+  }
+
+  @Override
+  protected List<HoodieRecord> generateRecords(WriteStatus writeStatus) {
+    List<HoodieRecord> allRecords = new ArrayList<>();
+    for (HoodieRecordDelegate recordDelegate : writeStatus.getIndexStats().getWrittenRecordDelegates()) {
+      if (!writeStatus.isErrored(recordDelegate.getHoodieKey())) {
+        if (recordDelegate.getIgnoreIndexUpdate()) {
+          continue;
+        }
+        HoodieRecord hoodieRecord;
+        Option<HoodieRecordLocation> newLocation = recordDelegate.getNewLocation();
+        if (newLocation.isPresent()) {
+          if (recordDelegate.getCurrentLocation().isPresent()) {
+            // This is an update, no need to update index if the location has not changed
+            // newLocation should have the same fileID as currentLocation. The instantTimes differ as newLocation's
+            // instantTime refers to the current commit which was completed.
+            if (!recordDelegate.getCurrentLocation().get().getFileId().equals(newLocation.get().getFileId())) {
+              final String msg = String.format("Detected update in location of record with key %s from %s to %s. The fileID should not change.",
+                  recordDelegate, recordDelegate.getCurrentLocation().get(), newLocation.get());
+              LOG.error(msg);
+              throw new HoodieMetadataException(msg);
+            }
+            // for updates, we can skip updating RLI partition in MDT
+          } else {
+            // Insert new record case
+            hoodieRecord = HoodieMetadataPayload.createRecordIndexUpdate(
+                recordDelegate.getRecordKey(), recordDelegate.getPartitionPath(),
+                newLocation.get().getFileId(), newLocation.get().getInstantTime(), dataWriteConfig.getWritesFileIdEncoding());
+            allRecords.add(hoodieRecord);
+          }
+        } else {
+          // Delete existing index for a deleted record
+          hoodieRecord = HoodieMetadataPayload.createRecordIndexDelete(recordDelegate.getRecordKey(), recordDelegate.getPartitionPath(), dataWriteConfig.isPartitionedRecordIndexEnabled());
+          allRecords.add(hoodieRecord);
+        }
+      }
+    }
+    return allRecords;
+  }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexMapper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexMapper.java
new file mode 100644
index 000000000000..b05901392057
--- /dev/null
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexMapper.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.metadata;
+
+import org.apache.hudi.client.SecondaryIndexStats;
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hudi.metadata.HoodieTableMetadataUtil.reduceByKeys;
+
+public class SecondaryIndexMapper extends MetadataIndexMapper {
+  public SecondaryIndexMapper(HoodieWriteConfig dataWriteConfig) {
+    super(dataWriteConfig);
+  }
+
+  @Override
+  protected List<HoodieRecord> generateRecords(WriteStatus writeStatus) {
+    List<HoodieRecord> secondaryIndexRecords = new ArrayList<>(writeStatus.getIndexStats().getSecondaryIndexStats().size());
+    for (Map.Entry<String, List<SecondaryIndexStats>> entry : writeStatus.getIndexStats().getSecondaryIndexStats().entrySet()) {
+      String indexPartitionName = entry.getKey();
+      List<SecondaryIndexStats> secondaryIndexStats = entry.getValue();
+      for (SecondaryIndexStats stat : secondaryIndexStats) {
+        secondaryIndexRecords.add(HoodieMetadataPayload.createSecondaryIndexRecord(stat.getRecordKey(), stat.getSecondaryKeyValue(), indexPartitionName, stat.isDeleted()));
+      }
+    }
+    return secondaryIndexRecords;
+  }
+
+  /**
+   * Post-processes secondary index records by deduplicating records with the same metadata record key.
+   * This handles partition path updates where the same record generates both DELETE and INSERT.
+   * Note that unlike record index, the record may have both partition path update and secondary key
+   * update; in this case, the delete record to the secondary index has to be honored. This is
+   * guaranteed as part of the reduceByKeys operation as in such a case, the DELETE and INSERT
+   * to the secondary index have different metadata record key.
+   */
+  @Override
+  public HoodieData<HoodieRecord> postProcess(HoodieData<HoodieRecord> records) {
+    // Deduplicate by metadata record key (secondaryKey$recordKey)
+    // usePartitionInKey = false because SI keys are globally unique
+    int parallelism = Math.max(
+        records.getNumPartitions(), dataWriteConfig.getMetadataConfig().getSecondaryIndexParallelism());
+    return reduceByKeys(records, parallelism, false);
+  }
+}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
index 2a6599244e05..1bf39c639d9e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/SecondaryIndexRecordGenerationUtils.java
@@ -84,7 +84,6 @@ public class SecondaryIndexRecordGenerationUtils {
    * @param instantTime     instant time
    * @param indexDefinition secondary index definition
    * @param metadataConfig  metadata config
-   * @param fsView          file system view as of instant time
    * @param dataMetaClient  data table meta client
    * @param engineContext   engine context
    * @param writeConfig     hoodie write config.
@@ -97,8 +96,7 @@ public class SecondaryIndexRecordGenerationUtils {
                                                                                       HoodieMetadataConfig metadataConfig,
                                                                                       HoodieTableMetaClient dataMetaClient,
                                                                                       HoodieEngineContext engineContext,
-                                                                                      HoodieWriteConfig writeConfig
-                                                                                      ) {
+                                                                                      HoodieWriteConfig writeConfig) {
     TypedProperties props = writeConfig.getProps();
     // Secondary index cannot support logs having inserts with current offering. So, lets validate that.
     if (allWriteStats.stream().anyMatch(writeStat -> {
@@ -118,7 +116,7 @@ public class SecondaryIndexRecordGenerationUtils {
     int parallelism = Math.max(Math.min(writeStatsByFileId.size(), metadataConfig.getSecondaryIndexParallelism()), 1);
 
     ReaderContextFactory<T> readerContextFactory = engineContext.getReaderContextFactory(dataMetaClient);
-    return engineContext.parallelize(new ArrayList<>(writeStatsByFileId.entrySet()), parallelism).flatMap(writeStatsByFileIdEntry -> {
+    HoodieData<HoodieRecord> secondaryIndexRecords = engineContext.parallelize(new ArrayList<>(writeStatsByFileId.entrySet()), parallelism).flatMap(writeStatsByFileIdEntry -> {
       String fileId = writeStatsByFileIdEntry.getKey();
       List<HoodieWriteStat> writeStats = writeStatsByFileIdEntry.getValue();
       String partition = writeStats.get(0).getPartitionPath();
@@ -199,6 +197,13 @@ public class SecondaryIndexRecordGenerationUtils {
       });
       return records.iterator();
     });
+
+    // Deduplicate secondary index records by grouping by the secondary index key
+    // (secondaryKey$recordKey). This handles the case where a record moves from one file group to
+    // another (partition path update), which generates both a delete (from old fileId) and an
+    // insert (to new fileId). Similar to how Record Level Index handles partition path update,
+    // we prefer non-deleted records.
+    return HoodieTableMetadataUtil.reduceByKeys(secondaryIndexRecords, parallelism, false);
   }
 
   private static TableFileSystemView.SliceView getSliceView(HoodieWriteConfig config, HoodieTableMetaClient dataMetaClient) {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexGenerator.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexMapper.java
similarity index 85%
rename from hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexGenerator.java
rename to hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexMapper.java
index 188e93099ffe..4fe658ce3dcc 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexGenerator.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/metadata/TestMetadataIndexMapper.java
@@ -32,7 +32,6 @@ import org.apache.hudi.config.HoodieWriteConfig;
 
 import org.junit.jupiter.api.Test;
 
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.UUID;
@@ -42,7 +41,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
-public class TestMetadataIndexGenerator extends HoodieCommonTestHarness {
+public class TestMetadataIndexMapper extends HoodieCommonTestHarness {
 
   @Test
   public void testRLIIndexMapperWithInsertsAndUpserts() throws Exception {
@@ -66,9 +65,8 @@ public class TestMetadataIndexGenerator extends HoodieCommonTestHarness {
     HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
         .withPath("random")
         .build();
-    MetadataIndexGenerator.WriteStatusBasedMetadataIndexMapper writeStatusBasedMetadataIndexMapper = new MetadataIndexGenerator.WriteStatusBasedMetadataIndexMapper(
-        Collections.singletonList(MetadataPartitionType.RECORD_INDEX), writeConfig);
-    Iterator<HoodieRecord> rliRecords = writeStatusBasedMetadataIndexMapper.apply(writeStatus);
+    RecordIndexMapper recordIndexMapper = new RecordIndexMapper(writeConfig);
+    Iterator<HoodieRecord> rliRecords = recordIndexMapper.apply(writeStatus);
     AtomicInteger totalRLIRecords = new AtomicInteger();
     rliRecords.forEachRemaining(rliRecord -> {
       totalRLIRecords.getAndIncrement();
@@ -103,9 +101,8 @@ public class TestMetadataIndexGenerator extends HoodieCommonTestHarness {
     HoodieWriteConfig writeConfig = HoodieWriteConfig.newBuilder()
         .withPath("random")
         .build();
-    MetadataIndexGenerator.WriteStatusBasedMetadataIndexMapper writeStatusBasedMetadataIndexMapper = new MetadataIndexGenerator.WriteStatusBasedMetadataIndexMapper(
-        Collections.singletonList(MetadataPartitionType.RECORD_INDEX), writeConfig);
-    Iterator<HoodieRecord> rliRecords = writeStatusBasedMetadataIndexMapper.apply(writeStatus);
+    RecordIndexMapper recordIndexMapper = new RecordIndexMapper(writeConfig);
+    Iterator<HoodieRecord> rliRecords = recordIndexMapper.apply(writeStatus);
     AtomicInteger totalRLIRecords = new AtomicInteger();
     rliRecords.forEachRemaining(rliRecord -> {
       totalRLIRecords.getAndIncrement();
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
index aa2c04db44e4..e9a947a4d5a4 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/metadata/FlinkHoodieBackedTableMetadataWriter.java
@@ -168,11 +168,6 @@ public class FlinkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
     throw new UnsupportedOperationException("Not implemented for Flink engine yet");
   }
 
-  @Override
-  MetadataIndexGenerator initializeMetadataIndexGenerator() {
-    throw new UnsupportedOperationException("Streaming writes are not supported for Flink");
-  }
-
   @Override
   protected EngineType getEngineType() {
     return EngineType.FLINK;
diff --git a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
index 16cfe2835d35..41ecca1fe7f1 100644
--- a/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-java-client/src/main/java/org/apache/hudi/metadata/JavaHoodieBackedTableMetadataWriter.java
@@ -63,11 +63,6 @@ public class JavaHoodieBackedTableMetadataWriter extends HoodieBackedTableMetada
     super(storageConf, writeConfig, failedWritesCleaningPolicy, engineContext, inflightInstantTimestamp);
   }
 
-  @Override
-  MetadataIndexGenerator initializeMetadataIndexGenerator() {
-    throw new UnsupportedOperationException("Streaming writes are not supported for Java");
-  }
-
   public static HoodieTableMetadataWriter create(StorageConfiguration<?> conf,
                                                  HoodieWriteConfig writeConfig,
                                                  HoodieEngineContext context,
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
index d36e59728e90..7b79be758776 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriter.java
@@ -288,10 +288,6 @@ public class SparkHoodieBackedTableMetadataWriter extends HoodieBackedTableMetad
     return exprIndexRecords;
   }
 
-  protected MetadataIndexGenerator initializeMetadataIndexGenerator() {
-    return new MetadataIndexGenerator();
-  }
-
   protected SparkRDDMetadataWriteClient getSparkWriteClient(Option<BaseHoodieWriteClient<?, JavaRDD<HoodieRecord>, ?, JavaRDD<WriteStatus>>> writeClientOpt) {
     return ((SparkRDDMetadataWriteClient) writeClientOpt.orElse(getWriteClient()));
   }
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
index 587759829f83..70e3cfb7ac3b 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/metadata/SparkHoodieBackedTableMetadataWriterTableVersionSix.java
@@ -95,11 +95,6 @@ public class SparkHoodieBackedTableMetadataWriterTableVersionSix extends HoodieB
     super(hadoopConf, writeConfig, failedWritesCleaningPolicy, engineContext, inflightInstantTimestamp);
   }
 
-  @Override
-  MetadataIndexGenerator initializeMetadataIndexGenerator() {
-    throw new UnsupportedOperationException("Streaming writes are not supported for Spark table version six");
-  }
-
   @Override
   protected void initRegistry() {
     if (metadataWriteConfig.isMetricsOn()) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
index ab999f7589ef..39cd361b7102 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java
@@ -1089,25 +1089,40 @@ public class HoodieTableMetadataUtil {
   }
 
   /**
-   * There are chances that same record key from data table has 2 entries (1 delete from older partition and 1 insert to newer partition)
-   * So, this method performs reduce by key to ignore the deleted entry.
-   * @param recordIndexRecords hoodie records after rli index lookup.
-   * @param parallelism parallelism to use.
-   * @return
+   * Reduces metadata table index records by their metadata record key to ensure only a single
+   * update per metadata record key.
+   * <p>
+   * This is critical for handling scenarios where the same metadata record key appears multiple times:
+   * - Partition movements: A data record moving from one partition to another generates both
+   * a delete entry (from old partition) and an insert entry (to new partition)
+   * - File group movements: A record moving from one file group to another
+   * - When usePartitionInKey is true: Groups by (metadata record key, partition path) tuple
+   * - When usePartitionInKey is false: Groups by metadata record key only
+   * <p>
+   * The reduce logic prefers non-deleted records over deleted ones to properly handle movements.
+   * For Record Level Index: When both records are non-deleted (indicating duplicate inserts for
+   * the same key), an exception is thrown since this represents an invalid state.
+   * For Secondary Index: When both records are non-deleted, the later record (record2) is preferred.
+   *
+   * @param metadataRecords   {@link HoodieData} of metadata table index records to be reduced by keys
+   * @param parallelism       parallelism for the reduce-by operation
+   * @param usePartitionInKey whether to use partition path as part of the grouping key
+   *                          (true for partitioned RLI, false for non-partitioned RLI and SI)
+   * @return HoodieData of deduplicated metadata records with one record per metadata record key
    */
   @VisibleForTesting
-  public static HoodieData<HoodieRecord> reduceByKeys(HoodieData<HoodieRecord> recordIndexRecords, int parallelism, boolean isPartitionedRLI) {
-    HoodiePairData<HoodieKey, HoodieRecord> recordIndexRecordsPair;
-    if (isPartitionedRLI) {
-      recordIndexRecordsPair = recordIndexRecords.mapToPair(r -> {
+  public static HoodieData<HoodieRecord> reduceByKeys(HoodieData<HoodieRecord> metadataRecords, int parallelism, boolean usePartitionInKey) {
+    HoodiePairData<HoodieKey, HoodieRecord> metadataRecordsPair;
+    if (usePartitionInKey) {
+      metadataRecordsPair = metadataRecords.mapToPair(r -> {
         String recordPartitionPath = r.isDelete(DELETE_CONTEXT, CollectionUtils.emptyProps())
             ? ((EmptyHoodieRecordPayloadWithPartition) r.getData()).getPartitionPath() : ((HoodieMetadataPayload) r.getData()).getDataPartition();
         return Pair.of(new HoodieKey(r.getRecordKey(), recordPartitionPath), r);
       });
     } else {
-      recordIndexRecordsPair = recordIndexRecords.mapToPair(r -> Pair.of(r.getKey(), r));
+      metadataRecordsPair = metadataRecords.mapToPair(r -> Pair.of(r.getKey(), r));
     }
-    return recordIndexRecordsPair.reduceByKey((SerializableBiFunction<HoodieRecord, HoodieRecord, HoodieRecord>) (record1, record2) -> {
+    return metadataRecordsPair.reduceByKey((SerializableBiFunction<HoodieRecord, HoodieRecord, HoodieRecord>) (record1, record2) -> {
       boolean isRecord1Deleted = record1.isDelete(DELETE_CONTEXT, CollectionUtils.emptyProps());
       boolean isRecord2Deleted = record2.isDelete(DELETE_CONTEXT, CollectionUtils.emptyProps());
       if (isRecord1Deleted && !isRecord2Deleted) {
@@ -1118,7 +1133,8 @@ public class HoodieTableMetadataUtil {
         // let's delete just 1 of them
         return record1;
       } else {
-        throw new HoodieIOException("Two HoodieRecord updates to RLI is seen for same record key " + record2.getRecordKey() + ", record 1 : "
+        // Both records are non-deleted
+        throw new HoodieIOException("Two HoodieRecord updates to the index is seen for same record key " + record2.getRecordKey() + ", record 1 : "
             + record1 + ", record 2 : " + record2);
       }
     }, parallelism).values();
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
index 18cbb29163bb..2bbaf74ce588 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
@@ -49,7 +49,7 @@ import org.apache.spark.sql.types.StringType
 import org.junit.jupiter.api.{Tag, Test}
 import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
 import org.junit.jupiter.params.ParameterizedTest
-import org.junit.jupiter.params.provider.{Arguments, EnumSource, MethodSource, ValueSource}
+import org.junit.jupiter.params.provider.{Arguments, CsvSource, EnumSource, MethodSource, ValueSource}
 import org.junit.jupiter.params.provider.Arguments.arguments
 import org.scalatest.Assertions.{assertResult, assertThrows}
 
@@ -1767,6 +1767,146 @@ class TestSecondaryIndexPruning extends SparkClientFunctionalTestHarness {
     )
   }
 
+  /**
+   * Test Secondary Index with partition path update using global record index.
+   * This test validates that when a record moves from one partition (file group) to another
+   * using global index, the secondary index is correctly updated and queries work as expected.
+   *
+   * Test flow:
+   * 1. Create a table with global index enabled
+   * 2. Insert records into different partitions with a secondary index
+   * 3. Update partition path of a record (moving it from partition A to B)
+   * 4. Validate secondary index metadata is correct (no duplicates, no missing entry)
+   * 5. Validate query results using secondary index pruning
+   */
+  @ParameterizedTest
+  @CsvSource(Array("COPY_ON_WRITE,true", "COPY_ON_WRITE,false", "MERGE_ON_READ,true", "MERGE_ON_READ,false"))
+  def testSecondaryIndexWithPartitionPathUpdateUsingGlobalIndex(tableType: HoodieTableType,
+                                                                enableStreamingWrite: Boolean): Unit = {
+    val sqlTableType = if (tableType == HoodieTableType.COPY_ON_WRITE) "cow" else "mor"
+    val tableName = "test_secondary_index_with_partition_update_global_index_" + sqlTableType + "_" + enableStreamingWrite
+
+    spark.sql(
+      s"""
+         |create table $tableName (
+         |  ts bigint,
+         |  record_key_col string,
+         |  not_record_key_col string,
+         |  partition_key_col string
+         |) using hudi
+         | options (
+         |  primaryKey = 'record_key_col',
+         |  type = '$sqlTableType',
+         |  hoodie.metadata.enable = 'true',
+         |  hoodie.metadata.record.index.enable = 'true',
+         |  hoodie.datasource.write.recordkey.field = 'record_key_col',
+         |  hoodie.enable.data.skipping = 'true',
+         |  hoodie.datasource.write.payload.class = "org.apache.hudi.common.model.OverwriteWithLatestAvroPayload",
+         |  hoodie.index.type = 'RECORD_INDEX',
+         |  hoodie.record.index.update.partition.path = 'true',
+         |  hoodie.metadata.streaming.write.enabled = '$enableStreamingWrite'
+         | )
+         | partitioned by (partition_key_col)
+         | location '$basePath'
+       """.stripMargin)
+
+    spark.sql("set hoodie.parquet.small.file.limit=0")
+    // Insert initial records into different partitions
+    spark.sql(s"insert into $tableName values(1, 'row1', 'abc', 'p1')")
+    spark.sql(s"insert into $tableName values(2, 'row2', 'def', 'p2')")
+    spark.sql(s"insert into $tableName values(3, 'row3', 'ghi', 'p2')")
+    spark.sql(s"insert into $tableName values(4, 'row4', 'ghi', 'p2')")
+
+    // Create secondary index
+    spark.sql(s"create index idx_not_record_key_col on $tableName (not_record_key_col)")
+
+    // Validate index created successfully
+    metaClient = HoodieTableMetaClient.builder()
+      .setBasePath(basePath)
+      .setConf(HoodieTestUtils.getDefaultStorageConf)
+      .build()
+    assert(metaClient.getTableConfig.getMetadataPartitions.contains("secondary_index_idx_not_record_key_col"))
+
+    // Validate the secondary index records before partition update
+    checkAnswer(s"select key, SecondaryIndexMetadata.isDeleted from hudi_metadata('$basePath') where type=7")(
+      Seq(s"abc${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1", false),
+      Seq(s"def${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2", false),
+      Seq(s"ghi${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3", false),
+      Seq(s"ghi${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row4", false)
+    )
+
+    // Validate data skipping with filters on secondary key column
+    spark.sql("set hoodie.metadata.enable=true")
+    spark.sql("set hoodie.enable.data.skipping=true")
+    spark.sql("set hoodie.fileIndex.dataSkippingFailureMode=strict")
+    checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName where not_record_key_col = 'ghi'")(
+      Seq(3, "row3", "ghi", "p2"),
+      Seq(4, "row4", "ghi", "p2")
+    )
+
+    // Update partition path - move row3 from p2 to p3 using MERGE INTO
+    // This moves a record from file group A to file group B
+    spark.sql(
+      s"""
+         |MERGE INTO $tableName AS target
+         |USING (SELECT 3 as ts, 'row3' as record_key_col, 'ghi' as not_record_key_col, 'p3' as partition_key_col) AS source
+         |ON target.record_key_col = source.record_key_col
+         |WHEN MATCHED THEN UPDATE SET *
+       """.stripMargin)
+
+    // Validate the secondary index records after partition update
+    checkAnswer(s"select key, SecondaryIndexMetadata.isDeleted from hudi_metadata('$basePath') where type=7")(
+      Seq(s"abc${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1", false),
+      Seq(s"def${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2", false),
+      Seq(s"ghi${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3", false),
+      Seq(s"ghi${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row4", false)
+    )
+
+    // Validate data after partition update
+    checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName where not_record_key_col = 'ghi'")(
+      Seq(3, "row3", "ghi", "p3"),
+      Seq(4, "row4", "ghi", "p2")
+    )
+
+    // Validate all records are in correct partitions
+    checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName order by record_key_col")(
+      Seq(1, "row1", "abc", "p1"),
+      Seq(2, "row2", "def", "p2"),
+      Seq(3, "row3", "ghi", "p3"),
+      Seq(4, "row4", "ghi", "p2")
+    )
+
+    // Update both secondary column value and partition path
+    // This moves a record from file group A to file group B
+    spark.sql(
+      s"""
+         |MERGE INTO $tableName AS target
+         |USING (SELECT 4 as ts, 'row4' as record_key_col, 'jkl' as not_record_key_col, 'p3' as partition_key_col) AS source
+         |ON target.record_key_col = source.record_key_col
+         |WHEN MATCHED THEN UPDATE SET *
+       """.stripMargin)
+
+    checkAnswer(s"select key, SecondaryIndexMetadata.isDeleted from hudi_metadata('$basePath') where type=7")(
+      Seq(s"abc${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row1", false),
+      Seq(s"def${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row2", false),
+      Seq(s"ghi${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row3", false),
+      Seq(s"jkl${SECONDARY_INDEX_RECORD_KEY_SEPARATOR}row4", false)
+    )
+
+    // Validate data after partition update
+    checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName where not_record_key_col = 'jkl'")(
+      Seq(4, "row4", "jkl", "p3")
+    )
+
+    // Validate all records are in correct partitions
+    checkAnswer(s"select ts, record_key_col, not_record_key_col, partition_key_col from $tableName order by record_key_col")(
+      Seq(1, "row1", "abc", "p1"),
+      Seq(2, "row2", "def", "p2"),
+      Seq(3, "row3", "ghi", "p3"),
+      Seq(4, "row4", "jkl", "p3")
+    )
+  }
+
   private def deleteLastCompletedCommitFromTimeline(hudiOpts: Map[String, String], metaClient: HoodieTableMetaClient): Unit = {
     val lastInstant = metaClient.reloadActiveTimeline.getCommitsTimeline.lastInstant().get()
     assertTrue(hoodieStorage().deleteFile(new StoragePath(metaClient.getTimelinePath, HoodieTestUtils.INSTANT_FILE_NAME_GENERATOR.getFileName(lastInstant))))


(hudi) 05/19: perf: Reduce memory usage of writing HFile log block (#14078)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit ae90b950f20d47f2a0d813ae74bdea6d620cac78
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Fri Oct 24 00:58:59 2025 +0800

    perf: Reduce memory usage of writing HFile log block (#14078)
    
    
    ---------
    
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../org/apache/hudi/io/HoodieAppendHandle.java     |  2 +
 .../hudi/common/model/HoodieAvroIndexedRecord.java |  2 +-
 .../common/table/log/block/HoodieDataBlock.java    |  2 +-
 .../org/apache/hudi/common/util/HFileUtils.java    | 92 ++++++++++------------
 4 files changed, 46 insertions(+), 52 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index 5b49cc4c8f8e..1c2a07c752c6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -73,6 +73,7 @@ import java.io.Closeable;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -718,6 +719,7 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
       case HFILE_DATA_BLOCK:
         // Not supporting positions in HFile data blocks
         header.remove(HeaderMetadataType.BASE_FILE_INSTANT_TIME_OF_RECORD_POSITIONS);
+        records.sort(Comparator.comparing(HoodieRecord::getRecordKey));
         return new HoodieHFileDataBlock(
             records, header, writeConfig.getHFileCompressionAlgorithm(), new StoragePath(writeConfig.getBasePath()));
       case PARQUET_DATA_BLOCK:
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
index 8039ee9d46ec..5f9002b1bc77 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieAvroIndexedRecord.java
@@ -167,10 +167,10 @@ public class HoodieAvroIndexedRecord extends HoodieRecord<IndexedRecord> {
 
   @Override
   public String getRecordKey(Schema recordSchema, String keyFieldName) {
-    decodeRecord(recordSchema);
     if (key != null) {
       return key.getRecordKey();
     }
+    decodeRecord(recordSchema);
     return Option.ofNullable(data.getSchema().getField(keyFieldName))
         .map(keyField -> data.get(keyField.pos()))
         .map(Object::toString).orElse(null);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
index 625c678da6bc..f5327e2318b7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/log/block/HoodieDataBlock.java
@@ -62,7 +62,7 @@ public abstract class HoodieDataBlock extends HoodieLogBlock {
 
   // TODO rebase records/content to leverage Either to warrant
   //      that they are mutex (used by read/write flows respectively)
-  private final Option<List<HoodieRecord>> records;
+  protected Option<List<HoodieRecord>> records;
 
   /**
    * Key field's name w/in the record's schema
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
index 465667fe3c25..cca6410579b1 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java
@@ -56,7 +56,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
-import java.util.TreeMap;
 
 import static org.apache.hudi.common.config.HoodieStorageConfig.HFILE_COMPRESSION_ALGORITHM_NAME;
 import static org.apache.hudi.common.util.StringUtils.getUTF8Bytes;
@@ -191,58 +190,51 @@ public class HFileUtils extends FileFormatUtils {
                                                           Map<String, String> paramsMap) throws IOException {
     CompressionCodec compressionCodec = getHFileCompressionAlgorithm(paramsMap);
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
-    OutputStream ostream = new DataOutputStream(baos);
-
-    // Use simple incrementing counter as a key
-    boolean useIntegerKey = !getRecordKey(records.get(0), readerSchema, keyFieldName).isPresent();
-    // This is set here to avoid re-computing this in the loop
-    int keyWidth = useIntegerKey ? (int) Math.ceil(Math.log(records.size())) + 1 : -1;
-
-    // Serialize records into bytes
-    Map<String, byte[]> sortedRecordsMap = new TreeMap<>();
-
-    Iterator<HoodieRecord> itr = records.iterator();
-    int id = 0;
-    Option<Schema.Field> keyField = Option.ofNullable(writerSchema.getField(keyFieldName));
-    while (itr.hasNext()) {
-      HoodieRecord<?> record = itr.next();
-      String recordKey;
-      if (useIntegerKey) {
-        recordKey = String.format("%" + keyWidth + "s", id++);
-      } else {
-        recordKey = getRecordKey(record, readerSchema, keyFieldName).get();
-      }
+    try (OutputStream ostream = new DataOutputStream(baos)) {
+      HFileContext context = HFileContext.builder()
+          .blockSize(DEFAULT_BLOCK_SIZE_FOR_LOG_FILE)
+          .compressionCodec(compressionCodec)
+          .build();
+      // Use simple incrementing counter as a key
+      boolean useIntegerKey = !getRecordKey(records.get(0), readerSchema, keyFieldName).isPresent();
+      // This is set here to avoid re-computing this in the loop
+      int keyWidth = useIntegerKey ? (int) Math.ceil(Math.log(records.size())) + 1 : -1;
+      int id = 0;
+      Option<Schema.Field> keyField = Option.ofNullable(writerSchema.getField(keyFieldName));
+      try (HFileWriter writer = new HFileWriterImpl(context, ostream)) {
+        String previousRecordKey = null;
+        // It is assumed that the input records are sorted based on the record key
+        // for HFile block
+        for (int i = 0; i < records.size(); i++) {
+          HoodieRecord<?> record = records.get(i);
+          String recordKey = useIntegerKey
+              ? String.format("%" + keyWidth + "s", id++)
+              : getRecordKey(record, readerSchema, keyFieldName).get();
+          final byte[] recordBytes = serializeRecord(record, writerSchema, keyField);
+          // Since the list is sorted, duplicates will be adjacent.
+          if (i > 0 && recordKey.equals(previousRecordKey)) {
+            LOG.error("Found duplicate record with recordKey: {}", recordKey);
+            logRecordMetadata("Previous record",
+                serializeRecord(records.get(i - 1), writerSchema, keyField), writerSchema);
+            logRecordMetadata("Current record",
+                serializeRecord(record, writerSchema, keyField), writerSchema);
+            throw new HoodieException(String.format(
+                "Writing multiple records with same key %s not supported for Hfile format with Metadata table", recordKey));
+          }
+          try {
+            writer.append(recordKey, recordBytes);
+          } catch (IOException e) {
+            throw new HoodieIOException("IOException serializing records", e);
+          }
+          previousRecordKey = recordKey;
+        }
 
-      final byte[] recordBytes = serializeRecord(record, writerSchema, keyField);
-      if (sortedRecordsMap.containsKey(recordKey)) {
-        LOG.error("Found duplicate record with recordKey: {} ", recordKey);
-        logRecordMetadata("Previous record", sortedRecordsMap.get(recordKey), writerSchema);
-        logRecordMetadata("Current record", recordBytes, writerSchema);
-        throw new HoodieException(String.format("Writing multiple records with same key %s not supported for Hfile format with Metadata table",
-            recordKey));
+        writer.appendFileInfo(
+            HoodieAvroHFileReaderImplBase.SCHEMA_KEY,
+            getUTF8Bytes(readerSchema.toString()));
       }
-      sortedRecordsMap.put(recordKey, recordBytes);
+      ostream.flush();
     }
-
-    HFileContext context = HFileContext.builder()
-        .blockSize(DEFAULT_BLOCK_SIZE_FOR_LOG_FILE)
-        .compressionCodec(compressionCodec)
-        .build();
-    try (HFileWriter writer = new HFileWriterImpl(context, ostream)) {
-      sortedRecordsMap.forEach((recordKey,recordBytes) -> {
-        try {
-          writer.append(recordKey, recordBytes);
-        } catch (IOException e) {
-          throw new HoodieIOException("IOException serializing records", e);
-        }
-      });
-      writer.appendFileInfo(
-          HoodieAvroHFileReaderImplBase.SCHEMA_KEY,
-          getUTF8Bytes(readerSchema.toString()));
-    }
-
-    ostream.flush();
-    ostream.close();
     return baos;
   }
 


(hudi) 11/19: fix: Fix the instant time issue for row writer bulk insert hoodie streamer (#14153)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit dec9c8ed2adc035ee2c80b0fafbc4e21b08eef65
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Sun Oct 26 20:03:23 2025 +0530

    fix: Fix the instant time issue for row writer bulk insert hoodie streamer (#14153)
    
    
    ---------
    
    Co-authored-by: Vamsi <va...@onehouse.ai>
---
 .../org/apache/hudi/common/util/CommitUtils.java   |  2 +-
 .../apache/hudi/common/util/TestCommitUtils.java   | 35 ++++++++++++++++++++++
 .../BaseDatasetBulkInsertCommitActionExecutor.java |  4 +--
 .../DatasetBucketRescaleCommitActionExecutor.java  |  4 +--
 .../DatasetBulkInsertCommitActionExecutor.java     |  4 +--
 ...setBulkInsertOverwriteCommitActionExecutor.java |  4 +--
 ...lkInsertOverwriteTableCommitActionExecutor.java |  4 +--
 ...eamerDatasetBulkInsertCommitActionExecutor.java |  4 +--
 .../org/apache/hudi/HoodieSparkSqlWriter.scala     | 12 ++++----
 .../apache/hudi/utilities/streamer/StreamSync.java | 10 +++++--
 .../deltastreamer/HoodieDeltaStreamerTestBase.java | 26 +++++++++++++---
 .../deltastreamer/TestHoodieDeltaStreamer.java     | 20 +++++++------
 ...TestHoodieDeltaStreamerSchemaEvolutionBase.java |  2 +-
 13 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/CommitUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/CommitUtils.java
index 780874a04664..e75e0a0b728a 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/CommitUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/CommitUtils.java
@@ -59,7 +59,7 @@ public class CommitUtils {
    */
   public static String getCommitActionType(WriteOperationType operation, HoodieTableType tableType) {
     if (operation == WriteOperationType.INSERT_OVERWRITE || operation == WriteOperationType.INSERT_OVERWRITE_TABLE
-        || operation == WriteOperationType.DELETE_PARTITION) {
+        || operation == WriteOperationType.DELETE_PARTITION || operation == WriteOperationType.BUCKET_RESCALE) {
       return HoodieTimeline.REPLACE_COMMIT_ACTION;
     } else {
       return getCommitActionType(tableType);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
index 506d3c0b75b2..14255024800a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/util/TestCommitUtils.java
@@ -35,6 +35,8 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -152,6 +154,39 @@ public class TestCommitUtils {
         Option.empty(), CommitUtils.getValidCheckpointForCurrentWriter(timeline, SINK_CHECKPOINT_KEY, ID3));
   }
 
+  @ParameterizedTest
+  @EnumSource(
+      value = WriteOperationType.class,
+      names = {"INSERT_OVERWRITE", "INSERT_OVERWRITE_TABLE", "DELETE_PARTITION", "BUCKET_RESCALE"}
+  )
+  public void testSpecialOverwriteOperationsReturnReplaceCommit(WriteOperationType op) {
+    String resultCOW = CommitUtils.getCommitActionType(op, HoodieTableType.COPY_ON_WRITE);
+    String resultMOR = CommitUtils.getCommitActionType(op, HoodieTableType.MERGE_ON_READ);
+
+    assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, resultCOW);
+    assertEquals(HoodieTimeline.REPLACE_COMMIT_ACTION, resultMOR);
+  }
+
+  @ParameterizedTest
+  @EnumSource(
+      value = WriteOperationType.class,
+      mode = EnumSource.Mode.EXCLUDE,
+      names = {"INSERT_OVERWRITE", "INSERT_OVERWRITE_TABLE", "DELETE_PARTITION", "BUCKET_RESCALE"}
+  )
+  public void testNormalOperationsDelegateBasedOnTableType(WriteOperationType op) {
+    String resultCOW = CommitUtils.getCommitActionType(op, HoodieTableType.COPY_ON_WRITE);
+    String resultMOR = CommitUtils.getCommitActionType(op, HoodieTableType.MERGE_ON_READ);
+
+    assertEquals(HoodieActiveTimeline.COMMIT_ACTION, resultCOW);
+    assertEquals(HoodieActiveTimeline.DELTA_COMMIT_ACTION, resultMOR);
+  }
+
+  @Test
+  public void testNullOperationFallsBackToTableType() {
+    String result = CommitUtils.getCommitActionType(null, HoodieTableType.COPY_ON_WRITE);
+    assertEquals(HoodieActiveTimeline.COMMIT_ACTION, result);
+  }
+
   private HoodieWriteStat createWriteStat(String partition, String fileId) {
     HoodieWriteStat writeStat1 = new HoodieWriteStat();
     writeStat1.setPartitionPath(partition);
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java
index 7612ad95fe3c..194c837f3d7c 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/BaseDatasetBulkInsertCommitActionExecutor.java
@@ -61,13 +61,13 @@ public abstract class BaseDatasetBulkInsertCommitActionExecutor implements Seria
   protected HoodieTable table;
 
   public BaseDatasetBulkInsertCommitActionExecutor(HoodieWriteConfig config,
-                                                   SparkRDDWriteClient writeClient) {
+                                                   SparkRDDWriteClient writeClient, String instantTime) {
     this.writeConfig = config;
     this.writeClient = writeClient;
+    this.instantTime = instantTime;
   }
 
   protected void preExecute() {
-    instantTime = writeClient.startCommit(getCommitActionType());
     table = writeClient.initTable(getWriteOperationType(), Option.ofNullable(instantTime));
     table.validateInsertSchema();
     writeClient.preWrite(instantTime, getWriteOperationType(), table.getMetaClient());
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBucketRescaleCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBucketRescaleCommitActionExecutor.java
index a5bf6013c52a..c18bf38a8ba7 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBucketRescaleCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBucketRescaleCommitActionExecutor.java
@@ -46,8 +46,8 @@ public class DatasetBucketRescaleCommitActionExecutor extends DatasetBulkInsertO
   private final int bucketNumber;
 
   public DatasetBucketRescaleCommitActionExecutor(HoodieWriteConfig config,
-                                                  SparkRDDWriteClient writeClient) {
-    super(config, writeClient);
+                                                  SparkRDDWriteClient writeClient, String instantTime) {
+    super(config, writeClient, instantTime);
     expression = config.getBucketIndexPartitionExpression();
     rule = config.getBucketIndexPartitionRuleType();
     bucketNumber = config.getBucketIndexNumBuckets();
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java
index 8b124de565ca..9a499ca562e3 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertCommitActionExecutor.java
@@ -31,8 +31,8 @@ import java.util.Map;
 public class DatasetBulkInsertCommitActionExecutor extends BaseDatasetBulkInsertCommitActionExecutor {
 
   public DatasetBulkInsertCommitActionExecutor(HoodieWriteConfig config,
-                                               SparkRDDWriteClient writeClient) {
-    super(config, writeClient);
+                                               SparkRDDWriteClient writeClient, String instantTime) {
+    super(config, writeClient, instantTime);
   }
 
   @Override
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java
index 325b1334027a..4f9c16edcc88 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteCommitActionExecutor.java
@@ -43,8 +43,8 @@ import java.util.stream.Collectors;
 public class DatasetBulkInsertOverwriteCommitActionExecutor extends BaseDatasetBulkInsertCommitActionExecutor {
 
   public DatasetBulkInsertOverwriteCommitActionExecutor(HoodieWriteConfig config,
-                                                        SparkRDDWriteClient writeClient) {
-    super(config, writeClient);
+                                                        SparkRDDWriteClient writeClient, String instantTime) {
+    super(config, writeClient, instantTime);
   }
 
   @Override
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
index 4dc82a34064d..d35325360745 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/DatasetBulkInsertOverwriteTableCommitActionExecutor.java
@@ -35,8 +35,8 @@ import java.util.Map;
 public class DatasetBulkInsertOverwriteTableCommitActionExecutor extends DatasetBulkInsertOverwriteCommitActionExecutor {
 
   public DatasetBulkInsertOverwriteTableCommitActionExecutor(HoodieWriteConfig config,
-                                                             SparkRDDWriteClient writeClient) {
-    super(config, writeClient);
+                                                             SparkRDDWriteClient writeClient, String instantTime) {
+    super(config, writeClient, instantTime);
   }
 
   @Override
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/HoodieStreamerDatasetBulkInsertCommitActionExecutor.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/HoodieStreamerDatasetBulkInsertCommitActionExecutor.java
index 8e183629f48b..9858901ad5a6 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/HoodieStreamerDatasetBulkInsertCommitActionExecutor.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/commit/HoodieStreamerDatasetBulkInsertCommitActionExecutor.java
@@ -34,8 +34,8 @@ import java.util.Map;
  */
 public class HoodieStreamerDatasetBulkInsertCommitActionExecutor extends BaseDatasetBulkInsertCommitActionExecutor {
 
-  public HoodieStreamerDatasetBulkInsertCommitActionExecutor(HoodieWriteConfig config, SparkRDDWriteClient writeClient) {
-    super(config, writeClient);
+  public HoodieStreamerDatasetBulkInsertCommitActionExecutor(HoodieWriteConfig config, SparkRDDWriteClient writeClient, String instantTime) {
+    super(config, writeClient, instantTime);
   }
 
   @Override
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 02546c243458..dbcd065552f0 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -817,23 +817,25 @@ class HoodieSparkSqlWriterInternal {
     val overwriteOperationType = Option(hoodieConfig.getString(HoodieInternalConfig.BULKINSERT_OVERWRITE_OPERATION_TYPE))
       .map(WriteOperationType.fromValue)
       .orNull
+
+    val commitActionType = CommitUtils.getCommitActionType(overwriteOperationType, writeConfig.getTableType)
+    val instantTime = writeClient.startCommit(commitActionType)
     val executor = mode match {
       case _ if overwriteOperationType == null =>
         // Don't need to overwrite
-        new DatasetBulkInsertCommitActionExecutor(writeConfig, writeClient)
+        new DatasetBulkInsertCommitActionExecutor(writeConfig, writeClient, instantTime)
       case SaveMode.Append if overwriteOperationType == WriteOperationType.INSERT_OVERWRITE =>
         // INSERT OVERWRITE PARTITION uses Append mode
-        new DatasetBulkInsertOverwriteCommitActionExecutor(writeConfig, writeClient)
+        new DatasetBulkInsertOverwriteCommitActionExecutor(writeConfig, writeClient, instantTime)
       case SaveMode.Append if overwriteOperationType == WriteOperationType.BUCKET_RESCALE =>
-        new DatasetBucketRescaleCommitActionExecutor(writeConfig, writeClient)
+        new DatasetBucketRescaleCommitActionExecutor(writeConfig, writeClient, instantTime)
       case SaveMode.Overwrite if overwriteOperationType == WriteOperationType.INSERT_OVERWRITE_TABLE =>
-        new DatasetBulkInsertOverwriteTableCommitActionExecutor(writeConfig, writeClient)
+        new DatasetBulkInsertOverwriteTableCommitActionExecutor(writeConfig, writeClient, instantTime)
       case _ =>
         throw new HoodieException(s"$mode with bulk_insert in row writer path is not supported yet");
     }
 
     val writeResult = executor.execute(df, tableConfig.isTablePartitioned)
-    val instantTime = executor.getInstantTime
 
     try {
       val (writeSuccessful, compactionInstant, clusteringInstant) = commitAndPerformPostOperations(
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index 9b4498045649..5985abcf5423 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -810,9 +810,13 @@ public class StreamSync implements Serializable, Closeable {
   private HoodieWriteConfig prepareHoodieConfigForRowWriter(Schema writerSchema) {
     HoodieConfig hoodieConfig = new HoodieConfig(HoodieStreamer.Config.getProps(conf, cfg));
     hoodieConfig.setValue(DataSourceWriteOptions.TABLE_TYPE(), cfg.tableType);
-    hoodieConfig.setValue(DataSourceWriteOptions.PAYLOAD_CLASS_NAME().key(), cfg.payloadClassName);
+    if (cfg.payloadClassName != null) {
+      hoodieConfig.setValue(DataSourceWriteOptions.PAYLOAD_CLASS_NAME().key(), cfg.payloadClassName);
+    }
     hoodieConfig.setValue(DataSourceWriteOptions.RECORD_MERGE_MODE().key(), cfg.recordMergeMode.name());
-    hoodieConfig.setValue(DataSourceWriteOptions.RECORD_MERGE_STRATEGY_ID().key(), cfg.recordMergeStrategyId);
+    if (cfg.recordMergeStrategyId != null) {
+      hoodieConfig.setValue(DataSourceWriteOptions.RECORD_MERGE_STRATEGY_ID().key(), cfg.recordMergeStrategyId);
+    }
     hoodieConfig.setValue(HoodieWriteConfig.KEYGENERATOR_CLASS_NAME.key(), HoodieSparkKeyGeneratorFactory.getKeyGeneratorClassName(props));
     hoodieConfig.setValue("path", cfg.targetBasePath);
     return HoodieSparkSqlWriter.getBulkInsertRowConfig(writerSchema != InputBatch.NULL_SCHEMA ? Option.of(writerSchema) : Option.empty(),
@@ -944,7 +948,7 @@ public class StreamSync implements Serializable, Closeable {
     if (useRowWriter) {
       Dataset<Row> df = (Dataset<Row>) inputBatch.getBatch().orElseGet(() -> hoodieSparkContext.getSqlContext().emptyDataFrame());
       HoodieWriteConfig hoodieWriteConfig = prepareHoodieConfigForRowWriter(inputBatch.getSchemaProvider().getTargetSchema());
-      BaseDatasetBulkInsertCommitActionExecutor executor = new HoodieStreamerDatasetBulkInsertCommitActionExecutor(hoodieWriteConfig, writeClient);
+      BaseDatasetBulkInsertCommitActionExecutor executor = new HoodieStreamerDatasetBulkInsertCommitActionExecutor(hoodieWriteConfig, writeClient, instantTime);
       writeClientWriteResult = new WriteClientWriteResult(executor.execute(df, !HoodieStreamerUtils.getPartitionColumns(props).isEmpty()).getWriteStatuses());
     } else {
       TypedProperties mergeProps = ConfigUtils.getMergeProps(props, metaClient.getTableConfig());
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
index 8a0add29b516..ca3edb212195 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
@@ -406,13 +406,20 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
                                          String propsFileName, String parquetSourceRoot, boolean addCommonProps,
                                          String partitionPath, String emptyBatchParam, boolean skipRecordKeyField) throws IOException {
     prepareParquetDFSSource(useSchemaProvider, hasTransformer, sourceSchemaFile, targetSchemaFile, propsFileName, parquetSourceRoot, addCommonProps,
-        partitionPath, emptyBatchParam, null, skipRecordKeyField);
+        partitionPath, emptyBatchParam, null, skipRecordKeyField, false);
+  }
+
+  protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
+                                         String propsFileName, String parquetSourceRoot, boolean addCommonProps,
+                                         String partitionPath, String emptyBatchParam, boolean skipRecordKeyField, boolean useRowWriter) throws IOException {
+    prepareParquetDFSSource(useSchemaProvider, hasTransformer, sourceSchemaFile, targetSchemaFile, propsFileName, parquetSourceRoot, addCommonProps,
+        partitionPath, emptyBatchParam, null, skipRecordKeyField, useRowWriter);
   }
 
   protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
                                        String propsFileName, String parquetSourceRoot, boolean addCommonProps,
                                        String partitionPath, String emptyBatchParam, TypedProperties extraProps,
-                                         boolean skipRecordKeyField) throws IOException {
+                                         boolean skipRecordKeyField, boolean useRowWriter) throws IOException {
     // Properties used for testing delta-streamer with Parquet source
     TypedProperties parquetProps = TypedProperties.copy(extraProps);
 
@@ -424,6 +431,9 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
 
     parquetProps.setProperty("include", "base.properties");
     parquetProps.setProperty("hoodie.embed.timeline.server", "false");
+    if (useRowWriter) {
+      parquetProps.setProperty("hoodie.streamer.write.row.writer.enable", "true");
+    }
     if (!skipRecordKeyField) {
       parquetProps.setProperty("hoodie.datasource.write.recordkey.field", "_row_key");
     }
@@ -626,10 +636,18 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
           tableType, sourceOrderingField, checkpoint, false);
     }
 
+    public static HoodieDeltaStreamer.Config makeConfig(String basePath, WriteOperationType op, String sourceClassName,
+                                                        List<String> transformerClassNames, String propsFilename, boolean enableHiveSync, boolean useSchemaProviderClass,
+                                                        int sourceLimit, boolean updatePayloadClass, String payloadClassName, String tableType, String sourceOrderingField,
+                                                        String checkpoint, boolean allowCommitOnNoCheckpointChange) {
+      return makeConfig(basePath, op, sourceClassName, transformerClassNames, propsFilename, enableHiveSync, useSchemaProviderClass, sourceLimit, updatePayloadClass, payloadClassName,
+          tableType, sourceOrderingField, checkpoint, allowCommitOnNoCheckpointChange, HoodieTableVersion.current());
+    }
+
     static HoodieDeltaStreamer.Config makeConfig(String basePath, WriteOperationType op, String sourceClassName,
                                                  List<String> transformerClassNames, String propsFilename, boolean enableHiveSync, boolean useSchemaProviderClass,
                                                  int sourceLimit, boolean updatePayloadClass, String payloadClassName, String tableType, String sourceOrderingField,
-                                                 String checkpoint, boolean allowCommitOnNoCheckpointChange) {
+                                                 String checkpoint, boolean allowCommitOnNoCheckpointChange, HoodieTableVersion tableVersion) {
       HoodieDeltaStreamer.Config cfg = new HoodieDeltaStreamer.Config();
       cfg.targetBasePath = basePath;
       cfg.targetTableName = "hoodie_trips";
@@ -652,7 +670,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
       Triple<RecordMergeMode, String, String> mergeCfgs =
           HoodieTableConfig.inferMergingConfigsForWrites(
               cfg.recordMergeMode, cfg.payloadClassName, cfg.recordMergeStrategyId, cfg.sourceOrderingFields,
-              HoodieTableVersion.current());
+              tableVersion);
       cfg.recordMergeMode = mergeCfgs.getLeft();
       cfg.payloadClassName = mergeCfgs.getMiddle();
       cfg.recordMergeStrategyId = mergeCfgs.getRight();
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 4d6729bd2da4..42f5aab7559a 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -1163,7 +1163,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
     extraProps.setProperty("hoodie.datasource.write.table.type", "MERGE_ON_READ");
     extraProps.setProperty("hoodie.datasource.compaction.async.enable", "false");
     prepareParquetDFSSource(false, false, "source.avsc", "target.avsc", PROPS_FILENAME_TEST_PARQUET,
-        PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false);
+        PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false, false);
     String tableBasePath = basePath + "test_parquet_table" + testNum;
     HoodieDeltaStreamer.Config deltaCfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, ParquetDFSSource.class.getName(),
         null, PROPS_FILENAME_TEST_PARQUET, false,
@@ -1835,34 +1835,36 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
     testBulkInsertRowWriterMultiBatches(true, null);
   }
 
-  @Test
-  public void testBulkInsertRowWriterWithSchemaProviderAndTransformer() throws Exception {
-    testBulkInsertRowWriterMultiBatches(true, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
+  @ParameterizedTest
+  @EnumSource(value = HoodieTableVersion.class, names = {"SIX", "EIGHT", "NINE"})
+  public void testBulkInsertRowWriterWithSchemaProviderAndTransformer(HoodieTableVersion tableVersion) throws Exception {
+    testBulkInsertRowWriterMultiBatches(true, Collections.singletonList(TripsWithDistanceTransformer.class.getName()), false, tableVersion);
   }
 
   @Test
   public void testBulkInsertRowWriterForEmptyBatch() throws Exception {
-    testBulkInsertRowWriterMultiBatches(false, null, true);
+    testBulkInsertRowWriterMultiBatches(false, null, true, HoodieTableVersion.current());
   }
 
   private void testBulkInsertRowWriterMultiBatches(boolean useSchemaProvider, List<String> transformerClassNames) throws Exception {
-    testBulkInsertRowWriterMultiBatches(useSchemaProvider, transformerClassNames, false);
+    testBulkInsertRowWriterMultiBatches(useSchemaProvider, transformerClassNames, false, HoodieTableVersion.current());
   }
 
-  private void testBulkInsertRowWriterMultiBatches(Boolean useSchemaProvider, List<String> transformerClassNames, boolean testEmptyBatch) throws Exception {
+  private void testBulkInsertRowWriterMultiBatches(Boolean useSchemaProvider, List<String> transformerClassNames, boolean testEmptyBatch, HoodieTableVersion hoodieTableVersion) throws Exception {
     PARQUET_SOURCE_ROOT = basePath + "/parquetFilesDfs" + testNum;
     int parquetRecordsCount = 100;
     boolean hasTransformer = transformerClassNames != null && !transformerClassNames.isEmpty();
     prepareParquetDFSFiles(parquetRecordsCount, PARQUET_SOURCE_ROOT, FIRST_PARQUET_FILE_NAME, false, null, null);
     prepareParquetDFSSource(useSchemaProvider, hasTransformer, "source.avsc", "target.avsc", PROPS_FILENAME_TEST_PARQUET,
-        PARQUET_SOURCE_ROOT, false, "partition_path", "");
+        PARQUET_SOURCE_ROOT, false, "partition_path", "", false, true);
 
     String tableBasePath = basePath + "/test_parquet_table" + testNum;
     HoodieDeltaStreamer.Config cfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.BULK_INSERT, testEmptyBatch ? TestParquetDFSSourceEmptyBatch.class.getName()
             : ParquetDFSSource.class.getName(),
         transformerClassNames, PROPS_FILENAME_TEST_PARQUET, false,
-        useSchemaProvider, 100000, false, null, null, "timestamp", null);
+        useSchemaProvider, 100000, false, null, null, "timestamp", null, false, hoodieTableVersion);
     cfg.configs.add(DataSourceWriteOptions.ENABLE_ROW_WRITER().key() + "=true");
+    cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + hoodieTableVersion.versionCode());
     HoodieDeltaStreamer deltaStreamer = new HoodieDeltaStreamer(cfg, jsc);
     deltaStreamer.sync();
     assertRecordCount(parquetRecordsCount, tableBasePath, sqlContext);
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
index 620d9d23ac6b..d1caf5058ecb 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
@@ -215,7 +215,7 @@ public class TestHoodieDeltaStreamerSchemaEvolutionBase extends HoodieDeltaStrea
           transformerClassNames, PROPS_FILENAME_TEST_AVRO_KAFKA, false,  useSchemaProvider, 100000, false, null, tableType, "timestamp", null);
     } else {
       prepareParquetDFSSource(false, hasTransformer, sourceSchemaFile, targetSchemaFile, PROPS_FILENAME_TEST_PARQUET,
-          PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false);
+          PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false, false);
       cfg = TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, ParquetDFSSource.class.getName(),
           transformerClassNames, PROPS_FILENAME_TEST_PARQUET, false,
           useSchemaProvider, dfsSourceLimitBytes, false, null, tableType, "timestamp", null);


(hudi) 15/19: fix: Give proper error message for multi-writer scenarios without lock provider set (#14119)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 7dcccd6073938f96f0ba6ebb2e68828554d75bfe
Author: Lin Liu <14...@users.noreply.github.com>
AuthorDate: Sun Oct 26 20:46:19 2025 -0700

    fix: Give proper error message for multi-writer scenarios without lock provider set (#14119)
    
    
    ---------
    
    Co-authored-by: Lokesh Jain <lj...@192.168.0.234>
    Co-authored-by: Lokesh Jain <lj...@Lokeshs-MacBook-Pro.local>
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../apache/hudi/index/HoodieSparkIndexClient.java  | 25 ++++++----------------
 .../TestPartitionStatsIndexWithSql.scala           |  4 ++--
 .../functional/TestRecordLevelIndexWithSQL.scala   |  2 ++
 .../functional/TestSecondaryIndexDataTypes.scala   |  5 +++++
 .../functional/TestSecondaryIndexPruning.scala     |  8 ++++++-
 .../hudi/feature/index/TestExpressionIndex.scala   |  1 +
 .../TestHoodieBackedTableMetadataIndexLookup.scala |  1 +
 .../sql/hudi/feature/index/TestIndexSyntax.scala   |  4 ++++
 .../hudi/feature/index/TestSecondaryIndex.scala    |  4 ++++
 .../TestHoodieMetadataTableValidator.java          |  3 +++
 10 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/HoodieSparkIndexClient.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/HoodieSparkIndexClient.java
index ad2f909ef63c..c2c0854f1233 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/HoodieSparkIndexClient.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/HoodieSparkIndexClient.java
@@ -21,8 +21,6 @@ package org.apache.hudi.index;
 
 import org.apache.hudi.client.SparkRDDWriteClient;
 import org.apache.hudi.client.common.HoodieSparkEngineContext;
-import org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider;
-import org.apache.hudi.common.config.HoodieCommonConfig;
 import org.apache.hudi.common.config.HoodieIndexingConfig;
 import org.apache.hudi.common.config.HoodieMetadataConfig;
 import org.apache.hudi.common.config.TypedProperties;
@@ -35,6 +33,7 @@ import org.apache.hudi.common.model.WriteConcurrencyMode;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.TableSchemaResolver;
 import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.ValidationUtils;
 import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.exception.HoodieException;
@@ -43,7 +42,6 @@ import org.apache.hudi.exception.HoodieMetadataIndexException;
 import org.apache.hudi.index.record.HoodieRecordIndex;
 import org.apache.hudi.metadata.HoodieIndexVersion;
 import org.apache.hudi.metadata.MetadataPartitionType;
-import org.apache.hudi.storage.StorageSchemes;
 import org.apache.hudi.table.action.index.BaseHoodieIndexClient;
 
 import org.apache.spark.api.java.JavaSparkContext;
@@ -51,7 +49,6 @@ import org.apache.spark.sql.SparkSession;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
@@ -222,6 +219,12 @@ public class HoodieSparkIndexClient extends BaseHoodieIndexClient {
           .withEngineType(EngineType.SPARK)
           .withProps(configs)
           .build();
+      // Validate if a lock provide class is set properly.
+      if (localWriteConfig.getWriteConcurrencyMode().supportsMultiWriter() && StringUtils.isNullOrEmpty(localWriteConfig.getLockProviderClass())) {
+        throw new IllegalArgumentException(
+            "To create index asynchronously, multi-writer configurations need to be enabled and hence 'hoodie.write.lock.provider' is expected to be set for such cases. "
+                + "For single writer mode, feel free to set the config value to org.apache.hudi.client.transaction.lock.InProcessLockProvider and retry index creation");
+      }
       return new SparkRDDWriteClient(engineContextOpt.get(), localWriteConfig, Option.empty());
     } catch (Exception e) {
       throw new HoodieException("Failed to create write client while performing index operation ", e);
@@ -259,8 +262,6 @@ public class HoodieSparkIndexClient extends BaseHoodieIndexClient {
     Map<String, String> writeConfig = new HashMap<>();
     if (metaClient.getTableConfig().isMetadataTableAvailable()) {
       writeConfig.put(HoodieWriteConfig.WRITE_CONCURRENCY_MODE.key(), WriteConcurrencyMode.OPTIMISTIC_CONCURRENCY_CONTROL.name());
-      writeConfig.putAll(getLockOptions(metaClient.getBasePath().toString(),
-          metaClient.getBasePath().toUri().getScheme(), new TypedProperties()));
 
       // [HUDI-7472] Ensure write-config contains the existing MDT partition to prevent those from getting deleted
       metaClient.getTableConfig().getMetadataPartitions().forEach(partitionPath -> {
@@ -289,16 +290,4 @@ public class HoodieSparkIndexClient extends BaseHoodieIndexClient {
         HoodieIndexingConfig.fromIndexDefinition(indexDefinition).getProps().forEach((key, value) -> writeConfig.put(key.toString(), value.toString())));
     return writeConfig;
   }
-
-  static Map<String, String> getLockOptions(String tablePath, String scheme, TypedProperties lockConfig) {
-    List<String> customSupportedFSs = lockConfig.getStringList(HoodieCommonConfig.HOODIE_FS_ATOMIC_CREATION_SUPPORT.key(), ",", new ArrayList<String>());
-    if (scheme == null || customSupportedFSs.contains(scheme) || StorageSchemes.isAtomicCreationSupported(scheme)) {
-      TypedProperties props = FileSystemBasedLockProvider.getLockConfig(tablePath);
-      Map<String, String> toReturn = new HashMap<>();
-      props.stringPropertyNames().stream().forEach(key -> toReturn.put(key, props.getString(key)));
-      return toReturn;
-    } else {
-      return Collections.emptyMap();
-    }
-  }
 }
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
index 25a7fea6e205..6629b9523541 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
@@ -44,9 +44,9 @@ class TestPartitionStatsIndexWithSql extends HoodieSparkSqlTestBase {
 
   val sqlTempTable = "hudi_tbl"
 
-  @BeforeAll
-  def init(): Unit = {
+  override protected def beforeAll(): Unit = {
     initQueryIndexConf()
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
   }
 
   test("Test drop partition stats index") {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala
index 2e4a83ff0b4d..131da859edd0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndexWithSQL.scala
@@ -54,6 +54,7 @@ class TestRecordLevelIndexWithSQL extends RecordLevelIndexTestBase {
       "hoodie.metadata.index.column.stats.enable" -> "false",
       HoodieMetadataConfig.RECORD_INDEX_ENABLE_PROP.key -> "false")
 
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
     doWriteAndValidateDataAndRecordIndex(hudiOpts,
       operation = DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL,
       saveMode = SaveMode.Overwrite,
@@ -65,6 +66,7 @@ class TestRecordLevelIndexWithSQL extends RecordLevelIndexTestBase {
     metaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(HoodieTestUtils.getDefaultStorageConf).build()
     assertEquals(isPartitioned, HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndex(MetadataPartitionType.RECORD_INDEX.getPartitionPath).get()))
     assertTrue(MetadataPartitionType.RECORD_INDEX.isMetadataPartitionAvailable(getLatestMetaClient(true)))
+    spark.sql("set hoodie.write.lock.provider=")
     spark.sql(s"drop table $sqlTempTable")
   }
 
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexDataTypes.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexDataTypes.scala
index 1305f0f26d94..d59038ec0d40 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexDataTypes.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexDataTypes.scala
@@ -34,6 +34,11 @@ import org.junit.jupiter.api.Tag
  */
 @Tag("functional")
 class TestSecondaryIndexDataTypes extends HoodieSparkSqlTestBase {
+
+  override protected def beforeAll(): Unit = {
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
+  }
+
   /**
    * Test secondary index creation with all data types and verify query behavior.
    *
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
index 2bbaf74ce588..79ade67dc3fa 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSecondaryIndexPruning.scala
@@ -46,7 +46,7 @@ import org.apache.spark.SparkConf
 import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, EqualTo, Expression, Literal}
 import org.apache.spark.sql.types.StringType
-import org.junit.jupiter.api.{Tag, Test}
+import org.junit.jupiter.api.{BeforeEach, Tag, Test}
 import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.{Arguments, CsvSource, EnumSource, MethodSource, ValueSource}
@@ -86,6 +86,12 @@ class TestSecondaryIndexPruning extends SparkClientFunctionalTestHarness {
 
   override def conf: SparkConf = conf(getSparkSqlConf)
 
+  @BeforeEach
+  override def runBeforeEach(): Unit = {
+    super.runBeforeEach()
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
+  }
+
   @Test
   def testSecondaryIndexWithoutRecordIndex(): Unit = {
     tableName += "test_secondary_index_without_rli"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
index abdb72fac284..e7a75fbc2a6a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
@@ -68,6 +68,7 @@ class TestExpressionIndex extends HoodieSparkSqlTestBase with SparkAdapterSuppor
 
   override protected def beforeAll(): Unit = {
     spark.sql("set hoodie.metadata.index.column.stats.enable=false")
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
     spark.sparkContext.persistentRdds.foreach(rdd => rdd._2.unpersist())
     initQueryIndexConf()
   }
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestHoodieBackedTableMetadataIndexLookup.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestHoodieBackedTableMetadataIndexLookup.scala
index 29cff81beebb..e04fd6cbdc6b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestHoodieBackedTableMetadataIndexLookup.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestHoodieBackedTableMetadataIndexLookup.scala
@@ -124,6 +124,7 @@ abstract class HoodieBackedTableMetadataIndexLookupTestBase extends HoodieSparkS
     tmpDir = Utils.createTempDir()
 
     spark.sql("set hoodie.parquet.small.file.limit=0")
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
     // Setup shared test data
     setupSharedTestData()
   }
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
index 78ba187d924f..8fb3fd8648c0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestIndexSyntax.scala
@@ -32,6 +32,10 @@ import org.junit.jupiter.api.Assertions.{assertFalse, assertTrue}
 
 class TestIndexSyntax extends HoodieSparkSqlTestBase {
 
+  override protected def beforeAll(): Unit = {
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
+  }
+
   test("Test Create/Drop/Show/Refresh Index") {
     withTempDir { tmp =>
       Seq("cow", "mor").foreach { tableType =>
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala
index 563c3bf363c9..f31d92d6c753 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestSecondaryIndex.scala
@@ -62,6 +62,10 @@ class TestSecondaryIndex extends HoodieSparkSqlTestBase {
     DataSourceWriteOptions.RECORD_MERGE_MODE.key() -> RecordMergeMode.COMMIT_TIME_ORDERING.name()
   ) ++ metadataOpts
 
+  override protected def beforeAll(): Unit = {
+    spark.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider")
+  }
+
   test("Test Create/Show/Drop Secondary Index with External Table") {
     withRDDPersistenceValidation {
       withTempDir { tmp =>
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index 19cd9473f128..ec781d9bb616 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -313,6 +313,7 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
       // To overwrite the table properties created during test setup
       storage.deleteDirectory(metaClient.getBasePath());
 
+      sparkSession.sql("set hoodie.write.lock.provider = org.apache.hudi.client.transaction.lock.InProcessLockProvider");
       sparkSession.sql(
           "create table tbl ("
               + "ts bigint, "
@@ -355,6 +356,8 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
 
       // validate MDT partition stats
       validateSecondaryIndex();
+
+      sparkSession.sql("set hoodie.write.lock.provider=");
     });
   }
 


(hudi) 14/19: fi: Fixing secondary index read for V1 (#14149)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 63ec30032ff991a8b6ddcf3ac0b6d5b06b9add71
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Sun Oct 26 20:45:42 2025 -0700

    fi: Fixing secondary index read for V1 (#14149)
---
 .../java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java  | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
index 03e6629f5a37..4ec41d412a6e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadata.java
@@ -48,7 +48,6 @@ import org.apache.hudi.common.table.read.buffer.ReusableFileGroupRecordBufferLoa
 import org.apache.hudi.common.table.timeline.HoodieInstant;
 import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
 import org.apache.hudi.common.util.ConfigUtils;
-import org.apache.hudi.common.util.HoodieDataUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.StringUtils;
 import org.apache.hudi.common.util.ValidationUtils;
@@ -457,11 +456,8 @@ public class HoodieBackedTableMetadata extends BaseTableMetadata {
         dataMetaClient.getTableConfig().getMetadataPartitions().contains(partitionName),
         () -> "Secondary index is not initialized in MDT for: " + partitionName);
     // Fetch secondary-index records
-    Map<String, Set<String>> secondaryKeyRecords = HoodieDataUtils.collectPairDataAsMap(
-        readSecondaryIndexDataTableRecordKeysWithKeys(HoodieListData.eager(secondaryKeys.collectAsList()), partitionName));
-    // Now collect the record-keys and fetch the RLI records
-    List<String> recordKeys = new ArrayList<>();
-    secondaryKeyRecords.values().forEach(recordKeys::addAll);
+    HoodiePairData<String, String> pairedData = readSecondaryIndexDataTableRecordKeysWithKeys(HoodieListData.eager(secondaryKeys.collectAsList()), partitionName);
+    List<String> recordKeys = pairedData.map(Pair::getValue).collectAsList();
     return readRecordIndexLocationsWithKeys(HoodieListData.eager(recordKeys));
   }
 


(hudi) 08/19: fix: Upgrade Java xmlbuilder version to fix CVE-2014-125087 (#14144)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 3b1dc608ef6195e49883d4d5c1b02632cbb3b808
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Fri Oct 24 11:46:36 2025 +0530

    fix: Upgrade Java xmlbuilder version to fix CVE-2014-125087 (#14144)
    
    * fix CVE-2014-125087
    
    * Trigger build
    
    * Trigger build
    
    ---------
    
    Co-authored-by: Vamsi <va...@onehouse.ai>
---
 pom.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/pom.xml b/pom.xml
index 33b598b40d0f..badbafae3fe6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1715,6 +1715,11 @@
         <artifactId>jaxb-api</artifactId>
         <version>2.3.1</version>
       </dependency>
+      <dependency>
+        <groupId>com.jamesmurty.utils</groupId>
+        <artifactId>java-xmlbuilder</artifactId>
+        <version>1.2</version>
+      </dependency>
 
       <!--  Spring Boot  -->
       <dependency>


(hudi) 03/19: fix(core): add table level validation for decimal evolution (#14089)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit d13d8ae9c82e7a7d8a78fdf08ae25f55713954c2
Author: Jon Vexler <jb...@gmail.com>
AuthorDate: Wed Oct 22 11:40:40 2025 -0400

    fix(core): add table level validation for decimal evolution (#14089)
    
    Co-authored-by: Jonathan Vexler <=>
---
 .../apache/hudi/avro/AvroSchemaCompatibility.java  | 40 +++++++++++--
 .../org/apache/hudi/avro/TestAvroSchemaUtils.java  | 70 ++++++++++++++++++++++
 2 files changed, 106 insertions(+), 4 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
index 27e326fd2c5f..a1ad79236e2b 100644
--- a/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
+++ b/hudi-common/src/main/java/org/apache/hudi/avro/AvroSchemaCompatibility.java
@@ -22,6 +22,7 @@ import org.apache.hudi.common.util.Either;
 import org.apache.hudi.common.util.Option;
 
 import org.apache.avro.AvroRuntimeException;
+import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
 import org.apache.avro.Schema.Type;
@@ -522,15 +523,46 @@ public class AvroSchemaCompatibility {
 
     private SchemaCompatibilityResult checkFixedSize(final Schema reader, final Schema writer,
                                                      final Deque<LocationInfo> locations) {
-      SchemaCompatibilityResult result = SchemaCompatibilityResult.compatible();
       int actual = reader.getFixedSize();
       int expected = writer.getFixedSize();
       if (actual != expected) {
         String message = String.format("Fixed size field '%s' expected: %d, found: %d", getLocationName(locations, reader.getType()), expected, actual);
-        result = SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, reader, writer,
+        return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.FIXED_SIZE_MISMATCH, reader, writer,
             message, asList(locations));
       }
-      return result;
+      return checkDecimalWidening(reader, writer, locations);
+    }
+
+    private SchemaCompatibilityResult checkDecimalWidening(final Schema reader, final Schema writer,
+                                                           final Deque<LocationInfo> locations) {
+      boolean isReaderDecimal = reader.getLogicalType() instanceof LogicalTypes.Decimal;
+      boolean isWriterDecimal = writer.getLogicalType() instanceof LogicalTypes.Decimal;
+      if (!isReaderDecimal && !isWriterDecimal) {
+        return SchemaCompatibilityResult.compatible();
+      }
+
+      if (!isReaderDecimal || !isWriterDecimal) {
+        String message = String.format("Decimal field '%s' expected: %s, found: %s", getLocationName(locations, reader.getType()), writer, reader);
+        return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.DECIMAL_MISMATCH, reader, writer,
+            message, asList(locations));
+      }
+
+      int readerScale = ((LogicalTypes.Decimal) reader.getLogicalType()).getScale();
+      int writerScale = ((LogicalTypes.Decimal) writer.getLogicalType()).getScale();
+      int readerPrecision = ((LogicalTypes.Decimal) reader.getLogicalType()).getPrecision();
+      int writerPrecision = ((LogicalTypes.Decimal) writer.getLogicalType()).getPrecision();
+      if (readerScale == writerScale && readerPrecision == writerPrecision) {
+        return SchemaCompatibilityResult.compatible();
+      }
+
+      if (((readerPrecision - readerScale) < (writerPrecision - writerScale)) || (readerScale < writerScale)) {
+        String message = String.format("Decimal field '%s' evolution is lossy. Existing precision: %d, scale: %d, Incoming precision: %d, scale: %d",
+            getLocationName(locations, reader.getType()), writerPrecision, writerScale, readerPrecision, readerScale);
+        return SchemaCompatibilityResult.incompatible(SchemaIncompatibilityType.DECIMAL_MISMATCH, reader, writer,
+            message, asList(locations));
+      }
+
+      return SchemaCompatibilityResult.compatible();
     }
 
     private SchemaCompatibilityResult checkSchemaNames(final Schema reader, final Schema writer,
@@ -593,7 +625,7 @@ public class AvroSchemaCompatibility {
 
   public enum SchemaIncompatibilityType {
     NAME_MISMATCH, FIXED_SIZE_MISMATCH, MISSING_ENUM_SYMBOLS, READER_FIELD_MISSING_DEFAULT_VALUE, TYPE_MISMATCH,
-    MISSING_UNION_BRANCH
+    DECIMAL_MISMATCH, MISSING_UNION_BRANCH
   }
 
   /**
diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
index 41b2d1cf1be8..ccfdded4b592 100644
--- a/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/avro/TestAvroSchemaUtils.java
@@ -27,8 +27,11 @@ import org.apache.avro.LogicalTypes;
 import org.apache.avro.Schema;
 import org.apache.parquet.avro.AvroSchemaConverter;
 import org.apache.parquet.schema.MessageType;
+import org.apache.avro.SchemaBuilder;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.util.Arrays;
@@ -36,7 +39,9 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertNull;
@@ -336,6 +341,71 @@ public class TestAvroSchemaUtils {
     AvroSchemaUtils.checkSchemaCompatible(FULL_SCHEMA, SHORT_SCHEMA, shouldValidate, true, Collections.emptySet());
   }
 
+  @ParameterizedTest(name = "[{index}] oldSize={0}, oldPrecision={1}, oldScale={2} -> newSize={3}, newPrecision={4}, newScale={5}")
+  @MethodSource("provideCompatibleDecimalSchemas")
+  void testCompatibleDecimalSchemas(int oldSize, int oldPrecision, int oldScale,
+                                    int newSize, int newPrecision, int newScale) {
+    Schema oldSchema = createFixedDecimalSchema(oldSize, oldPrecision, oldScale);
+    Schema newSchema = createFixedDecimalSchema(newSize, newPrecision, newScale);
+
+    assertDoesNotThrow(() ->
+            AvroSchemaUtils.checkSchemaCompatible(oldSchema, newSchema, true, false, Collections.emptySet()),
+        "Schemas should be compatible"
+    );
+  }
+
+  @ParameterizedTest(name = "[{index}] oldSize={0}, oldPrecision={1}, oldScale={2} -> newSize={3}, newPrecision={4}, newScale={5}")
+  @MethodSource("provideIncompatibleDecimalSchemas")
+  void testIncompatibleDecimalSchemas(int oldSize, int oldPrecision, int oldScale,
+                                      int newSize, int newPrecision, int newScale) {
+    Schema oldSchema = createFixedDecimalSchema(oldSize, oldPrecision, oldScale);
+    Schema newSchema = createFixedDecimalSchema(newSize, newPrecision, newScale);
+
+    assertThrows(Exception.class, () ->
+            AvroSchemaUtils.checkSchemaCompatible(oldSchema, newSchema, true, false, Collections.emptySet()),
+        "Schemas should be incompatible"
+    );
+  }
+
+  private static Stream<Arguments> provideCompatibleDecimalSchemas() {
+    return Stream.of(
+        // Same size, same precision and scale
+        Arguments.of(8, 10, 2, 8, 10, 2),
+
+        // Same size, increased precision, same scale
+        Arguments.of(8, 10, 2, 8, 15, 2),
+
+        // Same size, increased precision and increased scale
+        Arguments.of(16, 20, 5, 16, 25, 10)
+    );
+  }
+
+  private static Stream<Arguments> provideIncompatibleDecimalSchemas() {
+    return Stream.of(
+        // Same size, decreased precision
+        Arguments.of(8, 15, 2, 8, 10, 2),
+
+        // Same size, same precision, increased scale
+        Arguments.of(8, 10, 2, 8, 10, 5),
+
+        // Same size, decreased precision, same scale
+        Arguments.of(16, 25, 3, 16, 20, 3),
+
+        // Same size, both decreased precision and increased scale
+        Arguments.of(8, 18, 4, 8, 15, 6)
+    );
+  }
+
+  private Schema createFixedDecimalSchema(int size, int precision, int scale) {
+    Schema fixedSchema = SchemaBuilder.fixed("FixedDecimal").size(size);
+    Schema decimalSchema = LogicalTypes.decimal(precision, scale).addToSchema(fixedSchema);
+
+    return SchemaBuilder.record("FixedDecimalSchema")
+        .fields()
+        .name("decimalField").type(decimalSchema).noDefault()
+        .endRecord();
+  }
+
   @ParameterizedTest
   @ValueSource(booleans = {false, true})
   public void testIsCompatiblePartitionDropCols(boolean shouldValidate) {


(hudi) 07/19: fix: Upgrade parquet-avro version to 1.15.1 in trino bundle and plugin (#14140)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 0568b0a90df80e6f19f5a1781a75cdeddb701282
Author: vamsikarnika <va...@gmail.com>
AuthorDate: Fri Oct 24 11:43:31 2025 +0530

    fix: Upgrade parquet-avro version to 1.15.1 in trino bundle and plugin (#14140)
    
    
    ---------
    
    Co-authored-by: Lokesh Jain <lj...@192.168.0.234>
    Co-authored-by: Vamsi <va...@onehouse.ai>
---
 hudi-trino-plugin/pom.xml           | 2 ++
 packaging/hudi-trino-bundle/pom.xml | 2 +-
 pom.xml                             | 1 +
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/hudi-trino-plugin/pom.xml b/hudi-trino-plugin/pom.xml
index 126bfb48e001..45a3e8a3ff6d 100644
--- a/hudi-trino-plugin/pom.xml
+++ b/hudi-trino-plugin/pom.xml
@@ -17,6 +17,7 @@
     <properties>
         <air.compiler.fail-warnings>true</air.compiler.fail-warnings>
         <dep.hudi.version>1.0.2</dep.hudi.version>
+        <trino.parquet.version>1.15.1</trino.parquet.version>
     </properties>
 
     <dependencies>
@@ -418,6 +419,7 @@
         <dependency>
             <groupId>org.apache.parquet</groupId>
             <artifactId>parquet-avro</artifactId>
+            <version>${trino.parquet.version}</version>
             <scope>test</scope>
         </dependency>
 
diff --git a/packaging/hudi-trino-bundle/pom.xml b/packaging/hudi-trino-bundle/pom.xml
index abd5a73113ee..aa229f07f6de 100644
--- a/packaging/hudi-trino-bundle/pom.xml
+++ b/packaging/hudi-trino-bundle/pom.xml
@@ -202,7 +202,7 @@
     <dependency>
       <groupId>org.apache.parquet</groupId>
       <artifactId>parquet-avro</artifactId>
-      <version>${parquet.version}</version>
+      <version>${trino.parquet.version}</version>
       <scope>compile</scope>
     </dependency>
 
diff --git a/pom.xml b/pom.xml
index 2b89de9c7064..33b598b40d0f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -214,6 +214,7 @@
     <presto.bundle.bootstrap.shade.prefix>org.apache.hudi.</presto.bundle.bootstrap.shade.prefix>
     <presto.commons-lang3.version>3.18.0</presto.commons-lang3.version>
     <presto.parquet.version>1.15.1</presto.parquet.version>
+    <trino.parquet.version>1.15.1</trino.parquet.version>
     <trino.bundle.bootstrap.scope>compile</trino.bundle.bootstrap.scope>
     <trino.bundle.bootstrap.shade.prefix>org.apache.hudi.</trino.bundle.bootstrap.shade.prefix>
     <twitter.chill.version>0.10.0</twitter.chill.version>


(hudi) 01/19: fix: Fixing error msg when table is not found with MergeIntoCommand (#14118)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 68707164cb3b6a93192774e1ab7f173cc2079b74
Author: Vamshi Krishna Kyatham <68...@users.noreply.github.com>
AuthorDate: Mon Oct 20 22:28:31 2025 -0700

    fix: Fixing error msg when table is not found with MergeIntoCommand (#14118)
---
 .../spark/sql/HoodieCatalystPlansUtils.scala       |  5 ++++
 .../hudi/analysis/HoodieSparkBaseAnalysis.scala    | 10 ++++++-
 .../sql/hudi/dml/others/TestMergeIntoTable.scala   | 33 ++++++++++++++++++++++
 .../spark/sql/HoodieSpark33CatalystPlanUtils.scala |  4 +++
 .../spark/sql/HoodieSpark34CatalystPlanUtils.scala |  6 ++++
 .../spark/sql/HoodieSpark35CatalystPlanUtils.scala |  6 ++++
 .../spark/sql/HoodieSpark40CatalystPlanUtils.scala |  6 ++++
 7 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystPlansUtils.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystPlansUtils.scala
index 42dcc02cca43..a7a652a393f2 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystPlansUtils.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/HoodieCatalystPlansUtils.scala
@@ -144,6 +144,11 @@ trait HoodieCatalystPlansUtils {
    */
   def failAnalysisForMIT(a: Attribute, cols: String): Unit = {}
 
+  /**
+   * Throws TABLE_OR_VIEW_NOT_FOUND error for non-existent table
+   */
+  def failTableNotFound(tableName: String): Unit = {}
+
   def createMITJoin(left: LogicalPlan, right: LogicalPlan, joinType: JoinType, condition: Option[Expression], hint: String): LogicalPlan
 
   /**
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala
index 523049e2a3b7..4443889a6cd4 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/analysis/HoodieSparkBaseAnalysis.scala
@@ -22,7 +22,7 @@ import org.apache.hudi.storage.StoragePath
 
 import org.apache.spark.sql.{AnalysisException, SparkSession}
 import org.apache.spark.sql.BaseHoodieCatalystPlanUtils.MatchResolvedTable
-import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NamedRelation, ResolvedFieldName, UnresolvedAttribute, UnresolvedFieldName, UnresolvedPartitionSpec}
+import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, NamedRelation, ResolvedFieldName, UnresolvedAttribute, UnresolvedFieldName, UnresolvedPartitionSpec, UnresolvedRelation}
 import org.apache.spark.sql.catalyst.analysis.SimpleAnalyzer.resolveExpressionByPlanChildren
 import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogUtils}
 import org.apache.spark.sql.catalyst.expressions.Expression
@@ -155,6 +155,14 @@ case class ResolveReferences(spark: SparkSession) extends Rule[LogicalPlan]
       if !mO.resolved =>
       lazy val analyzer = spark.sessionState.analyzer
       val targetTable = if (targetTableO.resolved) targetTableO else analyzer.execute(targetTableO)
+      EliminateSubqueryAliases(targetTable) match {
+        case u: UnresolvedRelation =>
+          // If target table is still unresolved after analysis, it means the table doesn't exist
+          sparkAdapter.getCatalystPlanUtils.failTableNotFound(u.multipartIdentifier.mkString("."))
+        case _ =>
+          // Target table exists, proceed with normal resolution
+      }
+
       val sourceTable = if (sourceTableO.resolved) sourceTableO else analyzer.execute(sourceTableO)
       val m = mO.asInstanceOf[MergeIntoTable].copy(targetTable = targetTable, sourceTable = sourceTable)
       // END: custom Hudi change
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeIntoTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeIntoTable.scala
index 243ebd93b414..de5abd03fa57 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeIntoTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/others/TestMergeIntoTable.scala
@@ -1759,4 +1759,37 @@ class TestMergeIntoTable extends HoodieSparkSqlTestBase with ScalaAssertionSuppo
       }
     }
   }
+
+  test("Test MergeInto with non existent target table") {
+    withTempDir { tmp =>
+      val sourceTable = generateTableName
+      spark.sql(
+        s"""
+           | create table $sourceTable (
+           |  id int,
+           |  name string,
+           |  price double,
+           |  ts int
+           | ) using parquet
+           | location '${tmp.getCanonicalPath}/$sourceTable'
+         """.stripMargin)
+      spark.sql(s"insert into $sourceTable values(1, 'a1', 10, 1000)")
+      val nonExistentTable = "hudi_test_table"
+      val exception = intercept[org.apache.spark.sql.AnalysisException] {
+        spark.sql(
+          s"""
+             | MERGE INTO $nonExistentTable AS target
+             | USING $sourceTable AS source
+             | ON target.id = source.id
+             | WHEN MATCHED THEN UPDATE SET
+             |   name = source.name,
+             |   ts = source.ts
+             | WHEN NOT MATCHED THEN INSERT *
+         """.stripMargin)
+      }
+      assert(exception.getMessage.contains("TABLE_OR_VIEW_NOT_FOUND") ||
+             exception.getMessage.contains("Table or view not found"),
+        s"Expected TABLE_OR_VIEW_NOT_FOUND error but got: ${exception.getMessage}")
+    }
+  }
 }
diff --git a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
index 7eb13cf43cd5..9a434e90e245 100644
--- a/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
+++ b/hudi-spark-datasource/hudi-spark3.3.x/src/main/scala/org/apache/spark/sql/HoodieSpark33CatalystPlanUtils.scala
@@ -76,6 +76,10 @@ object HoodieSpark33CatalystPlanUtils extends BaseHoodieCatalystPlanUtils {
     a.failAnalysis(s"cannot resolve ${a.sql} in MERGE command given columns [$cols]")
   }
 
+  override def failTableNotFound(tableName: String): Unit = {
+    throw new AnalysisException(s"Table or view not found: $tableName")
+  }
+
   override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = {
     plan match {
       case ci @ CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) =>
diff --git a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala
index 635341f14f9c..8fecd76a89cf 100644
--- a/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala
+++ b/hudi-spark-datasource/hudi-spark3.4.x/src/main/scala/org/apache/spark/sql/HoodieSpark34CatalystPlanUtils.scala
@@ -81,6 +81,12 @@ object HoodieSpark34CatalystPlanUtils extends BaseHoodieCatalystPlanUtils {
         "cols" -> cols))
   }
 
+  override def failTableNotFound(tableName: String): Unit = {
+    throw new AnalysisException(
+      errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+      messageParameters = Map("relationName" -> s"`$tableName`"))
+  }
+
   override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = {
     plan match {
       case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) =>
diff --git a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala
index 7b9666e10b23..64b8a985559b 100644
--- a/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala
+++ b/hudi-spark-datasource/hudi-spark3.5.x/src/main/scala/org/apache/spark/sql/HoodieSpark35CatalystPlanUtils.scala
@@ -80,6 +80,12 @@ object HoodieSpark35CatalystPlanUtils extends BaseHoodieCatalystPlanUtils {
         "proposal" -> cols))
   }
 
+  override def failTableNotFound(tableName: String): Unit = {
+    throw new AnalysisException(
+      errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+      messageParameters = Map("relationName" -> s"`$tableName`"))
+  }
+
   override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = {
     plan match {
       case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) =>
diff --git a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala
index 0111249e959c..859ba13530bd 100644
--- a/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala
+++ b/hudi-spark-datasource/hudi-spark4.0.x/src/main/scala/org/apache/spark/sql/HoodieSpark40CatalystPlanUtils.scala
@@ -80,6 +80,12 @@ object HoodieSpark40CatalystPlanUtils extends BaseHoodieCatalystPlanUtils {
         "proposal" -> cols))
   }
 
+  override def failTableNotFound(tableName: String): Unit = {
+    throw new AnalysisException(
+      errorClass = "TABLE_OR_VIEW_NOT_FOUND",
+      messageParameters = Map("relationName" -> s"`$tableName`"))
+  }
+
   override def unapplyCreateIndex(plan: LogicalPlan): Option[(LogicalPlan, String, String, Boolean, Seq[(Seq[String], Map[String, String])], Map[String, String])] = {
     plan match {
       case ci@CreateIndex(table, indexName, indexType, ignoreIfExists, columns, properties) =>


(hudi) 06/19: fix: Fix cleaning of historical internal schema files (#14126)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 3468a173c2634a2cd8fd2927a65ae859c241f89c
Author: Shuo Cheng <nj...@gmail.com>
AuthorDate: Fri Oct 24 11:53:39 2025 +0800

    fix: Fix cleaning of historical internal schema files (#14126)
---
 .../internal/schema/io/FileBasedInternalSchemaStorageManager.java     | 4 ++--
 .../internal/schema/io/TestFileBasedInternalSchemaStorageManager.java | 2 ++
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
index 633b1b9c5d6f..a436f14c1312 100644
--- a/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
+++ b/hudi-common/src/main/java/org/apache/hudi/internal/schema/io/FileBasedInternalSchemaStorageManager.java
@@ -107,7 +107,7 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
             .filter(f -> f.isFile())
             .map(file -> file.getPath().getName()).collect(Collectors.toList());
         List<String> residualSchemaFiles =
-            candidateSchemaFiles.stream().filter(f -> !validateCommits.contains(f.split("\\.")[0]))
+            candidateSchemaFiles.stream().filter(f -> !validateCommits.contains(getMetaClient().getInstantFileNameParser().extractTimestamp(f)))
                 .collect(Collectors.toList());
         // clean residual files
         residualSchemaFiles.forEach(f -> {
@@ -130,7 +130,7 @@ public class FileBasedInternalSchemaStorageManager extends AbstractInternalSchem
             .filter(f -> f.isFile())
             .map(file -> file.getPath().getName()).collect(Collectors.toList());
         List<String> validateSchemaFiles =
-            candidateSchemaFiles.stream().filter(f -> validateCommits.contains(f.split("\\.")[0]))
+            candidateSchemaFiles.stream().filter(f -> validateCommits.contains(getMetaClient().getInstantFileNameParser().extractTimestamp(f)))
                 .collect(Collectors.toList());
         for (int i = 0; i < validateSchemaFiles.size(); i++) {
           storage.deleteFile(new StoragePath(validateSchemaFiles.get(i)));
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java
index fe53e575fd80..9c92c107815e 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/internal/schema/io/TestFileBasedInternalSchemaStorageManager.java
@@ -94,6 +94,8 @@ public class TestFileBasedInternalSchemaStorageManager extends HoodieCommonTestH
     // now the residual file created by 3st persist should be removed.
     File f = new File(metaClient.getSchemaFolderName() + File.separator + "0002.schemacommit");
     assertTrue(!f.exists());
+    assertEquals(currentSchema, fm.getSchemaByKey("0").get());
+    assertEquals(secondSchema, fm.getSchemaByKey("1").get());
     assertEquals(lastSchema, fm.getSchemaByKey("3").get());
   }
 


(hudi) 16/19: feat: Partition predicate fix for Databricks runtime support (#14059)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 9510647bfa9b7b9f63780d57b9a4b55c0b4d2c4e
Author: Aditya Goenka <63...@users.noreply.github.com>
AuthorDate: Mon Oct 27 11:59:32 2025 +0530

    feat: Partition predicate fix for Databricks runtime support (#14059)
---
 .../org/apache/hudi/SparkHoodieTableFileIndex.scala  | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
index d84d0339499b..622e57f8bab2 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/SparkHoodieTableFileIndex.scala
@@ -41,7 +41,7 @@ import org.apache.spark.api.java.JavaSparkContext
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.{expressions, InternalRow}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, EmptyRow, EqualTo, Expression, InterpretedPredicate, Literal}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BasePredicate, BoundReference, EmptyRow, EqualTo, Expression, InterpretedPredicate, Literal}
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.datasources.{FileStatusCache, NoopCache}
 import org.apache.spark.sql.internal.SQLConf
@@ -248,12 +248,24 @@ class SparkHoodieTableFileIndex(spark: SparkSession,
       //       the whole table
       if (haveProperPartitionValues(partitionPaths.toSeq) && partitionSchema.nonEmpty) {
         val predicate = partitionPruningPredicates.reduce(expressions.And)
-        val boundPredicate = InterpretedPredicate(predicate.transform {
+        val transformedPredicate = predicate.transform {
           case a: AttributeReference =>
             val index = partitionSchema.indexWhere(a.name == _.name)
             BoundReference(index, partitionSchema(index).dataType, nullable = true)
-        })
-
+        }
+        val boundPredicate: BasePredicate = try {
+          // Try using 1-arg constructor via reflection
+          val clazz = Class.forName("org.apache.spark.sql.catalyst.expressions.InterpretedPredicate")
+          val ctor = clazz.getConstructor(classOf[Expression])
+          ctor.newInstance(transformedPredicate).asInstanceOf[BasePredicate]
+        } catch {
+          case _: NoSuchMethodException | _: IllegalArgumentException =>
+            // Fallback: Try using 2-arg constructor for certain Spark runtime
+            val clazz = Class.forName("org.apache.spark.sql.catalyst.expressions.InterpretedPredicate")
+            val ctor = clazz.getConstructor(classOf[Expression], classOf[Boolean])
+            ctor.newInstance(transformedPredicate, java.lang.Boolean.FALSE)
+              .asInstanceOf[BasePredicate]
+        }
         val prunedPartitionPaths = partitionPaths.filter {
           partitionPath => boundPredicate.eval(InternalRow.fromSeq(partitionPath.values))
         }.toSeq


(hudi) 17/19: fix: Partition stats should be controlled using column stats config (#14165)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit dfd26b46ffb6645f4ca42575a1cdb2d559138ad4
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Tue Oct 28 07:50:05 2025 +0530

    fix: Partition stats should be controlled using column stats config (#14165)
    
    
    ---------
    
    Co-authored-by: Lokesh Jain <lj...@192.168.0.234>
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../org/apache/hudi/config/HoodieWriteConfig.java  |  5 +---
 .../metadata/HoodieBackedTableMetadataWriter.java  |  3 +--
 .../table/functional/TestCleanPlanExecutor.java    |  3 +--
 .../hudi/common/config/HoodieMetadataConfig.java   | 30 ++++++----------------
 .../org/apache/hudi/sink/TestWriteCopyOnWrite.java |  2 --
 .../hudi/sink/TestWriteMergeOnReadWithCompact.java |  3 ---
 .../java/org/apache/hudi/source/TestFileIndex.java |  1 -
 .../hudi/source/TestIncrementalInputSplits.java    |  1 -
 .../apache/hudi/table/ITTestHoodieDataSource.java  |  2 --
 .../apache/hudi/table/ITTestSchemaEvolution.java   |  3 +--
 .../apache/hudi/table/TestHoodieTableSource.java   |  1 -
 .../hudi/metadata/TestHoodieTableMetadataUtil.java |  2 --
 .../TestDataSkippingWithMORColstats.java           |  3 ---
 .../hudi/functional/TestHoodieBackedMetadata.java  |  2 --
 .../apache/hudi/io/TestMetadataWriterCommit.java   |  6 ++---
 .../org/apache/hudi/TestHoodieSparkSqlWriter.scala |  2 +-
 .../functional/PartitionStatsIndexTestBase.scala   |  3 +--
 .../hudi/functional/TestBasicSchemaEvolution.scala |  3 +--
 .../hudi/functional/TestPartitionStatsIndex.scala  | 22 ----------------
 .../TestPartitionStatsIndexWithSql.scala           |  2 +-
 .../functional/TestPartitionStatsPruning.scala     | 19 ++++----------
 .../hudi/feature/index/TestExpressionIndex.scala   |  4 +--
 .../TestHoodieMetadataTableValidator.java          |  2 --
 23 files changed, 26 insertions(+), 98 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index ca36a5eb4b2e..c1ba48226bcb 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -2134,10 +2134,7 @@ public class HoodieWriteConfig extends HoodieConfig {
    * @return {@code true} if the partition stats index is enabled, {@code false} otherwise.
    */
   public boolean isPartitionStatsIndexEnabled() {
-    if (isMetadataColumnStatsIndexEnabled()) {
-      return isMetadataTableEnabled() && getMetadataConfig().isPartitionStatsIndexEnabled();
-    }
-    return false;
+    return isMetadataColumnStatsIndexEnabled();
   }
 
   /**
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 51d5a9016081..20f9d1f6601a 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -440,8 +440,7 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab
     while (iterator.hasNext()) {
       MetadataPartitionType partitionType = iterator.next();
       if (partitionType == PARTITION_STATS && !dataMetaClient.getTableConfig().isTablePartitioned()) {
-        LOG.debug("Partition stats index cannot be enabled for a non-partitioned table. Removing from initialization list. Please disable {}",
-            HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key());
+        // Partition stats index cannot be enabled for a non-partitioned table
         iterator.remove();
         this.enabledPartitionTypes.remove(partitionType);
       }
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
index 169c95e5911b..e572fef12260 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestCleanPlanExecutor.java
@@ -341,8 +341,7 @@ public class TestCleanPlanExecutor extends HoodieCleanerTestBase {
   public void testKeepLatestFileVersionsWithBootstrapFileClean() throws Exception {
     HoodieWriteConfig config =
         HoodieWriteConfig.newBuilder().withPath(basePath)
-            .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMetadataIndexColumnStats(false)
-                .withMetadataIndexPartitionStats(false).build())
+            .withMetadataConfig(HoodieMetadataConfig.newBuilder().withMetadataIndexColumnStats(false).build())
             .withCleanConfig(HoodieCleanConfig.newBuilder()
                 .withCleanBootstrapBaseFileEnabled(true)
                 .withCleanerParallelism(1)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
index 96c95708ab75..ace77303d52c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
@@ -420,21 +420,6 @@ public final class HoodieMetadataConfig extends HoodieConfig {
       .sinceVersion("1.0.1")
       .withDocumentation("Options for the expression index, e.g. \"expr='from_unixtime', format='yyyy-MM-dd'\"");
 
-  public static final ConfigProperty<Boolean> ENABLE_METADATA_INDEX_PARTITION_STATS = ConfigProperty
-      .key(METADATA_PREFIX + ".index.partition.stats.enable")
-      // The defaultValue(false) here is the initial default, but it's overridden later based on
-      // column stats setting.
-      .defaultValue(false)
-      .sinceVersion("1.0.0")
-      .withDocumentation("Enable aggregating stats for each column at the storage partition level. "
-          + "Enabling this can improve query performance by leveraging partition and column stats "
-          + "for (partition) filtering. "
-          + "Important: The default value for this configuration is dynamically set based on the "
-          + "effective value of " + ENABLE_METADATA_INDEX_COLUMN_STATS.key() + ". If column stats "
-          + "index is enabled (default for Spark engine), partition stats indexing will also be "
-          + "enabled by default. Conversely, if column stats indexing is disabled (default for "
-          + "Flink and Java engines), partition stats indexing will also be disabled by default.");
-
   public static final ConfigProperty<Integer> METADATA_INDEX_PARTITION_STATS_FILE_GROUP_COUNT = ConfigProperty
       .key(METADATA_PREFIX + ".index.partition.stats.file.group.count")
       .defaultValue(1)
@@ -811,7 +796,7 @@ public final class HoodieMetadataConfig extends HoodieConfig {
   }
 
   public boolean isPartitionStatsIndexEnabled() {
-    return getBooleanOrDefault(ENABLE_METADATA_INDEX_PARTITION_STATS);
+    return getBooleanOrDefault(ENABLE_METADATA_INDEX_COLUMN_STATS);
   }
 
   public int getPartitionStatsIndexFileGroupCount() {
@@ -1114,11 +1099,6 @@ public final class HoodieMetadataConfig extends HoodieConfig {
       return this;
     }
 
-    public Builder withMetadataIndexPartitionStats(boolean enable) {
-      metadataConfig.setValue(ENABLE_METADATA_INDEX_PARTITION_STATS, String.valueOf(enable));
-      return this;
-    }
-
     public Builder withMetadataIndexPartitionStatsFileGroupCount(int fileGroupCount) {
       metadataConfig.setValue(METADATA_INDEX_PARTITION_STATS_FILE_GROUP_COUNT, String.valueOf(fileGroupCount));
       return this;
@@ -1187,7 +1167,6 @@ public final class HoodieMetadataConfig extends HoodieConfig {
     public HoodieMetadataConfig build() {
       metadataConfig.setDefaultValue(ENABLE, getDefaultMetadataEnable(engineType));
       metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_COLUMN_STATS, getDefaultColStatsEnable(engineType));
-      metadataConfig.setDefaultValue(ENABLE_METADATA_INDEX_PARTITION_STATS, metadataConfig.isColumnStatsIndexEnabled());
       metadataConfig.setDefaultValue(SECONDARY_INDEX_ENABLE_PROP, getDefaultSecondaryIndexEnable(engineType));
       metadataConfig.setDefaultValue(STREAMING_WRITE_ENABLED, getDefaultForStreamingWriteEnabled(engineType));
       // fix me: disable when schema on read is enabled.
@@ -1244,6 +1223,13 @@ public final class HoodieMetadataConfig extends HoodieConfig {
     }
   }
 
+  /**
+   * The config is now deprecated. Partition stats are configured using the column stats config itself.
+   */
+  @Deprecated
+  public static final String ENABLE_METADATA_INDEX_PARTITION_STATS =
+      METADATA_PREFIX + ".index.partition.stats.enable";
+
   /**
    * @deprecated Use {@link #ENABLE} and its methods.
    */
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
index d6b7108e3217..c8e4dd4c6c06 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteCopyOnWrite.java
@@ -19,7 +19,6 @@
 package org.apache.hudi.sink;
 
 import org.apache.hudi.client.HoodieFlinkWriteClient;
-import org.apache.hudi.common.config.HoodieMetadataConfig;
 import org.apache.hudi.common.model.HoodieFailedWritesCleaningPolicy;
 import org.apache.hudi.common.model.HoodieKey;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
@@ -747,7 +746,6 @@ public class TestWriteCopyOnWrite extends TestWriteBase {
   public void testReuseEmbeddedServer() throws IOException {
     conf.setString("hoodie.filesystem.view.remote.timeout.secs", "500");
     conf.setString("hoodie.metadata.enable","true");
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false"); // HUDI-8814
 
     HoodieFlinkWriteClient writeClient = null;
     HoodieFlinkWriteClient writeClient2 = null;
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
index ee92a46fe681..7515f8904327 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestWriteMergeOnReadWithCompact.java
@@ -19,7 +19,6 @@
 package org.apache.hudi.sink;
 
 import org.apache.hudi.client.HoodieFlinkWriteClient;
-import org.apache.hudi.common.config.HoodieMetadataConfig;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.PartialUpdateAvroPayload;
 import org.apache.hudi.common.model.WriteConcurrencyMode;
@@ -97,7 +96,6 @@ public class TestWriteMergeOnReadWithCompact extends TestWriteCopyOnWrite {
     // disable schedule compaction in writers
     conf.set(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, false);
     conf.set(FlinkOptions.PRE_COMBINE, true);
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false"); // HUDI-8814
 
     // start pipeline1 and insert record: [id1,Danny,null,1,par1], suspend the tx commit
     List<RowData> dataset1 = Collections.singletonList(
@@ -292,7 +290,6 @@ public class TestWriteMergeOnReadWithCompact extends TestWriteCopyOnWrite {
     // disable schedule compaction in writers
     conf.set(FlinkOptions.COMPACTION_SCHEDULE_ENABLED, false);
     conf.set(FlinkOptions.PRE_COMBINE, true);
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false");
 
     Configuration conf1 = conf.clone();
     conf1.set(FlinkOptions.OPERATION, "BULK_INSERT");
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestFileIndex.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestFileIndex.java
index ef5d809996bf..a50f4a2a5243 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestFileIndex.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestFileIndex.java
@@ -173,7 +173,6 @@ public class TestFileIndex {
     conf.set(READ_DATA_SKIPPING_ENABLED, true);
     conf.set(METADATA_ENABLED, true);
     conf.set(TABLE_TYPE, tableType.name());
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true");
     conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true");
     if (tableType == HoodieTableType.MERGE_ON_READ) {
       // enable CSI for MOR table to collect col stats for delta write stats,
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
index 3595287b6c7d..fb9275c2fc35 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
@@ -360,7 +360,6 @@ public class TestIncrementalInputSplits extends HoodieCommonTestHarness {
     conf.set(FlinkOptions.READ_AS_STREAMING, true);
     conf.set(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true);
     conf.set(FlinkOptions.TABLE_TYPE, tableType.name());
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true");
     conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true");
     if (tableType == HoodieTableType.MERGE_ON_READ) {
       // enable CSI for MOR table to collect col stats for delta write stats,
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
index b08e772c198a..c89148173b10 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestHoodieDataSource.java
@@ -530,7 +530,6 @@ public class ITTestHoodieDataSource {
         .option(FlinkOptions.PATH, tempFile.getAbsolutePath())
         .option(FlinkOptions.METADATA_ENABLED, true)
         .option(FlinkOptions.READ_AS_STREAMING, true)
-        .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), true)
         .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), false)
         .option(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true)
         .option(FlinkOptions.TABLE_TYPE, tableType)
@@ -538,7 +537,6 @@ public class ITTestHoodieDataSource {
         .end();
     streamTableEnv.executeSql(hoodieTableDDL);
     Configuration conf = TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true");
     conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true");
     conf.set(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true);
     conf.set(FlinkOptions.TABLE_TYPE, tableType.name());
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java
index 895305db0f32..601625242991 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java
@@ -383,8 +383,7 @@ public class ITTestSchemaEvolution {
         FlinkOptions.COMPACTION_SCHEDULE_ENABLED.key(), false,
         HoodieWriteConfig.EMBEDDED_TIMELINE_SERVER_REUSE_ENABLED.key(), false,
         HoodieCommonConfig.SCHEMA_EVOLUTION_ENABLE.key(), true,
-        HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true",
-        HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false");
+        HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true");
   }
 
   private void checkAnswerEvolved(String... expectedResult) throws Exception {
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
index f12fc0b832f7..6d35ac1f1adf 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/TestHoodieTableSource.java
@@ -178,7 +178,6 @@ public class TestHoodieTableSource {
   void testDataSkippingWithPartitionStatsPruning(List<ResolvedExpression> filters, List<String> expectedPartitions) throws Exception {
     final String path = tempFile.getAbsolutePath();
     conf = TestConfigurations.getDefaultConf(path);
-    conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true");
     conf.setString(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key(), "true");
     conf.set(FlinkOptions.READ_DATA_SKIPPING_ENABLED, true);
     TestData.writeData(TestData.DATA_SET_INSERT, conf);
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
index 889eae6d32c0..643e51196ff9 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/metadata/TestHoodieTableMetadataUtil.java
@@ -172,7 +172,6 @@ public class TestHoodieTableMetadataUtil extends HoodieCommonTestHarness {
         partitionFileSlicePairs,
         HoodieMetadataConfig.newBuilder().enable(true)
             .withMetadataIndexColumnStats(true)
-            .withMetadataIndexPartitionStats(true)
             .withColumnStatsIndexForColumns("rider,driver")
             .withPartitionStatsIndexParallelism(1)
             .build(),
@@ -290,7 +289,6 @@ public class TestHoodieTableMetadataUtil extends HoodieCommonTestHarness {
         partitionFileSlicePairs,
         HoodieMetadataConfig.newBuilder().enable(true)
             .withMetadataIndexColumnStats(true)
-            .withMetadataIndexPartitionStats(true)
             .withColumnStatsIndexForColumns("rider,driver")
             .withPartitionStatsIndexParallelism(1)
             .build(),
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestDataSkippingWithMORColstats.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestDataSkippingWithMORColstats.java
index 432d8e90c7f0..67e95008903b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestDataSkippingWithMORColstats.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestDataSkippingWithMORColstats.java
@@ -119,7 +119,6 @@ public class TestDataSkippingWithMORColstats extends HoodieSparkClientTestBase {
     // we have specified which columns to index
     options.put(HoodieMetadataConfig.COLUMN_STATS_INDEX_MAX_COLUMNS.key(), "1");
 
-    options.put(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false");
     Dataset<Row> inserts = makeInsertDf("000", 100);
     Dataset<Row> batch1 = inserts.where(matchCond);
     Dataset<Row> batch2 = inserts.where(nonMatchCond);
@@ -190,7 +189,6 @@ public class TestDataSkippingWithMORColstats extends HoodieSparkClientTestBase {
    */
   @Test
   public void testBaseFileAndLogFileUpdateMatchesDeleteBlockCompact() {
-    options.put(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false");
     testBaseFileAndLogFileUpdateMatchesHelper(false, true,true, false);
   }
 
@@ -204,7 +202,6 @@ public class TestDataSkippingWithMORColstats extends HoodieSparkClientTestBase {
    */
   @Test
   public void testBaseFileAndLogFileUpdateMatchesAndRollBack() {
-    options.put(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "false");
     testBaseFileAndLogFileUpdateMatchesHelper(false, false,false, true);
   }
 
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
index 1be6c3d1218f..e20bb31abfb9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestHoodieBackedMetadata.java
@@ -342,7 +342,6 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
         .withMetadataConfig(HoodieMetadataConfig.newBuilder()
             .withProperties(cfg.getMetadataConfig().getProps())
             .withMetadataIndexColumnStats(false)
-            .withMetadataIndexPartitionStats(false)
             .build())
         .build();
 
@@ -1145,7 +1144,6 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
             // Disable the other two default index for this test because the test orchestrates
             // the rollback with the assumption of init commits being in certain order
             .withMetadataIndexColumnStats(false)
-            .withMetadataIndexPartitionStats(false)
             .build())
         .build();
 
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java
index 6ad06102511b..48d01e13768a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/TestMetadataWriterCommit.java
@@ -129,7 +129,7 @@ public class TestMetadataWriterCommit extends BaseTestHandle {
     assertEquals(10, mdtCommitMetadata.getPartitionToWriteStats().get(RECORD_INDEX.getPartitionPath()).size());
     assertFalse(mdtCommitMetadata.getPartitionToWriteStats().containsKey(COLUMN_STATS.getPartitionPath()));
 
-    // Create commit in MDT with col stats enabled
+    // Create commit in MDT with col stats enabled (partition stats is enabled with column stats)
     config.getMetadataConfig().setValue(HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS, "true");
     instantTime = InProcessTimeGenerator.createNewInstantTime();
     mdtWriter = (HoodieBackedTableMetadataWriter) SparkMetadataWriterFactory.createWithStreamingWrites(storageConf, config,
@@ -138,8 +138,8 @@ public class TestMetadataWriterCommit extends BaseTestHandle {
     mdtWriteStatus = mdtWriter.streamWriteToMetadataPartitions(HoodieJavaRDD.of(Collections.singletonList(writeStatus), context, 1), instantTime);
     mdtWriteStats = mdtWriteStatus.collectAsList().stream().map(WriteStatus::getStat).collect(Collectors.toList());
     mdtWriter.completeStreamingCommit(instantTime, context, mdtWriteStats, commitMetadata);
-    // 3 bootstrap commits for 3 enabled partitions, 2 commits due to update
-    assertEquals(5, mdtMetaClient.reloadActiveTimeline().filterCompletedInstants().countInstants());
+    // 3 bootstrap commits for 4 enabled partitions, 2 commits due to update
+    assertEquals(6, mdtMetaClient.reloadActiveTimeline().filterCompletedInstants().countInstants());
 
     // Verify commit metadata
     mdtCommitMetadata = mdtMetaClient.getActiveTimeline().readCommitMetadata(mdtMetaClient.getActiveTimeline().lastInstant().get());
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 496c7b1790f2..689ce52dc009 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -895,7 +895,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
     val partitionStatsIndex = new PartitionStatsIndexSupport(
       spark,
       inputDf.schema,
-      HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build(),
+      HoodieMetadataConfig.newBuilder().enable(true).build(),
       metaClient)
     val partitionStats = partitionStatsIndex.loadColumnStatsIndexRecords(List("partition", "ts"), shouldReadInMemory = true).collectAsList()
     partitionStats.forEach(stat => {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
index 5517ce654d45..ef6012518ebc 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/PartitionStatsIndexTestBase.scala
@@ -42,7 +42,6 @@ class PartitionStatsIndexTestBase extends HoodieStatsIndexTestBase {
   val targetColumnsToIndex: Seq[String] = Seq("rider", "driver")
   val metadataOpts: Map[String, String] = Map(
     HoodieMetadataConfig.ENABLE.key -> "true",
-    HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
     HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
     HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> targetColumnsToIndex.mkString(",")
   )
@@ -113,7 +112,7 @@ class PartitionStatsIndexTestBase extends HoodieStatsIndexTestBase {
     val partitionStatsIndex = new PartitionStatsIndexSupport(
       spark,
       inputDf.schema,
-      HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build(),
+      HoodieMetadataConfig.newBuilder().enable(true).build(),
       metaClient)
     val partitionStats = partitionStatsIndex.loadColumnStatsIndexRecords(List("partition", "trip_type"), shouldReadInMemory = true).collectAsList()
     assertEquals(0, partitionStats.size())
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
index 66ee2fa88008..26fa1c722344 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
@@ -55,8 +55,7 @@ class TestBasicSchemaEvolution extends HoodieSparkClientTestBase with ScalaAsser
     DataSourceWriteOptions.RECORDKEY_FIELD.key -> "_row_key",
     DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "partition",
     HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
-    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
-    HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true"
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test"
   )
 
   val verificationCol: String = "driver"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
index 108f59ba82e6..1d9362769158 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndex.scala
@@ -65,24 +65,6 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
 
   val sqlTempTable = "hudi_tbl"
 
-  /**
-   * Test case to validate partition stats cannot be created without column stats.
-   */
-  @Test
-  def testPartitionStatsWithoutColumnStats(): Unit = {
-    // remove column stats enable key from commonOpts
-    val hudiOpts = commonOpts + (HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false")
-    // should throw an exception as column stats is required for partition stats
-    try {
-      doWriteAndValidateDataAndPartitionStats(hudiOpts,
-        operation = DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL,
-        saveMode = SaveMode.Overwrite)
-      fail("Should have thrown exception")
-    } catch {
-      case e: HoodieException => assertTrue(e.getCause.getMessage.startsWith("Column stats partition must be enabled to generate partition stats."))
-    }
-  }
-
   /**
    * Test case to validate partition stats for a logical type column
    */
@@ -411,7 +393,6 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
       DataSourceWriteOptions.TABLE_TYPE.key -> HoodieTableType.MERGE_ON_READ.name(),
       HoodieMetadataConfig.ENABLE.key -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
       HoodieMetadataConfig.RECORD_INDEX_ENABLE_PROP.key -> "true")
 
     // First ingest.
@@ -483,7 +464,6 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
     val partitionStatsIndex = new PartitionStatsIndexSupport(
       spark, latestDf.schema, HoodieMetadataConfig.newBuilder()
         .enable(true)
-        .withMetadataIndexPartitionStats(true)
         .build(),
       metaClient)
     val partitionStats = partitionStatsIndex.loadColumnStatsIndexRecords(
@@ -495,7 +475,6 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
     val columnStatsIndex = new ColumnStatsIndexSupport(
       spark, latestDf.schema, HoodieMetadataConfig.newBuilder()
         .enable(true)
-        .withMetadataIndexPartitionStats(true)
         .build(),
       metaClient)
     val columnStats = columnStatsIndex
@@ -559,7 +538,6 @@ class TestPartitionStatsIndex extends PartitionStatsIndexTestBase {
       DataSourceWriteOptions.TABLE_TYPE.key -> HoodieTableType.MERGE_ON_READ.name(),
       HoodieMetadataConfig.ENABLE.key() -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key() -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key() -> "true",
       DataSourceReadOptions.ENABLE_DATA_SKIPPING.key -> "true")
 
     doWriteAndValidateDataAndPartitionStats(
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
index 6629b9523541..30274ab2b565 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsIndexWithSql.scala
@@ -687,7 +687,7 @@ class TestPartitionStatsIndexWithSql extends HoodieSparkSqlTestBase {
   }
 
   private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieTableFileSystemView = {
-    val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build()
+    val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build()
     val metadataTable = new HoodieBackedTableMetadata(new HoodieSparkEngineContext(spark.sparkContext),
       metaClient.getStorage, metadataConfig, metaClient.getBasePath.toString)
     new HoodieTableFileSystemView(
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsPruning.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsPruning.scala
index 2e07b702592d..3b5d3eb0320a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsPruning.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestPartitionStatsPruning.scala
@@ -47,8 +47,7 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
   def testMetadataPSISimple(testCase: ColumnStatsTestCase): Unit = {
     val metadataOpts = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true"
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true"
     )
 
     val commonOpts = Map(
@@ -77,8 +76,7 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
   def testMetadataColumnStatsIndex(testCase: ColumnStatsTestCase): Unit = {
     val metadataOpts = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true"
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true"
     )
 
     val commonOpts = Map(
@@ -125,7 +123,6 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     val metadataOpts1 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
       HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "c1,c2,c3,c5,c6,c7,c8" // ignore c4
     )
 
@@ -145,7 +142,6 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     val metadataOpts2 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
       HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "c1,c2,c3,c5,c7,c8" // ignore c4,c6
     )
 
@@ -163,8 +159,7 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     // disable cols stats
     val metadataOpts3 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "false"
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false"
     )
 
     // disable col stats
@@ -186,8 +181,7 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     val partitionCol : String = "c8"
     val metadataOpts = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "false"
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false"
     )
 
     val commonOpts = Map(
@@ -216,8 +210,7 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
 
     val metadataOpts0 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "false"
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "false"
     )
 
     // updates
@@ -245,7 +238,6 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     val metadataOpts1 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
       HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "c1,c2,c3,c4,c5,c6,c7,c8"
     )
 
@@ -342,7 +334,6 @@ class TestPartitionStatsPruning extends ColumnStatIndexTestBase {
     val metadataOpts1 = Map(
       HoodieMetadataConfig.ENABLE.key -> "true",
       HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key -> "true",
-      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key -> "true",
       HoodieMetadataConfig.COLUMN_STATS_INDEX_FOR_COLUMNS.key -> "c1,c2,c3,c4,c5,c6,c7,c8"
     )
 
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
index e7a75fbc2a6a..b4a98a58130b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/feature/index/TestExpressionIndex.scala
@@ -2065,7 +2065,7 @@ class TestExpressionIndex extends HoodieSparkSqlTestBase with SparkAdapterSuppor
       val lastCompletedInstant = metaClient.reloadActiveTimeline().getCommitsTimeline.filterCompletedInstants().lastInstant()
       val configBuilder = getWriteConfigBuilder(Map.empty, metaClient.getBasePath.toString)
       configBuilder.withMetadataConfig(HoodieMetadataConfig.newBuilder()
-        .withMetadataIndexColumnStats(false).withMetadataIndexPartitionStats(false).build())
+        .withMetadataIndexColumnStats(false).build())
       val writeClient = new SparkRDDWriteClient(new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext)), configBuilder.build())
       writeClient.rollback(lastCompletedInstant.get().requestedTime)
       writeClient.close()
@@ -2387,7 +2387,7 @@ class TestExpressionIndex extends HoodieSparkSqlTestBase with SparkAdapterSuppor
 
   private def getTableFileSystemView(metaClient: HoodieTableMetaClient): HoodieTableFileSystemView = {
     val engineContext = new HoodieSparkEngineContext(new JavaSparkContext(spark.sparkContext))
-    val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).withMetadataIndexPartitionStats(true).build()
+    val metadataConfig = HoodieMetadataConfig.newBuilder().enable(true).build()
     val metadataTable = new HoodieBackedTableMetadata(engineContext, metaClient.getStorage, metadataConfig, metaClient.getBasePath.toString)
     new HoodieTableFileSystemView(
       metadataTable,
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
index ec781d9bb616..7cc016463ead 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHoodieMetadataTableValidator.java
@@ -472,7 +472,6 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
       Dataset<Row> inserts = makeInsertDf("000", 5);
       inserts.write().format("hudi").options(writeOptions)
           .option(DataSourceWriteOptions.OPERATION().key(), WriteOperationType.BULK_INSERT.value())
-          .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true")
           .mode(SaveMode.Overwrite)
           .save(basePath);
       // validate MDT partition stats
@@ -481,7 +480,6 @@ public class TestHoodieMetadataTableValidator extends HoodieSparkClientTestBase
       Dataset<Row> updates = makeUpdateDf("001", 5);
       updates.write().format("hudi").options(writeOptions)
           .option(DataSourceWriteOptions.OPERATION().key(), WriteOperationType.UPSERT.value())
-          .option(HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key(), "true")
           .mode(SaveMode.Append)
           .save(basePath);
 


(hudi) 12/19: fix: Fixed the recovering method for the older versions where checksum is not present (#14148)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 256523affc3e2a3269488ac289dec8566d033aae
Author: Trivedhi <85...@users.noreply.github.com>
AuthorDate: Mon Oct 27 08:49:43 2025 +0530

    fix: Fixed the recovering method for the older versions where checksum is not present (#14148)
---
 .../org/apache/hudi/common/util/ConfigUtils.java   | 15 ++++++++++--
 .../apache/hudi/common/util/TestConfigUtils.java   | 27 ++++++++++++++++++++++
 .../hudi/common/table/TestHoodieTableConfig.java   | 13 ++++++++---
 3 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
index 0aa9e979e9da..d480d56d83dc 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/ConfigUtils.java
@@ -59,6 +59,7 @@ import static org.apache.hudi.common.config.HoodieCommonConfig.DISK_MAP_BITCASK_
 import static org.apache.hudi.common.config.HoodieCommonConfig.SPILLABLE_DISK_MAP_TYPE;
 import static org.apache.hudi.common.config.HoodieMemoryConfig.MAX_MEMORY_FOR_MERGE;
 import static org.apache.hudi.common.config.HoodieMemoryConfig.SPILLABLE_MAP_BASE_PATH;
+import static org.apache.hudi.common.table.HoodieTableConfig.NAME;
 import static org.apache.hudi.common.table.HoodieTableConfig.TABLE_CHECKSUM;
 import static org.apache.hudi.keygen.constant.KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED;
 
@@ -705,6 +706,16 @@ public class ConfigUtils {
     }
   }
 
+  public static boolean isPropertiesInvalid(TypedProperties props) {
+    // For older versions if checksum is not present, table name needs to be present to generate the checksum
+    if (!props.containsKey(TABLE_CHECKSUM.key())) {
+      return !props.containsKey(NAME.key());
+    }
+
+    // If checkSum is present we need to validate
+    return !HoodieTableConfig.validateChecksum(props);
+  }
+
   public static void recoverIfNeeded(HoodieStorage storage, StoragePath cfgPath,
                                      StoragePath backupCfgPath) throws IOException {
     boolean needCopy = false;
@@ -714,7 +725,7 @@ public class ConfigUtils {
       TypedProperties props = new TypedProperties();
       try (InputStream in = storage.open(cfgPath)) {
         props.load(in);
-        if (!props.containsKey(TABLE_CHECKSUM.key()) || !HoodieTableConfig.validateChecksum(props)) {
+        if (isPropertiesInvalid(props)) {
           // the cfg file is invalid
           storage.deleteFile(cfgPath);
           needCopy = true;
@@ -727,7 +738,7 @@ public class ConfigUtils {
       try (InputStream backupStream = new ByteArrayInputStream(bytes)) {
         TypedProperties backupProps = new TypedProperties();
         backupProps.load(backupStream);
-        if (!backupProps.containsKey(TABLE_CHECKSUM.key()) || !HoodieTableConfig.validateChecksum(backupProps)) {
+        if (isPropertiesInvalid(backupProps)) {
           // need to delete the backup as anyway reads will also fail
           // subsequent writes will recover and update
           storage.deleteFile(backupCfgPath);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java
index 5bf1dd2ac958..ea2b4a1ef85f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestConfigUtils.java
@@ -427,4 +427,31 @@ public class TestConfigUtils {
     assertEquals(1, props.size());
     assertEquals("overwrite", props.get("strategy"));
   }
+
+  @Test
+  void testIsPropertiesInvalid() {
+    TypedProperties props = new TypedProperties();
+    // Case-1 : Empty properties - should be invalid
+    assertTrue(ConfigUtils.isPropertiesInvalid(props));
+
+    // Case-2 : Valid properties with valid checksum
+    props.setProperty(HoodieTableConfig.NAME.key(), "test_db.test_table");
+    props.setProperty(HoodieTableConfig.TYPE.key(), "COPY_ON_WRITE");
+    props.setProperty(HoodieTableConfig.VERSION.key(), "6");
+    props.setProperty(HoodieTableConfig.TABLE_CHECKSUM.key(), String.valueOf(HoodieTableConfig.generateChecksum(props)));
+
+    assertFalse(ConfigUtils.isPropertiesInvalid(props));
+
+    // Case-3 : Invalid checksum
+    props.setProperty(HoodieTableConfig.TABLE_CHECKSUM.key(), "0");
+    assertTrue(ConfigUtils.isPropertiesInvalid(props));
+
+    // Case-4: without checksum property, valid properties
+    props.remove(HoodieTableConfig.TABLE_CHECKSUM.key());
+    assertFalse(ConfigUtils.isPropertiesInvalid(props));
+
+    // Case-5: without checksum property, invalid properties
+    props.remove(HoodieTableConfig.NAME.key());
+    assertTrue(ConfigUtils.isPropertiesInvalid(props));
+  }
 }
\ No newline at end of file
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
index a59ae8c5c3f2..b240bb2dfabd 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/table/TestHoodieTableConfig.java
@@ -21,6 +21,7 @@ package org.apache.hudi.common.table;
 import org.apache.hudi.common.config.ConfigProperty;
 import org.apache.hudi.common.config.HoodieConfig;
 import org.apache.hudi.common.config.RecordMergeMode;
+import org.apache.hudi.common.config.TypedProperties;
 import org.apache.hudi.common.model.AWSDmsAvroPayload;
 import org.apache.hudi.common.model.DefaultHoodieRecordPayload;
 import org.apache.hudi.common.model.EventTimeAvroPayload;
@@ -50,6 +51,7 @@ import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.util.Arrays;
 import java.util.Collections;
@@ -257,10 +259,15 @@ class TestHoodieTableConfig extends HoodieCommonTestHarness {
     assertEquals(7, config.getProps().size());
 
     // 2. Backup properties file is also invalid
+    try (InputStream in = storage.open(cfgPath);
+            OutputStream out = storage.create(backupCfgPath)) {
+      TypedProperties props = new TypedProperties();
+      props.load(in);
+      // Keeping the invalid checksum
+      props.setProperty(TABLE_CHECKSUM.key(), "000000000");
+      props.store(out, "");
+    }
     storage.deleteFile(cfgPath);
-    // create the empty files
-    storage.create(cfgPath);
-    storage.create(backupCfgPath);
     assertThrows(IOException.class, () -> recoverIfNeeded(storage, cfgPath, backupCfgPath));
     assertFalse(storage.exists(backupCfgPath));
     assertFalse(storage.exists(cfgPath));


(hudi) 10/19: fix: updating error messages thrown to end users (#14115)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 90a886a153ade08461e567a1c8f05de199affa2e
Author: Vamshi Krishna Kyatham <68...@users.noreply.github.com>
AuthorDate: Sun Oct 26 07:18:29 2025 -0700

    fix: updating error messages thrown to end users (#14115)
    
    
    ---------
    
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../hudi/common/table/HoodieTableVersion.java      |  9 ++++++++-
 .../hudi/io/hadoop/HoodieBaseParquetWriter.java    |  4 ++++
 .../main/java/org/apache/hudi/DataSourceUtils.java | 15 +++++++++++++-
 .../hudi/hive/util/IMetaStoreClientUtil.java       | 23 ++++++++++++++++++++--
 .../apache/hudi/sync/common/HoodieSyncClient.java  | 19 ++++++++++++++++--
 .../apache/hudi/utilities/sources/InputBatch.java  |  7 ++++++-
 .../deltastreamer/TestHoodieDeltaStreamer.java     |  6 +++++-
 .../hudi/utilities/sources/TestInputBatch.java     |  6 +++++-
 8 files changed, 80 insertions(+), 9 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
index 08ec668f20b1..945ed2366607 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
@@ -78,7 +78,14 @@ public enum HoodieTableVersion {
   public static HoodieTableVersion fromVersionCode(int versionCode) {
     return Arrays.stream(HoodieTableVersion.values())
         .filter(v -> v.versionCode == versionCode).findAny()
-        .orElseThrow(() -> new HoodieException("Unknown table versionCode:" + versionCode));
+        .orElseThrow(() -> new HoodieException(
+            String.format(
+                "Unsupported table version code: %d.%n%n"
+                    + "This table version is not recognized by this Hudi version.%n%n"
+                    + "This can happen if:%n"
+                    + "  (1) The table was created with a newer version of Hudi (upgrade your readers),%n"
+                    + "  (2) The table metadata is corrupted.%n%n"
+                    + "See: https://hudi.apache.org/docs/migration_guide for more information", versionCode)));
   }
 
   public static HoodieTableVersion fromReleaseVersion(String releaseVersion) {
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
index ed20f0bca114..24ad77a6c135 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
@@ -150,6 +150,10 @@ public abstract class HoodieBaseParquetWriter<R> implements Closeable {
     writtenRecordCount.incrementAndGet();
   }
 
+  private static boolean isRequiredFieldNullError(String errorMessage) {
+    return errorMessage.contains("null") && errorMessage.contains("required");
+  }
+
   protected long getWrittenRecordCount() {
     return writtenRecordCount.get();
   }
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java
index db32fb13f6b7..fe05c4281761 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/DataSourceUtils.java
@@ -323,7 +323,20 @@ public class DataSourceUtils {
       if (totalErroredRecords > 0) {
         hasErrored.set(true);
         ValidationUtils.checkArgument(writeStatusesOpt.isPresent(), "RDD <WriteStatus> expected to be present when there are errors");
-        LOG.error("{} failed with errors", writeOperationType);
+        long errorCount = HoodieJavaRDD.getJavaRDD(writeStatusesOpt.get())
+            .filter(WriteStatus::hasErrors)
+            .count();
+
+        String errorSummary = String.format(
+            "%s operation failed with %d error(s).%n%n"
+                + "Total write statuses with errors: %d%n%n"
+                + "Check the driver logs for error stacktraces which provide more information on the failure.",
+            writeOperationType,
+            totalErroredRecords,
+            errorCount);
+
+        LOG.error(errorSummary);
+
         if (LOG.isTraceEnabled()) {
           LOG.trace("Printing out the top 100 errors");
 
diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java
index c2cadd5b1159..875d7ab388ef 100644
--- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java
+++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/IMetaStoreClientUtil.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hudi.exception.HoodieException;
 
 import java.lang.reflect.InvocationTargetException;
 
@@ -39,8 +40,26 @@ public class IMetaStoreClientUtil {
     try {
       metaStoreClient = ((Hive) Hive.class.getMethod("getWithoutRegisterFns", HiveConf.class).invoke(null, hiveConf)).getMSC();
     } catch (NoSuchMethodException | IllegalAccessException | IllegalArgumentException
-      | InvocationTargetException ex) {
-      metaStoreClient = Hive.get(hiveConf).getMSC();
+        | InvocationTargetException ex) {
+      try {
+        metaStoreClient = Hive.get(hiveConf).getMSC();
+      } catch (RuntimeException e) {
+        if (e.getMessage() != null && e.getMessage().contains("not org.apache.hudi.org.apache.hadoop")) {
+          throw new HoodieException(
+              String.format(
+                  "Hive Metastore compatibility issue detected. This usually happens due to:%n"
+                      + "  (1) Hive version mismatch,%n"
+                      + "  (2) Conflicting Hive libraries in classpath,%n"
+                      + "  (3) Incompatible hudi-spark-bundle version.%n%n"
+                      + "To resolve:%n"
+                      + "  - For Hive 2.x use hudi-spark-bundle with 'hive2' classifier,%n"
+                      + "  - For Hive 3.x use hudi-spark-bundle with 'hive3' classifier,%n"
+                      + "  - Ensure no conflicting Hive jars in Spark classpath.%n%n"
+                      + "Check: https://hudi.apache.org/docs/syncing_metastore.%n%n"
+                      + "Technical details: %s", e.getMessage()), e);
+        }
+        throw e;
+      }
     }
     return metaStoreClient;
   }
diff --git a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
index be5b76764783..c696ba222c64 100644
--- a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
+++ b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncClient.java
@@ -112,7 +112,7 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
     try {
       return tableSchemaResolver.getTableParquetSchema();
     } catch (Exception e) {
-      throw new HoodieSyncException("Failed to read schema from storage.", e);
+      throw new HoodieSyncException(buildSchemaReadErrorMessage(e), e);
     }
   }
 
@@ -121,10 +121,25 @@ public abstract class HoodieSyncClient implements HoodieMetaSyncOperations, Auto
     try {
       return tableSchemaResolver.getTableParquetSchema(includeMetadataField);
     } catch (Exception e) {
-      throw new HoodieSyncException("Failed to read schema from storage.", e);
+      throw new HoodieSyncException(buildSchemaReadErrorMessage(e), e);
     }
   }
 
+  private String buildSchemaReadErrorMessage(Exception e) {
+    String errorMessage = e.getMessage() != null ? e.getMessage() : e.getClass().getName();
+    if (e instanceof java.io.FileNotFoundException) {
+      return String.format(
+          "Cannot read Hudi table schema.%n%n"
+              + "Required data file missing .%n%n"
+              + "This indicates:%n"
+              + "  1. Aggressive cleaner retention compared to query run times\n"
+              + "  2. Manual file deletions (timeline files or data files)\n"
+              + "  3. Concurrent writers without proper locking or configurations set\n\n"
+              + "Original error: %s", errorMessage);
+    }
+    return String.format("Failed to read schema from storage.%nError: %s", errorMessage);
+  }
+
   /**
    * Gets all relative partitions paths in the Hudi table on storage.
    *
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
index 54879f72b7bb..32f11cb5b818 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/InputBatch.java
@@ -63,7 +63,12 @@ public class InputBatch<T> {
 
   public SchemaProvider getSchemaProvider() {
     if (batch.isPresent() && schemaProvider == null) {
-      throw new HoodieException("Please provide a valid schema provider class!");
+      throw new HoodieException(
+          "Schema provider is required for this operation and for the source of interest. "
+              + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. "
+              + "Based on the schema provider class chosen, additional configs might be required. "
+              + "For eg, if you choose 'org.apache.hudi.utilities.schema.SchemaRegistryProvider', "
+              + "you may need to set configs like 'hoodie.streamer.schemaprovider.registry.url'.");
     }
     return Option.ofNullable(schemaProvider).orElseGet(NullSchemaProvider::getInstance);
   }
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 89781741e39f..4d6729bd2da4 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -2102,7 +2102,11 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
       new HoodieDeltaStreamer(cfg, jsc, fs, hiveServer.getHiveConf()).sync();
     }, "Should error out when schema provider is not provided");
     LOG.debug("Expected error during reading data from source ", e);
-    assertTrue(e.getMessage().contains("Please provide a valid schema provider class!"));
+    assertTrue(e.getMessage().contains("Schema provider is required for this operation and for the source of interest. "
+        + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. "
+        + "Based on the schema provider class chosen, additional configs might be required. "
+        + "For eg, if you choose 'org.apache.hudi.utilities.schema.SchemaRegistryProvider', "
+        + "you may need to set configs like 'hoodie.streamer.schemaprovider.registry.url'."));
   }
 
   @Test
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java
index d5d03e246959..f7bb36076a7a 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/sources/TestInputBatch.java
@@ -36,7 +36,11 @@ public class TestInputBatch {
   public void getSchemaProviderShouldThrowException() {
     final InputBatch<String> inputBatch = new InputBatch<>(Option.of("foo"), (String) null, null);
     Throwable t = assertThrows(HoodieException.class, inputBatch::getSchemaProvider);
-    assertEquals("Please provide a valid schema provider class!", t.getMessage());
+    assertEquals("Schema provider is required for this operation and for the source of interest. "
+        + "Please set '--schemaprovider-class' in the top level HoodieStreamer config for the source of interest. "
+        + "Based on the schema provider class chosen, additional configs might be required. "
+        + "For eg, if you choose 'org.apache.hudi.utilities.schema.SchemaRegistryProvider', "
+        + "you may need to set configs like 'hoodie.streamer.schemaprovider.registry.url'.", t.getMessage());
   }
 
   @Test


(hudi) 18/19: fix: Fix upgrade handling for MySqlDebeziumAvroPayload with deltastreamer (#14159)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 72f4452981eb9299cbb84508570f9e702622e341
Author: Lokesh Jain <lj...@apache.org>
AuthorDate: Tue Oct 28 07:50:23 2025 +0530

    fix: Fix upgrade handling for MySqlDebeziumAvroPayload with deltastreamer (#14159)
    
    
    ---------
    
    Co-authored-by: Lokesh Jain <lj...@Lokeshs-MacBook-Pro.local>
    Co-authored-by: Lokesh Jain <lj...@192.168.0.234>
    Co-authored-by: sivabalan <n....@gmail.com>
---
 .../table/upgrade/EightToNineUpgradeHandler.java   |  44 ++++-----
 .../upgrade/TestEightToNineUpgradeHandler.java     | 107 +++++++++------------
 .../hudi/common/model/HoodieRecordPayload.java     |  21 ++--
 .../model/debezium/MySqlDebeziumAvroPayload.java   |   5 +
 .../hudi/common/table/HoodieTableConfig.java       |   7 ++
 .../hudi/utilities/streamer/HoodieStreamer.java    |   1 -
 .../utilities/streamer/HoodieStreamerUtils.java    |   8 +-
 .../apache/hudi/utilities/streamer/StreamSync.java |  20 +++-
 .../deltastreamer/TestHoodieDeltaStreamer.java     |  28 ++++++
 .../streamer/TestHoodieStreamerUtils.java          |   5 +-
 10 files changed, 140 insertions(+), 106 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
index 788a6deb7171..1621cecfd65d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
@@ -48,8 +48,6 @@ import java.util.Set;
 
 import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_KEY;
 import static org.apache.hudi.common.model.DefaultHoodieRecordPayload.DELETE_MARKER;
-import static org.apache.hudi.common.model.debezium.DebeziumConstants.FLATTENED_FILE_COL_NAME;
-import static org.apache.hudi.common.model.debezium.DebeziumConstants.FLATTENED_POS_COL_NAME;
 import static org.apache.hudi.common.model.HoodieRecordMerger.COMMIT_TIME_BASED_MERGE_STRATEGY_UUID;
 import static org.apache.hudi.common.model.HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID;
 import static org.apache.hudi.common.model.HoodieRecordMerger.EVENT_TIME_BASED_MERGE_STRATEGY_UUID;
@@ -127,22 +125,22 @@ public class EightToNineUpgradeHandler implements UpgradeHandler {
           metaClient.getTableConfig().getTableVersion());
     }
     // Handle merge mode config.
-    reconcileMergeModeConfig(tablePropsToAdd, tablePropsToRemove, tableConfig);
+    reconcileMergeModeConfig(tablePropsToAdd, tablePropsToRemove, tableConfig, config);
     // Handle partial update mode config.
-    reconcilePartialUpdateModeConfig(tablePropsToAdd, tableConfig);
+    reconcilePartialUpdateModeConfig(tablePropsToAdd, tableConfig, config);
     // Handle merge properties config.
     reconcileMergePropertiesConfig(tablePropsToAdd, tableConfig, config);
     // Handle payload class configs.
-    reconcilePayloadClassConfig(tablePropsToAdd, tablePropsToRemove, tableConfig);
+    reconcilePayloadClassConfig(tablePropsToAdd, tablePropsToRemove, tableConfig, config);
     // Handle ordering fields config.
-    reconcileOrderingFieldsConfig(tablePropsToAdd, tablePropsToRemove, tableConfig);
+    reconcileOrderingFieldsConfig(tablePropsToAdd, tablePropsToRemove, tableConfig, config);
     return new UpgradeDowngrade.TableConfigChangeSet(tablePropsToAdd, tablePropsToRemove);
   }
 
-  private void reconcileMergeModeConfig(Map<ConfigProperty, String> tablePropsToAdd,
-                                        Set<ConfigProperty> tablePropsToRemove,
-                                        HoodieTableConfig tableConfig) {
-    String payloadClass = tableConfig.getPayloadClass();
+  private void reconcileMergeModeConfig(Map<ConfigProperty, String> tablePropsToAdd, Set<ConfigProperty> tablePropsToRemove,
+                                        HoodieTableConfig tableConfig, HoodieWriteConfig config) {
+    String payloadClass = tableConfig.getPayloadClassIfPresent()
+        .orElse(config.getPayloadClass());
     RecordMergeMode mergeMode = tableConfig.getRecordMergeMode();
     if (mergeMode != RecordMergeMode.CUSTOM) {
       // For commit time or event time based table, remove merge strategy id.
@@ -162,10 +160,10 @@ public class EightToNineUpgradeHandler implements UpgradeHandler {
     // else: No op, which means merge strategy id and merge mode are not changed.
   }
 
-  private void reconcilePayloadClassConfig(Map<ConfigProperty, String> tablePropsToAdd,
-                                           Set<ConfigProperty> tablePropsToRemove,
-                                           HoodieTableConfig tableConfig) {
-    String payloadClass = tableConfig.getPayloadClass();
+  private void reconcilePayloadClassConfig(Map<ConfigProperty, String> tablePropsToAdd, Set<ConfigProperty> tablePropsToRemove,
+                                           HoodieTableConfig tableConfig, HoodieWriteConfig config) {
+    String payloadClass = tableConfig.getPayloadClassIfPresent()
+        .orElse(config.getPayloadClass());
     if (StringUtils.isNullOrEmpty(payloadClass)) {
       return;
     }
@@ -176,8 +174,9 @@ public class EightToNineUpgradeHandler implements UpgradeHandler {
   }
 
   private void reconcilePartialUpdateModeConfig(Map<ConfigProperty, String> tablePropsToAdd,
-                                                HoodieTableConfig tableConfig) {
-    String payloadClass = tableConfig.getPayloadClass();
+                                                HoodieTableConfig tableConfig, HoodieWriteConfig config) {
+    String payloadClass = tableConfig.getPayloadClassIfPresent()
+        .orElse(config.getPayloadClass());
     if (StringUtils.isNullOrEmpty(payloadClass)) {
       return;
     }
@@ -190,7 +189,8 @@ public class EightToNineUpgradeHandler implements UpgradeHandler {
   }
 
   private void reconcileMergePropertiesConfig(Map<ConfigProperty, String> tablePropsToAdd, HoodieTableConfig tableConfig, HoodieWriteConfig writeConfig) {
-    String payloadClass = tableConfig.getPayloadClass();
+    String payloadClass = tableConfig.getPayloadClassIfPresent()
+        .orElse(writeConfig.getPayloadClass());
     if (StringUtils.isNullOrEmpty(payloadClass)) {
       return;
     }
@@ -224,13 +224,13 @@ public class EightToNineUpgradeHandler implements UpgradeHandler {
     }
   }
 
-  private void reconcileOrderingFieldsConfig(Map<ConfigProperty, String> tablePropsToAdd,
-                                             Set<ConfigProperty> tablePropsToRemove,
-                                             HoodieTableConfig tableConfig) {
-    String payloadClass = tableConfig.getPayloadClass();
+  private void reconcileOrderingFieldsConfig(Map<ConfigProperty, String> tablePropsToAdd, Set<ConfigProperty> tablePropsToRemove,
+                                             HoodieTableConfig tableConfig, HoodieWriteConfig config) {
+    String payloadClass = tableConfig.getPayloadClassIfPresent()
+        .orElse(config.getPayloadClass());
     Option<String> orderingFieldsOpt;
     if (MySqlDebeziumAvroPayload.class.getName().equals(payloadClass)) {
-      orderingFieldsOpt = Option.of(FLATTENED_FILE_COL_NAME + "," + FLATTENED_POS_COL_NAME);
+      orderingFieldsOpt = Option.of(MySqlDebeziumAvroPayload.ORDERING_FIELDS);
     } else if (PostgresDebeziumAvroPayload.class.getName().equals(payloadClass)) {
       orderingFieldsOpt = Option.of(DebeziumConstants.FLATTENED_LSN_COL_NAME);
     } else {
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestEightToNineUpgradeHandler.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestEightToNineUpgradeHandler.java
index eb885f8be5ad..58946703091c 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestEightToNineUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestEightToNineUpgradeHandler.java
@@ -56,9 +56,11 @@ import org.mockito.MockedStatic;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Stream;
@@ -117,6 +119,7 @@ class TestEightToNineUpgradeHandler {
     when(table.getMetaClient()).thenReturn(metaClient);
     when(metaClient.getTableConfig()).thenReturn(tableConfig);
     when(config.autoUpgrade()).thenReturn(true);
+    when(config.getPayloadClass()).thenReturn(null);
 
     // Setup common mocks
     when(upgradeDowngradeHelper.getTable(config, context)).thenReturn(table);
@@ -125,6 +128,7 @@ class TestEightToNineUpgradeHandler {
     when(metaClient.getStorage()).thenReturn(storage);
     when(tableConfig.getTableVersion()).thenReturn(HoodieTableVersion.EIGHT);
     when(tableConfig.getOrderingFieldsStr()).thenReturn(Option.empty());
+    when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.empty());
 
     // Use a temp file for index definition path
     indexDefPath = new StoragePath(tempDir.resolve("index.json").toString());
@@ -143,81 +147,55 @@ class TestEightToNineUpgradeHandler {
   }
 
   static Stream<Arguments> payloadClassTestCases() {
-    return Stream.of(
-        Arguments.of(
-            DefaultHoodieRecordPayload.class.getName(),
-            "",
-            null,
-            null,
-            "DefaultHoodieRecordPayload"
-        ),
-        Arguments.of(
-            EventTimeAvroPayload.class.getName(),
-            "",
-            EVENT_TIME_ORDERING.name(),
-            null,
-            "EventTimeAvroPayload"
-        ),
-        Arguments.of(
-            OverwriteWithLatestAvroPayload.class.getName(),
-            "",
-            null,
-            null,
-            "OverwriteWithLatestAvroPayload"
-        ),
-        Arguments.of(
-            AWSDmsAvroPayload.class.getName(),
-            RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=Op,"
-                + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=D", // mergeProperties
-            COMMIT_TIME_ORDERING.name(),
-            null,
-            "AWSDmsAvroPayload"
-        ),
-        Arguments.of(
-            PostgresDebeziumAvroPayload.class.getName(),
-            RECORD_MERGE_PROPERTY_PREFIX + PARTIAL_UPDATE_UNAVAILABLE_VALUE + "=" + DEBEZIUM_UNAVAILABLE_VALUE + ","
-                + RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=_change_operation_type,"
-                + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=d",
-            EVENT_TIME_ORDERING.name(),
-            FILL_UNAVAILABLE.name(),
-            "PostgresDebeziumAvroPayload"
-        ),
-        Arguments.of(
-            PartialUpdateAvroPayload.class.getName(),
-            "",
-            EVENT_TIME_ORDERING.name(),
-            PartialUpdateMode.IGNORE_DEFAULTS.name(),
-            "PartialUpdateAvroPayload"
-        ),
-        Arguments.of(
-            MySqlDebeziumAvroPayload.class.getName(),
-            RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=_change_operation_type,"
-                + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=d",
-            EVENT_TIME_ORDERING.name(),
-            null,
-            "MySqlDebeziumAvroPayload"
-        ),
-        Arguments.of(
-            OverwriteNonDefaultsWithLatestAvroPayload.class.getName(),
-            "",
-            COMMIT_TIME_ORDERING.name(),
-            PartialUpdateMode.IGNORE_DEFAULTS.name(),
-            "OverwriteNonDefaultsWithLatestAvroPayload"
-        )
-    );
+    List<Arguments> arguments = new ArrayList<>();
+    arguments.addAll(getArguments(DefaultHoodieRecordPayload.class.getName(), "",
+        null, null, "DefaultHoodieRecordPayload"));
+    arguments.addAll(getArguments(EventTimeAvroPayload.class.getName(), "",
+        EVENT_TIME_ORDERING.name(), null, "EventTimeAvroPayload"));
+    arguments.addAll(getArguments(OverwriteWithLatestAvroPayload.class.getName(), "",
+        null, null, "OverwriteWithLatestAvroPayload"));
+    arguments.addAll(getArguments(AWSDmsAvroPayload.class.getName(), RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=Op,"
+            + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=D", // mergeProperties
+        COMMIT_TIME_ORDERING.name(), null, "AWSDmsAvroPayload"));
+    arguments.addAll(getArguments(PostgresDebeziumAvroPayload.class.getName(), RECORD_MERGE_PROPERTY_PREFIX + PARTIAL_UPDATE_UNAVAILABLE_VALUE + "=" + DEBEZIUM_UNAVAILABLE_VALUE + ","
+            + RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=_change_operation_type,"
+            + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=d",
+        EVENT_TIME_ORDERING.name(), FILL_UNAVAILABLE.name(), "PostgresDebeziumAvroPayload"));
+    arguments.addAll(getArguments(PartialUpdateAvroPayload.class.getName(), "",
+        EVENT_TIME_ORDERING.name(), PartialUpdateMode.IGNORE_DEFAULTS.name(), "PartialUpdateAvroPayload"));
+    arguments.addAll(getArguments(MySqlDebeziumAvroPayload.class.getName(), RECORD_MERGE_PROPERTY_PREFIX + DELETE_KEY + "=_change_operation_type,"
+            + RECORD_MERGE_PROPERTY_PREFIX + DELETE_MARKER + "=d",
+        EVENT_TIME_ORDERING.name(), null, "MySqlDebeziumAvroPayload"));
+    arguments.addAll(getArguments(OverwriteNonDefaultsWithLatestAvroPayload.class.getName(), "",
+        COMMIT_TIME_ORDERING.name(), PartialUpdateMode.IGNORE_DEFAULTS.name(), "OverwriteNonDefaultsWithLatestAvroPayload"));
+    return arguments.stream();
+  }
+
+  private static List<Arguments> getArguments(String payloadClassName, String expectedMergeProperties,
+                                              String expectedRecordMergeMode, String expectedPartialUpdateMode,
+                                              String testName) {
+    return Arrays.asList(
+        Arguments.of(payloadClassName, expectedMergeProperties,
+            expectedRecordMergeMode, expectedPartialUpdateMode, testName, true),
+        Arguments.of(payloadClassName, expectedMergeProperties,
+            expectedRecordMergeMode, expectedPartialUpdateMode, testName, false));
   }
 
   @ParameterizedTest(name = "testUpgradeWith{4}")
   @MethodSource("payloadClassTestCases")
   void testUpgradeWithPayloadClass(String payloadClassName, String expectedMergeProperties,
                                    String expectedRecordMergeMode, String expectedPartialUpdateMode,
-                                   String testName) {
+                                   String testName, boolean isPayloadClassConfiguredInTableConfig) {
     try (org.mockito.MockedStatic<UpgradeDowngradeUtils> utilities =
              org.mockito.Mockito.mockStatic(UpgradeDowngradeUtils.class)) {
       utilities.when(() -> UpgradeDowngradeUtils.rollbackFailedWritesAndCompact(
               any(), any(), any(), any(), anyBoolean(), any()))
           .thenAnswer(invocation -> null);
-      when(tableConfig.getPayloadClass()).thenReturn(payloadClassName);
+      if (isPayloadClassConfiguredInTableConfig) {
+        when(tableConfig.getPayloadClassIfPresent()).thenReturn(Option.ofNullable(payloadClassName));
+      } else {
+        when(config.getPayloadClass()).thenReturn(payloadClassName);
+      }
       when(tableConfig.getTableType()).thenReturn(HoodieTableType.MERGE_ON_READ);
       when(tableConfig.getRecordMergeStrategyId()).thenReturn(HoodieRecordMerger.CUSTOM_MERGE_STRATEGY_UUID);
       when(metaClient.getIndexMetadata()).thenReturn(Option.empty());
@@ -301,6 +279,7 @@ class TestEightToNineUpgradeHandler {
     if (payloadClass.equals(MySqlDebeziumAvroPayload.class.getName())) {
       assertTrue(propertiesToAdd.containsKey(HoodieTableConfig.ORDERING_FIELDS));
       assertEquals(FLATTENED_FILE_COL_NAME + "," + FLATTENED_POS_COL_NAME, propertiesToAdd.get(HoodieTableConfig.ORDERING_FIELDS));
+      assertTrue(propertiesToRemove.contains(HoodieTableConfig.PRECOMBINE_FIELD));
     } else if (payloadClass.equals(PostgresDebeziumAvroPayload.class.getName())) {
       assertEquals(FLATTENED_LSN_COL_NAME, propertiesToAdd.get(HoodieTableConfig.ORDERING_FIELDS));
     }
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java
index af838e31bade..45856dbf4f2e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/HoodieRecordPayload.java
@@ -184,19 +184,7 @@ public interface HoodieRecordPayload<T extends HoodieRecordPayload> extends Seri
   }
 
   static String getPayloadClassName(Properties props) {
-    Option<String> payloadOpt = getPayloadClassNameIfPresent(props);
-    if (payloadOpt.isPresent()) {
-      return payloadOpt.get();
-    }
-    // Note: starting from version 9, payload class is not necessary set, but
-    //       merge mode must exist. Therefore, we use merge mode to infer
-    //       the payload class for certain corner cases, like for MIT command.
-    if (ConfigUtils.containsConfigProperty(props, RECORD_MERGE_MODE)
-        && ConfigUtils.getStringWithAltKeys(props, RECORD_MERGE_MODE, StringUtils.EMPTY_STRING)
-        .equals(RecordMergeMode.COMMIT_TIME_ORDERING.name())) {
-      return OverwriteWithLatestAvroPayload.class.getName();
-    }
-    return HoodieTableConfig.getDefaultPayloadClassName();
+    return getPayloadClassNameIfPresent(props).orElse(HoodieTableConfig.getDefaultPayloadClassName());
   }
 
   // NOTE: PAYLOAD_CLASS_NAME is before LEGACY_PAYLOAD_CLASS_NAME to make sure
@@ -207,6 +195,13 @@ public interface HoodieRecordPayload<T extends HoodieRecordPayload> extends Seri
       payloadClassName = ConfigUtils.getStringWithAltKeys(props, PAYLOAD_CLASS_NAME);
     } else if (props.containsKey("hoodie.datasource.write.payload.class")) {
       payloadClassName = props.getProperty("hoodie.datasource.write.payload.class");
+    } else if (ConfigUtils.containsConfigProperty(props, RECORD_MERGE_MODE)
+        && ConfigUtils.getStringWithAltKeys(props, RECORD_MERGE_MODE, StringUtils.EMPTY_STRING)
+        .equals(RecordMergeMode.COMMIT_TIME_ORDERING.name())) {
+      // Note: starting from version 9, payload class is not necessary set, but
+      //       merge mode must exist. Therefore, we use merge mode to infer
+      //       the payload class for certain corner cases, like for MIT command.
+      payloadClassName = OverwriteWithLatestAvroPayload.class.getName();
     }
 
     // There could be tables written with payload class from com.uber.hoodie.
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java
index 46bf1b896ca2..aa3cf5eaea61 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/debezium/MySqlDebeziumAvroPayload.java
@@ -32,6 +32,9 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.Objects;
 
+import static org.apache.hudi.common.model.debezium.DebeziumConstants.FLATTENED_FILE_COL_NAME;
+import static org.apache.hudi.common.model.debezium.DebeziumConstants.FLATTENED_POS_COL_NAME;
+
 /**
  * Provides support for seamlessly applying changes captured via Debezium for MysqlDB.
  * <p>
@@ -48,6 +51,8 @@ public class MySqlDebeziumAvroPayload extends AbstractDebeziumAvroPayload {
 
   private static final Logger LOG = LoggerFactory.getLogger(MySqlDebeziumAvroPayload.class);
 
+  public static final String ORDERING_FIELDS = FLATTENED_FILE_COL_NAME + "," + FLATTENED_POS_COL_NAME;
+
   public MySqlDebeziumAvroPayload(GenericRecord record, Comparable orderingVal) {
     super(record, orderingVal);
   }
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
index 9b24f7d0c989..c5bcfbaff6f6 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
@@ -801,6 +801,13 @@ public class HoodieTableConfig extends HoodieConfig {
     return HoodieRecordPayload.getPayloadClassName(this);
   }
 
+  /**
+   * Read the payload class if present for HoodieRecords from the table properties.
+   */
+  public Option<String> getPayloadClassIfPresent() {
+    return HoodieRecordPayload.getPayloadClassNameIfPresent(this.getProps());
+  }
+
   public String getLegacyPayloadClass() {
     return getStringOrDefault(LEGACY_PAYLOAD_CLASS_NAME, EMPTY_STRING);
   }
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
index fda1b8b6391d..6b3c1b2f94a4 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamer.java
@@ -161,7 +161,6 @@ public class HoodieStreamer implements Serializable {
             cfg.recordMergeMode, cfg.payloadClassName, cfg.recordMergeStrategyId, cfg.sourceOrderingFields,
             HoodieTableVersion.fromVersionCode(ConfigUtils.getIntWithAltKeys(this.properties, HoodieWriteConfig.WRITE_TABLE_VERSION)));
     cfg.recordMergeMode = mergingConfigs.getLeft();
-    cfg.payloadClassName = mergingConfigs.getMiddle();
     cfg.recordMergeStrategyId = mergingConfigs.getRight();
     if (cfg.initialCheckpointProvider != null && cfg.checkpoint == null) {
       InitialCheckPointProvider checkPointProvider =
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
index 2d6b27f13434..7ab24034b633 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/HoodieStreamerUtils.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieRecordPayload;
 import org.apache.hudi.common.model.HoodieSparkRecord;
 import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.read.DeleteContext;
 import org.apache.hudi.common.util.ConfigUtils;
 import org.apache.hudi.common.util.Either;
@@ -88,9 +89,10 @@ public class HoodieStreamerUtils {
    */
   public static Option<JavaRDD<HoodieRecord>> createHoodieRecords(HoodieStreamer.Config cfg, TypedProperties props, Option<JavaRDD<GenericRecord>> avroRDDOptional,
                                                                   SchemaProvider schemaProvider, HoodieRecord.HoodieRecordType recordType, boolean autoGenerateRecordKeys,
-                                                                  String instantTime, Option<BaseErrorTableWriter> errorTableWriter) {
+                                                                  String instantTime, Option<BaseErrorTableWriter> errorTableWriter, HoodieTableConfig tableConfig) {
     boolean shouldCombine = cfg.filterDupes || cfg.operation.equals(WriteOperationType.UPSERT);
-    boolean shouldUseOrderingField = shouldCombine && !StringUtils.isNullOrEmpty(cfg.sourceOrderingFields);
+    String orderingFieldsStr = tableConfig.getOrderingFieldsStr().orElse(cfg.sourceOrderingFields);
+    boolean shouldUseOrderingField = shouldCombine && !StringUtils.isNullOrEmpty(orderingFieldsStr);
     boolean shouldErrorTable = errorTableWriter.isPresent() && props.getBoolean(ERROR_ENABLE_VALIDATE_RECORD_CREATION.key(), ERROR_ENABLE_VALIDATE_RECORD_CREATION.defaultValue());
     boolean useConsistentLogicalTimestamp = ConfigUtils.getBooleanWithAltKeys(
         props, KeyGeneratorOptions.KEYGENERATOR_CONSISTENT_LOGICAL_TIMESTAMP_ENABLED);
@@ -129,7 +131,7 @@ public class HoodieStreamerUtils {
                   GenericRecord gr = isDropPartitionColumns(props) ? HoodieAvroUtils.removeFields(genRec, partitionColumns) : genRec;
                   boolean isDelete = AvroRecordContext.getFieldAccessorInstance().isDeleteRecord(gr, deleteContext);
                   Comparable orderingValue = shouldUseOrderingField
-                      ? OrderingValues.create(cfg.sourceOrderingFields.split(","),
+                      ? OrderingValues.create(orderingFieldsStr.split(","),
                          field -> (Comparable) HoodieAvroUtils.getNestedFieldVal(gr, field, false, useConsistentLogicalTimestamp))
                       : null;
                   HoodieRecord record = shouldUseOrderingField ? HoodieRecordUtils.createHoodieRecord(gr, orderingValue, hoodieKey, payloadClassName, requiresPayload, isDelete)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
index 5985abcf5423..fb55eb144cb4 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java
@@ -47,6 +47,8 @@ import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
 import org.apache.hudi.common.model.HoodieTableType;
 import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.model.debezium.DebeziumConstants;
+import org.apache.hudi.common.model.debezium.MySqlDebeziumAvroPayload;
 import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.table.HoodieTableMetaClient;
 import org.apache.hudi.common.table.HoodieTableVersion;
@@ -75,6 +77,7 @@ import org.apache.hudi.data.HoodieJavaRDD;
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.exception.HoodieMetaSyncException;
+import org.apache.hudi.exception.HoodieValidationException;
 import org.apache.hudi.hadoop.fs.HadoopFSUtils;
 import org.apache.hudi.hive.HiveSyncConfig;
 import org.apache.hudi.hive.HiveSyncTool;
@@ -951,10 +954,11 @@ public class StreamSync implements Serializable, Closeable {
       BaseDatasetBulkInsertCommitActionExecutor executor = new HoodieStreamerDatasetBulkInsertCommitActionExecutor(hoodieWriteConfig, writeClient, instantTime);
       writeClientWriteResult = new WriteClientWriteResult(executor.execute(df, !HoodieStreamerUtils.getPartitionColumns(props).isEmpty()).getWriteStatuses());
     } else {
+      metaClient = HoodieTableMetaClient.reload(metaClient);
       TypedProperties mergeProps = ConfigUtils.getMergeProps(props, metaClient.getTableConfig());
       HoodieRecordType recordType = createRecordMerger(mergeProps).getRecordType();
       Option<JavaRDD<HoodieRecord>> recordsOption = HoodieStreamerUtils.createHoodieRecords(cfg, mergeProps, inputBatch.getBatch(), inputBatch.getSchemaProvider(),
-          recordType, autoGenerateRecordKeys, instantTime, errorTableWriter);
+          recordType, autoGenerateRecordKeys, instantTime, errorTableWriter, metaClient.getTableConfig());
       JavaRDD<HoodieRecord> records = recordsOption.orElseGet(() -> hoodieSparkContext.emptyRDD());
       // filter dupes if needed
       if (cfg.filterDupes) {
@@ -1149,6 +1153,20 @@ public class StreamSync implements Serializable, Closeable {
     if (!StringUtils.isNullOrEmpty(cfg.recordMergeStrategyId)) {
       builder.withRecordMergeStrategyId(cfg.recordMergeStrategyId);
     }
+
+    if (metaClient != null) {
+      HoodieTableConfig tableConfig = metaClient.getTableConfig();
+      // After upgrade to table version 9 with MySqlDebeziumAvroPayload, ordering fields are changed from
+      // `_event_seq` to `_event_bin_file,_event_pos`. The logic here ensures that deltastreamer config is updated
+      // if it points to older ordering field `_event_seq`.
+      if (tableConfig.getTableVersion().greaterThanOrEquals(HoodieTableVersion.NINE) && tableConfig.getLegacyPayloadClass().equals(MySqlDebeziumAvroPayload.class.getCanonicalName())
+          && cfg.sourceOrderingFields.equals(DebeziumConstants.ADDED_SEQ_COL_NAME)) {
+        cfg.sourceOrderingFields = MySqlDebeziumAvroPayload.ORDERING_FIELDS;
+      } else if (tableConfig.getOrderingFieldsStr().isPresent() && !StringUtils.isNullOrEmpty(cfg.sourceOrderingFields)
+          && !tableConfig.getOrderingFieldsStr().orElse("").equals(cfg.sourceOrderingFields)) {
+        throw new HoodieValidationException(String.format("Configured ordering fields: %s do not match table ordering fields: %s", cfg.sourceOrderingFields, tableConfig.getOrderingFields()));
+      }
+    }
     HoodiePayloadConfig.Builder payloadConfigBuilder =
         HoodiePayloadConfig.newBuilder().withPayloadOrderingFields(cfg.sourceOrderingFields);
     // Payload class can be NULL.
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index 42f5aab7559a..a5c393af1a0a 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -74,6 +74,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
 import org.apache.hudi.config.metrics.HoodieMetricsConfig;
 import org.apache.hudi.exception.HoodieException;
 import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieValidationException;
 import org.apache.hudi.exception.TableNotFoundException;
 import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
 import org.apache.hudi.hadoop.fs.HadoopFSUtils;
@@ -2267,6 +2268,33 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
     UtilitiesTestBase.Helpers.deleteFileFromDfs(fs, tableBasePath);
   }
 
+  @ParameterizedTest
+  @EnumSource(HoodieTableType.class)
+  public void testDeltaStreamerFailureWithChangingOrderingFields(HoodieTableType tableType) throws Exception {
+    String tableBasePath = basePath + "/test_with_changing_ordering_fields";
+    HoodieDeltaStreamer.Config cfg =
+        TestHelpers.makeConfig(tableBasePath, WriteOperationType.BULK_INSERT);
+    cfg.tableType = tableType.name();
+    cfg.filterDupes = true;
+    cfg.sourceOrderingFields = "timestamp,rider";
+    cfg.recordMergeMode = RecordMergeMode.EVENT_TIME_ORDERING;
+    cfg.payloadClassName = DefaultHoodieRecordPayload.class.getName();
+    cfg.recordMergeStrategyId = HoodieRecordMerger.EVENT_TIME_BASED_MERGE_STRATEGY_UUID;
+
+    TestDataSource.recordInstantTime = Option.of("001");
+    new HoodieStreamer(cfg, jsc).sync();
+    assertRecordCount(1000, tableBasePath, sqlContext);
+    TestHelpers.assertCommitMetadata("00000", tableBasePath, 1);
+
+    // Change ordering fields in deltastreamer
+    Exception e = assertThrows(HoodieValidationException.class, () -> {
+      cfg.sourceOrderingFields = "timestamp";
+      TestDataSource.recordInstantTime = Option.of("002");
+      runStreamSync(cfg, false, 10, WriteOperationType.UPSERT);
+    });
+    assertTrue(e.getMessage().equals("Configured ordering fields: timestamp do not match table ordering fields: [timestamp, rider]"));
+  }
+
   private static long getNumUpdates(HoodieCommitMetadata metadata) {
     return metadata.getPartitionToWriteStats().values().stream()
         .flatMap(Collection::stream)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerUtils.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerUtils.java
index 1c40eedf588c..eae9c0f0308d 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerUtils.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/streamer/TestHoodieStreamerUtils.java
@@ -23,6 +23,7 @@ import org.apache.hudi.common.config.TypedProperties;
 import org.apache.hudi.common.model.DefaultHoodieRecordPayload;
 import org.apache.hudi.common.model.HoodieRecord;
 import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
+import org.apache.hudi.common.table.HoodieTableConfig;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.exception.HoodieKeyException;
 import org.apache.hudi.exception.HoodieRecordCreationException;
@@ -108,7 +109,7 @@ public class TestHoodieStreamerUtils extends UtilitiesTestBase {
       doNothing().when(errorTableWriter.get()).addErrorEvents(errorEventCaptor.capture());
     }
     Option<JavaRDD<HoodieRecord>> recordOpt = HoodieStreamerUtils.createHoodieRecords(cfg, props, Option.of(recordRdd),
-        new SimpleSchemaProvider(jsc, schema, props), recordType, false, "000", errorTableWriter);
+        new SimpleSchemaProvider(jsc, schema, props), recordType, false, "000", errorTableWriter, new HoodieTableConfig());
 
     if (errorTableWriter.isPresent()) {
       assertEquals(0, errorEventCaptor.getValue().collect().size());
@@ -153,7 +154,7 @@ public class TestHoodieStreamerUtils extends UtilitiesTestBase {
       doNothing().when(errorTableWriter.get()).addErrorEvents(errorEventCaptor.capture());
     }
     Option<JavaRDD<HoodieRecord>> records = HoodieStreamerUtils.createHoodieRecords(cfg, props, Option.of(recordRdd),
-        schemaProvider, recordType, false, "000", errorTableWriter);
+        schemaProvider, recordType, false, "000", errorTableWriter, new HoodieTableConfig());
     assertTrue(records.isPresent());
 
     if (enableErrorTableWriter) {


(hudi) 13/19: refactor: Add required setter methods for Flink-CDC dev (#14150)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit c2fd476a8bb240879419e33fcf51a7ef5cce360a
Author: voonhous <vo...@gmail.com>
AuthorDate: Mon Oct 27 11:21:24 2025 +0800

    refactor: Add required setter methods for Flink-CDC dev (#14150)
---
 .../org/apache/hudi/sink/StreamWriteOperatorCoordinator.java   |  6 +++---
 .../apache/hudi/sink/common/AbstractStreamWriteFunction.java   | 10 +++++++++-
 .../main/java/org/apache/hudi/sink/event/Correspondent.java    |  8 ++++++++
 3 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
index 04506a3127fd..68bb54f8a1ee 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
@@ -127,7 +127,7 @@ public class StreamWriteOperatorCoordinator
   /**
    * Coordinator context.
    */
-  private final Context context;
+  protected final Context context;
 
   /**
    * Gateways for sending events to sub-tasks.
@@ -167,12 +167,12 @@ public class StreamWriteOperatorCoordinator
   /**
    * A single-thread executor to handle all the write metadata events.
    */
-  private NonThrownExecutor executor;
+  protected NonThrownExecutor executor;
 
   /**
    * A single-thread executor to handle the instant time request.
    */
-  private NonThrownExecutor instantRequestExecutor;
+  protected NonThrownExecutor instantRequestExecutor;
 
   /**
    * A single-thread executor to handle asynchronous hive sync.
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/common/AbstractStreamWriteFunction.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/common/AbstractStreamWriteFunction.java
index 4d101d649f8b..deedfbba8b0e 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/common/AbstractStreamWriteFunction.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/common/AbstractStreamWriteFunction.java
@@ -88,7 +88,7 @@ public abstract class AbstractStreamWriteFunction<I>
   /**
    * Correspondent to request the instant time.
    */
-  private transient Correspondent correspondent;
+  protected transient Correspondent correspondent;
 
   /**
    * Gateway to send operator events to the operator coordinator.
@@ -183,10 +183,18 @@ public abstract class AbstractStreamWriteFunction<I>
     this.correspondent = correspondent;
   }
 
+  public Correspondent getCorrespondent() {
+    return correspondent;
+  }
+
   public void setOperatorEventGateway(OperatorEventGateway operatorEventGateway) {
     this.eventGateway = operatorEventGateway;
   }
 
+  public OperatorEventGateway getOperatorEventGateway() {
+    return eventGateway;
+  }
+
   // -------------------------------------------------------------------------
   //  Utilities
   // -------------------------------------------------------------------------
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/event/Correspondent.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/event/Correspondent.java
index e23e7e851278..e4dd405bd40e 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/event/Correspondent.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/event/Correspondent.java
@@ -71,6 +71,14 @@ public class Correspondent {
     }
   }
 
+  public OperatorID getOperatorID() {
+    return operatorID;
+  }
+
+  public TaskOperatorEventGateway getGateway() {
+    return gateway;
+  }
+
   /**
    * A request for instant time with a given checkpoint id.
    */


(hudi) 04/19: fix: Handle missing valueType column after upgrade (#14105)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit d424808c6802dc405f2789a1d0da82e10b2bf6ca
Author: Lin Liu <14...@users.noreply.github.com>
AuthorDate: Thu Oct 23 09:56:44 2025 -0700

    fix: Handle missing valueType column after upgrade   (#14105)
    
    
    ---------
    
    Co-authored-by: Lokesh Jain <lj...@192.168.0.234>
---
 .../java/org/apache/hudi/stats/ValueMetadata.java  |   5 +
 .../hudi/table/upgrade/TestUpgradeDowngrade.java   |  17 ++-
 .../cow-tables/hudi-v8-table-cow.zip               | Bin 0 -> 380008 bytes
 .../generate-fixtures.sh                           |   3 +
 .../scala-templates/generate-fixture-cow.scala     | 153 +++++++++++++++++++++
 5 files changed, 171 insertions(+), 7 deletions(-)

diff --git a/hudi-common/src/main/java/org/apache/hudi/stats/ValueMetadata.java b/hudi-common/src/main/java/org/apache/hudi/stats/ValueMetadata.java
index 5c1e087b326f..cebcb2aac3ba 100644
--- a/hudi-common/src/main/java/org/apache/hudi/stats/ValueMetadata.java
+++ b/hudi-common/src/main/java/org/apache/hudi/stats/ValueMetadata.java
@@ -214,6 +214,11 @@ public class ValueMetadata implements Serializable {
       throw new IllegalStateException("ColumnStatsMetadata is null. Handling should happen in the caller.");
     }
 
+    // This may happen when the record is from old table versions.
+    if (!columnStatsRecord.hasField(COLUMN_STATS_FIELD_VALUE_TYPE)) {
+      return V1EmptyMetadata.get();
+    }
+
     GenericRecord valueTypeInfo = (GenericRecord) columnStatsRecord.get(COLUMN_STATS_FIELD_VALUE_TYPE);
     if (valueTypeInfo == null) {
       return V1EmptyMetadata.get();
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 5407cd8fda87..5ae4fc222a1c 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -82,6 +82,8 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
       return "/upgrade-downgrade-fixtures/complex-keygen-tables/";
     } else if (suffix.contains("mor")) {
       return "/upgrade-downgrade-fixtures/mor-tables/";
+    } else if (suffix.contains("cow")) {
+      return "/upgrade-downgrade-fixtures/cow-tables/";
     } else if (suffix.contains("payload")) {
       return "/upgrade-downgrade-fixtures/payload-tables/";
     } else {
@@ -92,12 +94,12 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
   @Disabled
   @ParameterizedTest
   @MethodSource("upgradeDowngradeVersionPairs")
-  public void testUpgradeOrDowngrade(HoodieTableVersion fromVersion, HoodieTableVersion toVersion) throws Exception {
+  public void testUpgradeOrDowngrade(HoodieTableVersion fromVersion, HoodieTableVersion toVersion, String suffix) throws Exception {
     boolean isUpgrade = fromVersion.lesserThan(toVersion);
     String operation = isUpgrade ? "upgrade" : "downgrade";
     LOG.info("Testing {} from version {} to {}", operation, fromVersion, toVersion);
     
-    HoodieTableMetaClient originalMetaClient = loadFixtureTable(fromVersion, "-mor");
+    HoodieTableMetaClient originalMetaClient = loadFixtureTable(fromVersion, suffix);
     assertEquals(fromVersion, originalMetaClient.getTableConfig().getTableVersion(),
         "Fixture table should be at expected version");
     
@@ -110,7 +112,7 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
     
     // Confirm that there are log files before rollback and compaction operations
     if (isRollbackAndCompactTransition(fromVersion, toVersion)) {
-      validateLogFilesCount(originalMetaClient, operation, true);
+      validateLogFilesCount(originalMetaClient, operation, suffix.equals("-mor"));
     }
     
     new UpgradeDowngrade(originalMetaClient, config, context(), SparkUpgradeDowngradeHelper.getInstance())
@@ -517,12 +519,13 @@ public class TestUpgradeDowngrade extends SparkClientFunctionalTestHarness {
   private static Stream<Arguments> upgradeDowngradeVersionPairs() {
     return Stream.of(
         // Upgrade test cases for six and greater
-        Arguments.of(HoodieTableVersion.SIX, HoodieTableVersion.EIGHT),   // V6 -> V8
-        Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.NINE),  // V8 -> V9
+        Arguments.of(HoodieTableVersion.SIX, HoodieTableVersion.EIGHT, "-mor"),   // V6 -> V8
+        Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.NINE, "-mor"),  // V8 -> V9
+        Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.NINE, "-cow"),  // V8 -> V9
 
         // Downgrade test cases til six
-        Arguments.of(HoodieTableVersion.NINE, HoodieTableVersion.EIGHT),  // V9 -> V8
-        Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.SIX)   // V8 -> V6
+        Arguments.of(HoodieTableVersion.NINE, HoodieTableVersion.EIGHT, "-mor"),  // V9 -> V8
+        Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.SIX, "-mor")   // V8 -> V6
     );
   }
 
diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/cow-tables/hudi-v8-table-cow.zip b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/cow-tables/hudi-v8-table-cow.zip
new file mode 100644
index 000000000000..54e4ff3240e1
Binary files /dev/null and b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/cow-tables/hudi-v8-table-cow.zip differ
diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/generate-fixtures.sh b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/generate-fixtures.sh
index 9b08d00881d0..ac5eff4e558d 100755
--- a/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/generate-fixtures.sh
+++ b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/generate-fixtures.sh
@@ -71,6 +71,9 @@ SCRIPT_SUFFIX="${SCRIPT_BASE#generate-fixture}"  # Remove generate-fixture prefi
 if [[ "$SCALA_SCRIPT_NAME" == *"mor"* ]]; then
     FIXTURES_DIR="$SCRIPT_DIR/mor-tables"
     echo "Using mor tables directory: $FIXTURES_DIR"
+elif [[ "$SCALA_SCRIPT_NAME" == *"cow"* ]]; then
+    FIXTURES_DIR="$SCRIPT_DIR/cow-tables"
+    echo "Using cow tables directory: $FIXTURES_DIR"
 elif [[ "$SCALA_SCRIPT_NAME" == *"complex-keygen"* ]]; then
     FIXTURES_DIR="$SCRIPT_DIR/complex-keygen-tables"
     echo "Using complex-keygen tables directory: $FIXTURES_DIR"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/scala-templates/generate-fixture-cow.scala b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/scala-templates/generate-fixture-cow.scala
new file mode 100644
index 000000000000..2c8e445f8a1f
--- /dev/null
+++ b/hudi-spark-datasource/hudi-spark/src/test/resources/upgrade-downgrade-fixtures/scala-templates/generate-fixture-cow.scala
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.spark.sql.SaveMode
+import org.apache.hudi.common.table.HoodieTableConfig
+import org.apache.hudi.DataSourceWriteOptions._
+import spark.implicits._
+
+val tableName = "${TABLE_NAME}"  // Backticks for SQL identifiers with hyphens
+val basePath = "${BASE_PATH}"
+
+println("Generating cow table with archival and clustering...")
+
+// Base cow settings used by all steps
+val cowConfig = Map(
+  "hoodie.compact.inline" -> "false",
+  "hoodie.clustering.inline" -> "true",
+  "hoodie.clustering.inline.max.commits" -> "4",
+  "hoodie.metadata.compact.max.delta.commits" -> "3",
+  "hoodie.keep.min.commits" -> "5",
+  "hoodie.keep.max.commits" -> "6",
+  "hoodie.cleaner.commits.retained" -> "5"
+)
+
+// Initial setup (table creation + file size settings for steps 1-2)
+val initialSetupConfig = Map(
+  HoodieTableConfig.PRECOMBINE_FIELD.key -> "ts",
+  RECORDKEY_FIELD.key -> "id",
+  PARTITIONPATH_FIELD.key -> "partition",
+  "hoodie.table.name" -> tableName,
+  "hoodie.datasource.write.table.type" -> "COPY_ON_WRITE",
+  "hoodie.parquet.max.file.size" -> "2048", // 2KB - very small files
+  "hoodie.parquet.small.file.limit" -> "1024", // 1KB threshold
+  "hoodie.clustering.plan.strategy.small.file.limit" -> "10240", // 10KB
+  "hoodie.clustering.plan.strategy.target.file.max.bytes" -> "10240" // 10KB target
+)
+
+// Archival settings
+val archivalConfig = Map(
+  "hoodie.archive.automatic" -> "true",
+  "hoodie.commits.archival.batch" -> "1",
+  "hoodie.archive.merge.files.batch.size" -> "1"
+)
+
+val initialData = Seq(
+  ("id1", "Alice", 1000L, "2023-01-01"),
+  ("id2", "Bob", 1001L, "2023-01-01"),
+  ("id3", "Charlie", 1002L, "2023-01-01"),
+  ("id4", "David", 1003L, "2023-01-02"),
+  ("id5", "Eve", 1004L, "2023-01-02")
+)
+
+val initialDf = initialData.toDF("id", "name", "ts", "partition")
+
+(initialDf.write.format("hudi").
+  options(initialSetupConfig ++ cowConfig).
+  option("hoodie.datasource.write.operation", "insert").
+  mode(SaveMode.Overwrite)).
+  save(basePath)
+
+
+println("Step 1: Initial data written as table")
+
+val moreData1 = Seq(
+  ("id6", "Frank", 2000L, "2023-01-01"),
+  ("id7", "Grace", 2001L, "2023-01-01")
+)
+
+((moreData1.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(initialSetupConfig ++ cowConfig).
+  option("hoodie.datasource.write.operation", "insert").
+  mode(SaveMode.Append))).
+  save(basePath)
+
+
+println("Step 2: Added more small files")
+
+val update1 = Seq(("id1", "Alice_v2", 3000L, "2023-01-01"))
+update1.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(cowConfig).
+  option("hoodie.datasource.write.operation", "upsert").
+  mode(SaveMode.Append).
+  save(basePath)
+
+
+println("Step 3: First update - will trigger COMPACTION after 3 delta commits (max.delta.commits=3)")
+
+val update2 = Seq(("id2", "Bob_v2", 4000L, "2023-01-01"))
+update2.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(cowConfig).
+  option("hoodie.datasource.write.operation", "upsert").
+  mode(SaveMode.Append).
+  save(basePath)
+
+
+println("Step 4: Second update - will trigger CLUSTERING after 4 commits (max.commits=4)")
+
+val finalData = Seq(("id8", "Final", 5000L, "2023-01-01"))
+finalData.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(cowConfig ++ archivalConfig).
+  option("hoodie.datasource.write.operation", "insert").
+  mode(SaveMode.Append).
+  save(basePath)
+
+
+println("Step 5: Insert - will trigger CLEANING (retained=5) and begin ARCHIVAL setup")
+
+val extraData = Seq(("id9", "Extra", 6000L, "2023-01-01"))
+extraData.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(cowConfig ++ archivalConfig).
+  option("hoodie.datasource.write.operation", "insert").
+  mode(SaveMode.Append).
+  save(basePath)
+
+println("Step 6: Extra insert - will trigger ARCHIVAL (keep.max.commits=6 exceeded)")
+
+val moreExtraData = Seq(("id10", "MoreExtra", 7000L, "2023-01-01"))
+moreExtraData.toDF("id", "name", "ts", "partition").write.format("hudi").
+  options(cowConfig ++ archivalConfig).
+  option("hoodie.datasource.write.operation", "insert").
+  mode(SaveMode.Append).
+  save(basePath)
+
+println("Step 7: More extra insert - ensures ARCHIVAL is completed")
+
+val deleteData = Seq(("id1", "Alice_v2", 9000L, "2023-01-01"))
+deleteData.toDF("id", "name", "ts", "partition").write.format("hudi").
+  option("hoodie.datasource.write.operation", "delete").
+  option("hoodie.compact.inline", "false"). // Disable compaction to keep log files uncompacted
+  option("hoodie.clustering.inline", "false").
+  option("hoodie.clean.automatic", "false").
+  option("hoodie.archive.automatic", "false").
+  mode(SaveMode.Append).
+  save(basePath)
+
+println("Step 8: Delete operation (creates uncompacted log files)")
+
+println(s"cow table fixture ${FIXTURE_NAME} generated!")
+System.exit(0)
\ No newline at end of file


(hudi) 09/19: fix: fix partition stats delete properly for downgrade from V9 to V8 (#14138)

Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.1.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit 83eac32573a86262acc9eeb4827995be29ad3103
Author: Vamshi Krishna Kyatham <68...@users.noreply.github.com>
AuthorDate: Fri Oct 24 22:51:48 2025 -0700

    fix: fix partition stats delete properly for downgrade from V9 to V8 (#14138)
---
 .../apache/hudi/client/BaseHoodieWriteClient.java  |   5 +-
 .../hudi/table/upgrade/UpgradeDowngradeUtils.java  |  14 ++-
 .../upgrade/TestNineToEightDowngradeHandler.java   | 120 +++++++++++++++++++++
 3 files changed, 136 insertions(+), 3 deletions(-)

diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index cb4c32049f1c..e5880bd29a54 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -1112,10 +1112,11 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
       HoodieTableMetaClient metaClient = table.getMetaClient();
       // For secondary index and expression index with wrong parameters, index definition for the MDT partition is
       // removed so that such indices are not recreated while initializing the writer.
+      // Also remove index definitions for col stats and partition stats when they are dropped (e.g., during downgrade).
       metadataPartitions.forEach(partition -> {
-        if (MetadataPartitionType.isExpressionOrSecondaryIndex(partition)) {
+        metaClient.getIndexForMetadataPartition(partition).ifPresent(indexDef -> {
           metaClient.deleteIndexDefinition(partition);
-        }
+        });
       });
 
       Option<HoodieTableMetadataWriter> metadataWriterOpt = table.getMetadataWriter(dropInstant);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
index c3cc4eef4ed5..b7c51f31c5cf 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
@@ -57,6 +57,7 @@ import org.apache.hudi.exception.HoodieIOException;
 import org.apache.hudi.metadata.HoodieTableMetadata;
 import org.apache.hudi.metadata.HoodieTableMetadataUtil;
 import org.apache.hudi.metadata.HoodieIndexVersion;
+import org.apache.hudi.metadata.MetadataPartitionType;
 import org.apache.hudi.storage.StoragePath;
 import org.apache.hudi.table.HoodieTable;
 import org.apache.hudi.table.action.HoodieWriteMetadata;
@@ -294,12 +295,23 @@ public class UpgradeDowngradeUtils {
                                               HoodieTable table, SupportsUpgradeDowngrade upgradeDowngradeHelper, String operationType) {
     HoodieTableMetaClient metaClient = table.getMetaClient();
     try (BaseHoodieWriteClient writeClient = upgradeDowngradeHelper.getWriteClient(config, context)) {
-      List<String> mdtPartitions = metaClient.getTableConfig().getMetadataPartitions()
+      Set<String> metadataPartitions = metaClient.getTableConfig().getMetadataPartitions();
+      List<String> mdtPartitions = metadataPartitions
           .stream()
           .filter(partition -> metaClient.getIndexForMetadataPartition(partition)
               .map(indexDef -> HoodieIndexVersion.V1.lowerThan(indexDef.getVersion()))
               .orElse(false))
           .collect(Collectors.toList());
+
+      // If col stats V2 is being deleted and partition stats exists, delete partition stats as well
+      // This handles the case where partition stats might not have an index definition in index.json
+      String colStatsPartition = MetadataPartitionType.COLUMN_STATS.getPartitionPath();
+      String partitionStatsPartition = MetadataPartitionType.PARTITION_STATS.getPartitionPath();
+      if (mdtPartitions.contains(colStatsPartition)
+          && metadataPartitions.contains(partitionStatsPartition)) {
+        mdtPartitions.add(partitionStatsPartition);
+      }
+
       LOG.info("Dropping from MDT partitions for {}: {}", operationType, mdtPartitions);
       if (!mdtPartitions.isEmpty()) {
         writeClient.dropIndex(mdtPartitions);
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToEightDowngradeHandler.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToEightDowngradeHandler.java
index 4dc4123a9d90..c274c5c845c4 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToEightDowngradeHandler.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/upgrade/TestNineToEightDowngradeHandler.java
@@ -325,4 +325,124 @@ class TestNineToEightDowngradeHandler {
       assertEquals(new HashSet<>(Arrays.asList("secondary_index_v2")), new HashSet<>(capturedIndexes));
     }
   }
+
+  @Test
+  void testDowngradeDropsPartitionStatsWhenColStatsV2IsDeleted() {
+    try (MockedStatic<UpgradeDowngradeUtils> mockedUtils = Mockito.mockStatic(UpgradeDowngradeUtils.class)) {
+      mockedUtils.when(() -> UpgradeDowngradeUtils.rollbackFailedWritesAndCompact(
+          any(HoodieTable.class),
+          any(HoodieEngineContext.class),
+          any(HoodieWriteConfig.class),
+          any(SupportsUpgradeDowngrade.class),
+          anyBoolean(),
+          any(HoodieTableVersion.class)
+      )).thenAnswer(invocation -> null);
+
+      mockedUtils.when(() -> UpgradeDowngradeUtils.dropNonV1IndexPartitions(
+          eq(config),
+          eq(context),
+          eq(table),
+          eq(upgradeDowngradeHelper),
+          any(String.class)
+      )).thenCallRealMethod();
+
+      // Mocking index definitions: col_stats V2 is present but partition_stats is NOT in index.json
+      Map<String, HoodieIndexDefinition> indexDefs = new HashMap<>();
+      indexDefs.put(MetadataPartitionType.COLUMN_STATS.getPartitionPath(), HoodieIndexDefinition.newBuilder()
+          .withIndexName(MetadataPartitionType.COLUMN_STATS.getPartitionPath())
+          .withIndexType(MetadataPartitionType.COLUMN_STATS.getPartitionPath())
+          .withVersion(HoodieIndexVersion.V2)
+          .build());
+      HoodieIndexMetadata metadata = new HoodieIndexMetadata(indexDefs);
+
+      when(table.getMetaClient()).thenReturn(metaClient);
+      when(metaClient.getIndexMetadata()).thenReturn(Option.of(metadata));
+      when(metaClient.getTableConfig()).thenReturn(tableConfig);
+
+      // Mocking metadata partitions to include both col_stats and partition_stats
+      Set<String> mdtPartitions = new HashSet<>();
+      mdtPartitions.add(MetadataPartitionType.COLUMN_STATS.getPartitionPath());
+      mdtPartitions.add(MetadataPartitionType.PARTITION_STATS.getPartitionPath()); // partition_stats exists but not in index.json
+      when(tableConfig.getMetadataPartitions()).thenReturn(mdtPartitions);
+
+      // Mocking getIndexForMetadataPartition to return the index definition for col_stats but empty for partition_stats
+      when(metaClient.getIndexForMetadataPartition(MetadataPartitionType.COLUMN_STATS.getPartitionPath()))
+          .thenReturn(Option.of(indexDefs.get(MetadataPartitionType.COLUMN_STATS.getPartitionPath())));
+      when(metaClient.getIndexForMetadataPartition(MetadataPartitionType.PARTITION_STATS.getPartitionPath()))
+          .thenReturn(Option.empty()); // partition_stats has no index definition
+
+      BaseHoodieWriteClient writeClient = mock(BaseHoodieWriteClient.class);
+      when(upgradeDowngradeHelper.getWriteClient(config, context)).thenReturn(writeClient);
+
+      handler.downgrade(config, context, "20240101120000", upgradeDowngradeHelper);
+
+      ArgumentCaptor<List<String>> argumentCaptor = ArgumentCaptor.forClass(List.class);
+      verify(writeClient, times(1)).dropIndex(argumentCaptor.capture());
+      List<String> capturedIndexes = argumentCaptor.getValue();
+      assertEquals(new HashSet<>(Arrays.asList(
+          MetadataPartitionType.COLUMN_STATS.getPartitionPath(),
+          MetadataPartitionType.PARTITION_STATS.getPartitionPath()
+      )), new HashSet<>(capturedIndexes));
+    }
+  }
+
+  @Test
+  void testDowngradeDoesNotDropV1Indexes() {
+    try (MockedStatic<UpgradeDowngradeUtils> mockedUtils = Mockito.mockStatic(UpgradeDowngradeUtils.class)) {
+      mockedUtils.when(() -> UpgradeDowngradeUtils.rollbackFailedWritesAndCompact(
+          any(HoodieTable.class),
+          any(HoodieEngineContext.class),
+          any(HoodieWriteConfig.class),
+          any(SupportsUpgradeDowngrade.class),
+          anyBoolean(),
+          any(HoodieTableVersion.class)
+      )).thenAnswer(invocation -> null);
+
+      mockedUtils.when(() -> UpgradeDowngradeUtils.dropNonV1IndexPartitions(
+          eq(config),
+          eq(context),
+          eq(table),
+          eq(upgradeDowngradeHelper),
+          any(String.class)
+      )).thenCallRealMethod();
+
+      // Mocking index definitions: col_stats and partition_stats both have V1 (created in V8, upgraded to V9)
+      Map<String, HoodieIndexDefinition> indexDefs = new HashMap<>();
+      indexDefs.put(MetadataPartitionType.COLUMN_STATS.getPartitionPath(), HoodieIndexDefinition.newBuilder()
+          .withIndexName(MetadataPartitionType.COLUMN_STATS.getPartitionPath())
+          .withIndexType(MetadataPartitionType.COLUMN_STATS.getPartitionPath())
+          .withVersion(HoodieIndexVersion.V1)
+          .build());
+      indexDefs.put(MetadataPartitionType.PARTITION_STATS.getPartitionPath(), HoodieIndexDefinition.newBuilder()
+          .withIndexName(MetadataPartitionType.PARTITION_STATS.getPartitionPath())
+          .withIndexType(MetadataPartitionType.PARTITION_STATS.getPartitionPath())
+          .withVersion(HoodieIndexVersion.V1)
+          .build());
+      HoodieIndexMetadata metadata = new HoodieIndexMetadata(indexDefs);
+
+      when(table.getMetaClient()).thenReturn(metaClient);
+      when(metaClient.getIndexMetadata()).thenReturn(Option.of(metadata));
+      when(metaClient.getTableConfig()).thenReturn(tableConfig);
+
+      // Mocking metadata partitions to include both col_stats and partition_stats
+      Set<String> mdtPartitions = new HashSet<>();
+      mdtPartitions.add(MetadataPartitionType.COLUMN_STATS.getPartitionPath());
+      mdtPartitions.add(MetadataPartitionType.PARTITION_STATS.getPartitionPath());
+      when(tableConfig.getMetadataPartitions()).thenReturn(mdtPartitions);
+
+      // Mocking getIndexForMetadataPartition to return V1 index definitions
+      when(metaClient.getIndexForMetadataPartition(MetadataPartitionType.COLUMN_STATS.getPartitionPath()))
+          .thenReturn(Option.of(indexDefs.get(MetadataPartitionType.COLUMN_STATS.getPartitionPath())));
+      when(metaClient.getIndexForMetadataPartition(MetadataPartitionType.PARTITION_STATS.getPartitionPath()))
+          .thenReturn(Option.of(indexDefs.get(MetadataPartitionType.PARTITION_STATS.getPartitionPath())));
+
+      BaseHoodieWriteClient writeClient = mock(BaseHoodieWriteClient.class);
+      when(upgradeDowngradeHelper.getWriteClient(config, context)).thenReturn(writeClient);
+
+      handler.downgrade(config, context, "20240101120000", upgradeDowngradeHelper);
+
+      // Verify that dropIndex is not called at all since there are no V2 indexes to drop
+      verify(writeClient, times(0)).dropIndex(any());
+    }
+  }
 }