You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/11/26 16:05:20 UTC
(hudi) branch master updated: [HUDI-8571] Fix the upgrade for OverwriteWithLatestPayload (#12335)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5c28762c80c [HUDI-8571] Fix the upgrade for OverwriteWithLatestPayload (#12335)
5c28762c80c is described below
commit 5c28762c80c678e66df662c0f4e4c2855766840f
Author: Lin Liu <14...@users.noreply.github.com>
AuthorDate: Tue Nov 26 08:05:11 2024 -0800
[HUDI-8571] Fix the upgrade for OverwriteWithLatestPayload (#12335)
---
.../hudi/table/upgrade/SevenToEightUpgradeHandler.java | 12 ++++++------
.../org/apache/hudi/functional/TestSevenToEightUpgrade.scala | 6 +++---
2 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
index 4926ac1dc2f..d57a5e0498e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/SevenToEightUpgradeHandler.java
@@ -69,7 +69,7 @@ public class SevenToEightUpgradeHandler implements UpgradeHandler {
Map<ConfigProperty, String> tablePropsToAdd = new HashMap<>();
tablePropsToAdd.put(HoodieTableConfig.TIMELINE_PATH, HoodieTableConfig.TIMELINE_PATH.defaultValue());
upgradePartitionFields(config, tableConfig, tablePropsToAdd);
- upgradeMergeMode(config, tableConfig, tablePropsToAdd);
+ upgradeMergeMode(tableConfig, tablePropsToAdd);
// Handle timeline upgrade:
// - Rewrite instants in active timeline to new format
@@ -126,20 +126,20 @@ public class SevenToEightUpgradeHandler implements UpgradeHandler {
}
}
- private static void upgradeMergeMode(HoodieWriteConfig config, HoodieTableConfig tableConfig, Map<ConfigProperty, String> tablePropsToAdd) {
+ private static void upgradeMergeMode(HoodieTableConfig tableConfig, Map<ConfigProperty, String> tablePropsToAdd) {
if (tableConfig.getPayloadClass() != null
&& tableConfig.getPayloadClass().equals(OverwriteWithLatestAvroPayload.class.getName())) {
if (HoodieTableType.COPY_ON_WRITE == tableConfig.getTableType()) {
+ tablePropsToAdd.put(
+ HoodieTableConfig.RECORD_MERGE_MODE,
+ RecordMergeMode.COMMIT_TIME_ORDERING.name());
+ } else {
tablePropsToAdd.put(
HoodieTableConfig.PAYLOAD_CLASS_NAME,
DefaultHoodieRecordPayload.class.getName());
tablePropsToAdd.put(
HoodieTableConfig.RECORD_MERGE_MODE,
RecordMergeMode.EVENT_TIME_ORDERING.name());
- } else {
- tablePropsToAdd.put(
- HoodieTableConfig.RECORD_MERGE_MODE,
- RecordMergeMode.COMMIT_TIME_ORDERING.name());
}
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala
index 18eafd16029..862cd4f9ae9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSevenToEightUpgrade.scala
@@ -79,12 +79,12 @@ class TestSevenToEightUpgrade extends RecordLevelIndexTestBase {
// After upgrade, based on the payload and table type, the merge mode is updated accordingly.
if (HoodieTableType.COPY_ON_WRITE == tableType) {
- assertEquals(classOf[DefaultHoodieRecordPayload].getName, metaClient.getTableConfig.getPayloadClass)
- assertEquals(RecordMergeMode.EVENT_TIME_ORDERING.name, metaClient.getTableConfig.getRecordMergeMode.name)
- } else {
assertEquals(classOf[OverwriteWithLatestAvroPayload].getName, metaClient.getTableConfig.getPayloadClass)
assertEquals(RecordMergeMode.COMMIT_TIME_ORDERING.name, metaClient.getTableConfig.getRecordMergeMode.name)
assertEquals(HoodieRecordMerger.COMMIT_TIME_BASED_MERGE_STRATEGY_UUID, metaClient.getTableConfig.getRecordMergeStrategyId)
+ } else {
+ assertEquals(classOf[DefaultHoodieRecordPayload].getName, metaClient.getTableConfig.getPayloadClass)
+ assertEquals(RecordMergeMode.EVENT_TIME_ORDERING.name, metaClient.getTableConfig.getRecordMergeMode.name)
}
}
}