You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by fo...@apache.org on 2023/01/15 06:02:18 UTC
[hudi] branch master updated: [HUDI-5554] Add UT TestHiveSyncTool#testSyncMergeOnReadWithStrategy for parameter HIVE_SYNC_TABLE_STRATEGY. (#7671)
This is an automated email from the ASF dual-hosted git repository.
forwardxu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 7c213d5ff72 [HUDI-5554] Add UT TestHiveSyncTool#testSyncMergeOnReadWithStrategy for parameter HIVE_SYNC_TABLE_STRATEGY. (#7671)
7c213d5ff72 is described below
commit 7c213d5ff72cb713bf40ca74f4510f7fefbf5f33
Author: HunterXHunter <13...@qq.com>
AuthorDate: Sun Jan 15 14:02:12 2023 +0800
[HUDI-5554] Add UT TestHiveSyncTool#testSyncMergeOnReadWithStrategy for parameter HIVE_SYNC_TABLE_STRATEGY. (#7671)
---
.../org/apache/hudi/hive/TestHiveSyncTool.java | 57 ++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 1096a02d560..7f315733c4f 100644
--- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -20,6 +20,7 @@ package org.apache.hudi.hive;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieSyncTableStrategy;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.NetworkTestUtils;
@@ -69,6 +70,7 @@ import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_COMMENT;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_MODE;
+import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_TABLE_STRATEGY;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_TABLE_PROPERTIES;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_TABLE_SERDE_PROPERTIES;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL;
@@ -120,6 +122,16 @@ public class TestHiveSyncTool {
return opts;
}
+ private static Iterable<Object[]> syncModeAndStrategy() {
+ List<Object[]> opts = new ArrayList<>();
+ for (Object mode : SYNC_MODES) {
+ opts.add(new Object[] {mode, HoodieSyncTableStrategy.ALL});
+ opts.add(new Object[] {mode, HoodieSyncTableStrategy.RO});
+ opts.add(new Object[] {mode, HoodieSyncTableStrategy.RT});
+ }
+ return opts;
+ }
+
private HiveSyncTool hiveSyncTool;
private HoodieHiveSyncClient hiveClient;
@@ -758,6 +770,51 @@ public class TestHiveSyncTool {
"The last commit that was synced should be 103");
}
+ @ParameterizedTest
+ @MethodSource("syncModeAndStrategy")
+ public void testSyncMergeOnReadWithStrategy(String syncMode, HoodieSyncTableStrategy strategy) throws Exception {
+ hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), syncMode);
+ hiveSyncProps.setProperty(HIVE_SYNC_TABLE_STRATEGY.key(), strategy.name());
+
+ String instantTime = "100";
+ String deltaCommitTime = "101";
+ HiveTestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, true);
+
+ String snapshotTableName = HiveTestUtil.TABLE_NAME + HiveSyncTool.SUFFIX_SNAPSHOT_TABLE;
+ String roTableName = HiveTestUtil.TABLE_NAME + HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE;
+ reinitHiveSyncClient();
+ assertFalse(hiveClient.tableExists(roTableName),
+ "Table " + roTableName + " should not exist initially");
+ assertFalse(hiveClient.tableExists(snapshotTableName),
+ "Table " + snapshotTableName + " should not exist initially");
+ reSyncHiveTable();
+ switch (strategy) {
+ case RO:
+ assertFalse(hiveClient.tableExists(snapshotTableName),
+ "Table " + snapshotTableName
+ + " should not exist initially");
+ assertTrue(hiveClient.tableExists(HiveTestUtil.TABLE_NAME),
+ "Table " + HiveTestUtil.TABLE_NAME
+ + " should exist after sync completes");
+ break;
+ case RT:
+ assertFalse(hiveClient.tableExists(roTableName),
+ "Table " + roTableName
+ + " should not exist initially");
+ assertTrue(hiveClient.tableExists(HiveTestUtil.TABLE_NAME),
+ "Table " + HiveTestUtil.TABLE_NAME
+ + " should exist after sync completes");
+ break;
+ default:
+ assertTrue(hiveClient.tableExists(roTableName),
+ "Table " + roTableName
+ + " should exist after sync completes");
+ assertTrue(hiveClient.tableExists(snapshotTableName),
+ "Table " + snapshotTableName
+ + " should exist after sync completes");
+ }
+ }
+
@ParameterizedTest
@MethodSource("syncModeAndEnablePushDown")
public void testMultiPartitionKeySync(String syncMode, String enablePushDown) throws Exception {