You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/15 11:59:01 UTC
(hudi) 08/21: Fixing deltastreamer tests for auto record key gen (#11099)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 60a0183a7916cf05702dc4455e7d1adf002acb21
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Mon May 6 08:00:11 2024 -0700
Fixing deltastreamer tests for auto record key gen (#11099)
---
.../deltastreamer/HoodieDeltaStreamerTestBase.java | 19 ++++++++++++++-----
.../deltastreamer/TestHoodieDeltaStreamer.java | 7 +++++--
.../TestHoodieDeltaStreamerSchemaEvolutionBase.java | 2 +-
3 files changed, 20 insertions(+), 8 deletions(-)
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
index cf0d197ff19..b03bccdca39 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
@@ -354,7 +354,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String emptyBatchParam) throws IOException {
prepareParquetDFSSource(useSchemaProvider, hasTransformer, "source.avsc", "target.avsc",
- PROPS_FILENAME_TEST_PARQUET, PARQUET_SOURCE_ROOT, false, "partition_path", emptyBatchParam);
+ PROPS_FILENAME_TEST_PARQUET, PARQUET_SOURCE_ROOT, false, "partition_path", emptyBatchParam, false);
}
protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer) throws IOException {
@@ -364,20 +364,27 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
String propsFileName, String parquetSourceRoot, boolean addCommonProps, String partitionPath) throws IOException {
prepareParquetDFSSource(useSchemaProvider, hasTransformer, sourceSchemaFile, targetSchemaFile, propsFileName, parquetSourceRoot, addCommonProps,
- partitionPath, "");
+ partitionPath, "", false);
}
protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
String propsFileName, String parquetSourceRoot, boolean addCommonProps,
String partitionPath, String emptyBatchParam) throws IOException {
prepareParquetDFSSource(useSchemaProvider, hasTransformer, sourceSchemaFile, targetSchemaFile, propsFileName, parquetSourceRoot, addCommonProps,
- partitionPath, emptyBatchParam, null);
+ partitionPath, emptyBatchParam, false);
+ }
+ protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
+ String propsFileName, String parquetSourceRoot, boolean addCommonProps,
+ String partitionPath, String emptyBatchParam, boolean skipRecordKeyField) throws IOException {
+ prepareParquetDFSSource(useSchemaProvider, hasTransformer, sourceSchemaFile, targetSchemaFile, propsFileName, parquetSourceRoot, addCommonProps,
+ partitionPath, emptyBatchParam, null, skipRecordKeyField);
}
protected void prepareParquetDFSSource(boolean useSchemaProvider, boolean hasTransformer, String sourceSchemaFile, String targetSchemaFile,
String propsFileName, String parquetSourceRoot, boolean addCommonProps,
- String partitionPath, String emptyBatchParam, TypedProperties extraProps) throws IOException {
+ String partitionPath, String emptyBatchParam, TypedProperties extraProps,
+ boolean skipRecordKeyField) throws IOException {
// Properties used for testing delta-streamer with Parquet source
TypedProperties parquetProps = new TypedProperties(extraProps);
@@ -389,7 +396,9 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
parquetProps.setProperty("include", "base.properties");
parquetProps.setProperty("hoodie.embed.timeline.server", "false");
- parquetProps.setProperty("hoodie.datasource.write.recordkey.field", "_row_key");
+ if (!skipRecordKeyField) {
+ parquetProps.setProperty("hoodie.datasource.write.recordkey.field", "_row_key");
+ }
parquetProps.setProperty("hoodie.datasource.write.partitionpath.field", partitionPath);
if (useSchemaProvider) {
parquetProps.setProperty("hoodie.streamer.schemaprovider.source.schema.file", basePath + "/" + sourceSchemaFile);
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index bb9dad96a3b..59ba56fb460 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -850,7 +850,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
extraProps.setProperty("hoodie.datasource.write.table.type", "MERGE_ON_READ");
extraProps.setProperty("hoodie.datasource.compaction.async.enable", "false");
prepareParquetDFSSource(false, false, "source.avsc", "target.avsc", PROPS_FILENAME_TEST_PARQUET,
- PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps);
+ PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false);
String tableBasePath = basePath + "test_parquet_table" + testNum;
HoodieDeltaStreamer.Config deltaCfg = TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, ParquetDFSSource.class.getName(),
null, PROPS_FILENAME_TEST_PARQUET, false,
@@ -2844,7 +2844,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
boolean hasTransformer = transformerClassNames != null && !transformerClassNames.isEmpty();
prepareParquetDFSFiles(parquetRecordsCount, PARQUET_SOURCE_ROOT, FIRST_PARQUET_FILE_NAME, false, null, null);
prepareParquetDFSSource(useSchemaProvider, hasTransformer, "source.avsc", "target.avsc", PROPS_FILENAME_TEST_PARQUET,
- PARQUET_SOURCE_ROOT, false, "partition_path", "");
+ PARQUET_SOURCE_ROOT, false, "partition_path", "", true);
String tableBasePath = basePath + "/test_parquet_table" + testNum;
HoodieDeltaStreamer.Config config = TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT, ParquetDFSSource.class.getName(),
@@ -2853,6 +2853,9 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
HoodieDeltaStreamer deltaStreamer = new HoodieDeltaStreamer(config, jsc);
deltaStreamer.sync();
assertRecordCount(parquetRecordsCount, tableBasePath, sqlContext);
+ // validate that auto record keys are enabled.
+ HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setBasePath(tableBasePath).setConf(jsc.hadoopConfiguration()).build();
+ assertFalse(metaClient.getTableConfig().getRecordKeyFields().isPresent());
prepareParquetDFSFiles(200, PARQUET_SOURCE_ROOT, "2.parquet", false, null, null);
deltaStreamer.sync();
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
index d9cb55c886a..c6f2afc2ef7 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamerSchemaEvolutionBase.java
@@ -198,7 +198,7 @@ public class TestHoodieDeltaStreamerSchemaEvolutionBase extends HoodieDeltaStrea
transformerClassNames, PROPS_FILENAME_TEST_AVRO_KAFKA, false, useSchemaProvider, 100000, false, null, tableType, "timestamp", null);
} else {
prepareParquetDFSSource(false, hasTransformer, sourceSchemaFile, targetSchemaFile, PROPS_FILENAME_TEST_PARQUET,
- PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps);
+ PARQUET_SOURCE_ROOT, false, "partition_path", "", extraProps, false);
cfg = TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT, ParquetDFSSource.class.getName(),
transformerClassNames, PROPS_FILENAME_TEST_PARQUET, false,
useSchemaProvider, 100000, false, null, tableType, "timestamp", null);