You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/14 23:51:17 UTC
(hudi) 12/22: [HUDI-7605] Allow merger strategy to be set in spark sql writer (#10999)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a5f4e8f1894e33dfa66318867b4fbb11e429dfe6
Author: Jon Vexler <jb...@gmail.com>
AuthorDate: Thu Apr 11 21:20:07 2024 -0400
[HUDI-7605] Allow merger strategy to be set in spark sql writer (#10999)
---
.../scala/org/apache/hudi/HoodieSparkSqlWriter.scala | 1 +
.../apache/hudi/functional/TestMORDataSource.scala | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+)
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 7020781faf0..ad19ec48c7a 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -302,6 +302,7 @@ class HoodieSparkSqlWriterInternal {
.setPartitionMetafileUseBaseFormat(useBaseFormatMetaFile)
.setShouldDropPartitionColumns(hoodieConfig.getBooleanOrDefault(HoodieTableConfig.DROP_PARTITION_COLUMNS))
.setCommitTimezone(timelineTimeZone)
+ .setRecordMergerStrategy(hoodieConfig.getStringOrDefault(DataSourceWriteOptions.RECORD_MERGER_STRATEGY))
.initTable(sparkContext.hadoopConfiguration, path)
}
val instantTime = HoodieActiveTimeline.createNewInstantTime()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
index 45bd3c645d4..b878eb76c40 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
@@ -1403,4 +1403,24 @@ class TestMORDataSource extends HoodieSparkClientTestBase with SparkDatasetMixin
basePath
}
}
+
+ @Test
+ def testMergerStrategySet(): Unit = {
+ val (writeOpts, _) = getWriterReaderOpts()
+ val input = recordsToStrings(dataGen.generateInserts("000", 1)).asScala
+ val inputDf= spark.read.json(spark.sparkContext.parallelize(input, 1))
+ val mergerStrategyName = "example_merger_strategy"
+ inputDf.write.format("hudi")
+ .options(writeOpts)
+ .option(DataSourceWriteOptions.TABLE_TYPE.key, "MERGE_ON_READ")
+ .option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
+ .option(DataSourceWriteOptions.RECORD_MERGER_STRATEGY.key(), mergerStrategyName)
+ .mode(SaveMode.Overwrite)
+ .save(basePath)
+ metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(spark.sessionState.newHadoopConf)
+ .build()
+ assertEquals(metaClient.getTableConfig.getRecordMergerStrategy, mergerStrategyName)
+ }
}