You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by GitBox <gi...@apache.org> on 2022/10/17 18:21:16 UTC
[GitHub] [hudi] alexeykudinkin commented on a diff in pull request #6949: [MINOR] Test case for hoodie.merge.allow.duplicate.on.inserts
alexeykudinkin commented on code in PR #6949:
URL: https://github.com/apache/hudi/pull/6949#discussion_r997376458
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala:
##########
@@ -901,4 +901,47 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Test enable hoodie.merge.allow.duplicate.on.inserts when write") {
+ spark.sql("set hoodie.datasource.write.operation = insert")
+ Seq("mor", "cow").foreach { tableType =>
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long,
+ | dt string
+ | ) using hudi
+ | partitioned by (dt)
+ | location '${tmp.getCanonicalPath}/$tableName'
+ | tblproperties (
+ | primaryKey = 'id',
+ | preCombineField = 'ts',
+ | type = '$tableType'
+ | )
+ """.stripMargin)
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a1', 10, 1000)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a1", 10, 1000, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a2', 20, 1001)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a2", 20, 1001, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a3', 30, 1002)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a2", 20, 1001, "2021-12-25"),
+ Seq(1, "a3", 30, 1002, "2021-12-25")
+ )
+ }
+ }
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
Review Comment:
I don't think it makes sense to set these after the test is done
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestInsertTable.scala:
##########
@@ -901,4 +901,47 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
}
}
}
+
+ test("Test enable hoodie.merge.allow.duplicate.on.inserts when write") {
+ spark.sql("set hoodie.datasource.write.operation = insert")
+ Seq("mor", "cow").foreach { tableType =>
+ withTempDir { tmp =>
+ val tableName = generateTableName
+ spark.sql(
+ s"""
+ | create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long,
+ | dt string
+ | ) using hudi
+ | partitioned by (dt)
+ | location '${tmp.getCanonicalPath}/$tableName'
+ | tblproperties (
+ | primaryKey = 'id',
+ | preCombineField = 'ts',
+ | type = '$tableType'
+ | )
+ """.stripMargin)
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a1', 10, 1000)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a1", 10, 1000, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = false")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a2', 20, 1001)")
+ checkAnswer(s"select id, name, price, ts, dt from $tableName")(
+ Seq(1, "a2", 20, 1001, "2021-12-25")
+ )
+ spark.sql("set hoodie.merge.allow.duplicate.on.inserts = true")
+ spark.sql(s"insert into $tableName partition(dt='2021-12-25') values (1, 'a3', 30, 1002)")
Review Comment:
I'd suggest we extract primary-key to a common var to make sharing apparent
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org