Re: [PR] test(spark): add TimestampBasedKeyGenerator tests with various configurations [hudi]

["voonhous (via GitHub)" <gi...@apache.org>]

voonhous commented on code in PR #19657:
URL: https://github.com/apache/hudi/pull/19657#discussion_r3940321894


##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala:
##########
@@ -1450,6 +1450,73 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
     assertTrue(recordsReadDF.filter(col("_hoodie_partition_path") =!= udf_date_format(col("current_ts"))).count() == 0)
   }
 
+  @ParameterizedTest
+  @EnumSource(value = classOf[HoodieRecordType], names = Array("AVRO", "SPARK"))
+  def testTimestampBasedKeyGeneratorWithVariousConfigurations(recordType: HoodieRecordType) {
+    val (writeOpts, readOpts) = getWriterReaderOptsLessPartitionPath(recordType)
+
+    val records = recordsToStrings(dataGen.generateInserts("000", 100)).asScala.toList
+    val inputDF = spark.read.json(spark.sparkContext.parallelize(records, 2))
+      .withColumn("current_ts_micros", col("current_ts") * 1000)
+      .withColumn("current_date_string",
+        date_format((col("current_ts") / 1000).cast("timestamp"), "yyyy-MM-dd HH:mm:ss"))
+      .withColumn("current_ts_hours", (col("current_ts") / 3600000).cast("long"))
+      .withColumn("current_ts_seconds", (col("current_ts") / 1000).cast("long"))
+
+    case class TestCase(partitionCol: String, tsType: String, outFmt: String,
+                        extraOpts: Map[String, String] = Map.empty,
+                        expectedPartitionUdf: org.apache.spark.sql.expressions.UserDefinedFunction)
+
+    def runTestCase(tc: TestCase): Unit = {
+      val writer = tc.extraOpts.foldLeft(
+        inputDF.write.format("hudi")
+          .options(writeOpts)
+          .option(KEYGENERATOR_CLASS_NAME.key(), classOf[TimestampBasedKeyGenerator].getName)
+          .mode(SaveMode.Overwrite)
+      ) { case (w, (k, v)) => w.option(k, v) }
+      writer.partitionBy(tc.partitionCol)
+        .option(TIMESTAMP_TYPE_FIELD.key, tc.tsType)
+        .option(TIMESTAMP_OUTPUT_DATE_FORMAT.key, tc.outFmt)
+        .save(basePath)
+      val readDF = spark.read.format("org.apache.hudi").options(readOpts).load(basePath)
+      assertTrue(readDF.filter(col("_hoodie_partition_path") =!= tc.expectedPartitionUdf(col(tc.partitionCol))).count() == 0)
+    }
+
+    // Test 1: EPOCHMILLISECONDS with timezone GMT+08:00
+    val tzMillisOutFmt = "yyyy-MM-dd HH"
+    val udfMillisTz = udf((millis: Long) =>
+      new DateTime(millis).withZone(org.joda.time.DateTimeZone.forID("GMT+08:00"))
+        .toString(DateTimeFormat.forPattern(tzMillisOutFmt).withZone(org.joda.time.DateTimeZone.forID("GMT+08:00"))))
+    runTestCase(TestCase("current_ts", "EPOCHMILLISECONDS", tzMillisOutFmt,
+      Map(TIMESTAMP_TIMEZONE_FORMAT.key -> "GMT+08:00"), udfMillisTz))
+
+    // Test 2: EPOCHMICROSECONDS
+    val microsOutFmt = "yyyy-MM-dd HH"
+    val udfMicros = udf((micros: Long) =>
+      new DateTime(micros / 1000).toString(DateTimeFormat.forPattern(microsOutFmt)))
+    runTestCase(TestCase("current_ts_micros", "EPOCHMICROSECONDS", microsOutFmt,
+      expectedPartitionUdf = udfMicros))
+
+    // Test 3: DATE_STRING with timezone
+    val dateStrOutFmt = "yyyy-MM-dd HH"
+    val dateStrInFmt = "yyyy-MM-dd HH:mm:ss"
+    val udfDateStrTz = udf((s: String) =>

Review Comment:
   The parse zone did move onto the input formatter in 45131da, which is what this asks for.
   
   It never ran, though: `DateTimeZone.forID("GMT+08:00")` throws `IllegalArgumentException: The datetime zone id 'GMT+08:00' is not recognised`. Joda's `forID` takes tz-db ids or bare offsets like `+08:00`, not `GMT+hh:mm`, so this UDF and the EPOCHMILLISECONDS one both blew up before the assertion. That was the Azure failure on 45131da.
   
   Fixed in 302bf99c by resolving the id the way `HoodieDateTimeParser` does, `DateTimeZone.forTimeZone(TimeZone.getTimeZone(id))`.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org