You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2021/08/03 07:31:52 UTC
[GitHub] [spark] sarutak commented on a change in pull request #33606: [SPARK-35815][SQL][FOLLOWUP] Add test considering the case spark.sql.legacy.interval.enabled is true
sarutak commented on a change in pull request #33606:
URL: https://github.com/apache/spark/pull/33606#discussion_r681508366
##########
File path: sql/core/src/test/scala/org/apache/spark/sql/streaming/EventTimeWatermarkSuite.scala
##########
@@ -766,55 +766,62 @@ class EventTimeWatermarkSuite extends StreamTest with BeforeAndAfter with Matche
}
}
- test("SPARK-35815: Support ANSI intervals for delay threshold") {
- val DAYS_PER_MONTH = 31
- Seq(
- // Conventional form and some variants
- (Seq("3 days", "Interval 3 day", "inTerval '3' day"), 3 * MILLIS_PER_DAY),
- (Seq(" 5 hours", "INTERVAL 5 hour", "interval '5' hour"), 5 * MILLIS_PER_HOUR),
- (Seq("\t8 minutes", "interval 8 minute", "interval '8' minute"), 8 * MILLIS_PER_MINUTE),
- (Seq("10 seconds", "interval 10 second", "interval '10' second"), 10 * MILLIS_PER_SECOND),
- (Seq("1 years", "interval 1 year", "interval '1' year"),
- MONTHS_PER_YEAR * DAYS_PER_MONTH * MILLIS_PER_DAY),
- (Seq("1 months", "interval 1 month", "interval '1' month"), DAYS_PER_MONTH * MILLIS_PER_DAY),
- (Seq(
- "1 day 2 hours 3 minutes 4 seconds",
- " interval 1 day 2 hours 3 minutes 4 seconds",
- "\tinterval '1' day '2' hours '3' minutes '4' seconds",
- "interval '1 2:3:4' day to second"),
- MILLIS_PER_DAY + 2 * MILLIS_PER_HOUR + 3 * MILLIS_PER_MINUTE + 4 * MILLIS_PER_SECOND),
- (Seq(
- " 1 year 2 months",
- "interval 1 year 2 month",
- "interval '1' year '2' month",
- "\tinterval '1-2' year to month"),
- (MONTHS_PER_YEAR * DAYS_PER_MONTH + 2 * DAYS_PER_MONTH) * MILLIS_PER_DAY)
- ).foreach { case (delayThresholdVariants, expectedMs) =>
- delayThresholdVariants.foreach { case delayThreshold =>
- val df = MemoryStream[Int].toDF
- .withColumn("eventTime", timestamp_seconds($"value"))
- .withWatermark("eventTime", delayThreshold)
- val eventTimeAttr = df.queryExecution.analyzed.output.find(a => a.name == "eventTime")
- assert(eventTimeAttr.isDefined)
- val metadata = eventTimeAttr.get.metadata
- assert(metadata.contains(EventTimeWatermark.delayKey))
- assert(metadata.getLong(EventTimeWatermark.delayKey) === expectedMs)
- }
- }
-
- // Invalid interval patterns
- Seq(
- "1 foo",
- "interva 2 day",
- "intrval '3' day",
- "interval 4 foo",
- "interval '5' foo",
- "interval '1 2:3:4' day to hour",
- "interval '1 2' year to month").foreach { delayThreshold =>
- intercept[AnalysisException] {
- val df = MemoryStream[Int].toDF
- .withColumn("eventTime", timestamp_seconds($"value"))
- .withWatermark("eventTime", delayThreshold)
+ Seq("true", "false").foreach { legacyIntervalEnabled =>
+ test("SPARK-35815: Support ANSI intervals for delay threshold" +
+ s"(${SQLConf.LEGACY_INTERVAL_ENABLED.key}=$legacyIntervalEnabled") {
Review comment:
Oops. Thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org