You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2021/07/06 12:17:40 UTC
[spark] branch master updated: [SPARK-36025][SQL][TESTS] Reduce the
run time of DateExpressionsSuite
This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new d5d1222 [SPARK-36025][SQL][TESTS] Reduce the run time of DateExpressionsSuite
d5d1222 is described below
commit d5d12226861f67243dd575c9240238bcd08e1a91
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Tue Jul 6 20:17:02 2021 +0800
[SPARK-36025][SQL][TESTS] Reduce the run time of DateExpressionsSuite
### What changes were proposed in this pull request?
Some of the test cases in `DateExpressionsSuite` are quite slow:
- `Hour`: 24s
- `Minute`: 26s
- `Day / DayOfMonth`: 8s
- `Year`: 4s
Each test case has a large loop. We should improve them.
### Why are the changes needed?
Save test running time
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Verified the run times on local:
- `Hour`: 2s
- `Minute`: 3.2
- `Day / DayOfMonth`:0.5s
- `Year`: 2s
Total reduced time: 54.3s
Closes #33229 from gengliangwang/improveTest.
Authored-by: Gengliang Wang <ge...@apache.org>
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../expressions/DateExpressionsSuite.scala | 49 ++++++++++------------
1 file changed, 23 insertions(+), 26 deletions(-)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index d33fb7d..afcc729 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -25,7 +25,9 @@ import java.time.temporal.ChronoUnit
import java.util.{Calendar, Locale, TimeZone}
import java.util.concurrent.TimeUnit._
+import scala.language.postfixOps
import scala.reflect.ClassTag
+import scala.util.Random
import org.apache.spark.{SparkFunSuite, SparkUpgradeException}
import org.apache.spark.sql.catalyst.InternalRow
@@ -122,8 +124,8 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
(2000 to 2002).foreach { y =>
(0 to 11 by 11).foreach { m =>
c.set(y, m, 28)
- (0 to 5 * 24).foreach { i =>
- c.add(Calendar.HOUR_OF_DAY, 1)
+ (0 to 12).foreach { i =>
+ c.add(Calendar.HOUR_OF_DAY, 10)
checkEvaluation(Year(Literal(new Date(c.getTimeInMillis))),
c.get(Calendar.YEAR))
}
@@ -195,8 +197,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val c = Calendar.getInstance()
(1999 to 2000).foreach { y =>
c.set(y, 0, 1, 0, 0, 0)
- (0 to 365).foreach { d =>
- c.add(Calendar.DATE, 1)
+ val random = new Random(System.nanoTime)
+ random.shuffle(0 to 365 toList).take(10).foreach { d =>
+ c.set(Calendar.DAY_OF_YEAR, d)
checkEvaluation(DayOfMonth(Literal(new Date(c.getTimeInMillis))),
c.get(Calendar.DAY_OF_MONTH))
}
@@ -332,19 +335,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val timeZoneId = Option(zid.getId)
c.setTimeZone(TimeZone.getTimeZone(zid))
(0 to 24 by 5).foreach { h =>
- (0 to 60 by 29).foreach { m =>
- (0 to 60 by 29).foreach { s =>
- // validate timestamp with local time zone
- c.set(2015, 18, 3, h, m, s)
- checkEvaluation(
- Hour(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
- c.get(Calendar.HOUR_OF_DAY))
+ // validate timestamp with local time zone
+ c.set(2015, 18, 3, h, 29, 59)
+ checkEvaluation(
+ Hour(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
+ c.get(Calendar.HOUR_OF_DAY))
- // validate timestamp without time zone
- val localDateTime = LocalDateTime.of(2015, 1, 3, h, m, s)
- checkEvaluation(Hour(Literal(localDateTime), timeZoneId), h)
- }
- }
+ // validate timestamp without time zone
+ val localDateTime = LocalDateTime.of(2015, 1, 3, h, 29, 59)
+ checkEvaluation(Hour(Literal(localDateTime), timeZoneId), h)
}
Seq(TimestampType, TimestampNTZType).foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen(
@@ -367,17 +366,15 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
val timeZoneId = Option(zid.getId)
c.setTimeZone(TimeZone.getTimeZone(zid))
(0 to 59 by 5).foreach { m =>
- (0 to 59 by 15).foreach { s =>
- // validate timestamp with local time zone
- c.set(2015, 18, 3, 3, m, s)
- checkEvaluation(
- Minute(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
- c.get(Calendar.MINUTE))
+ // validate timestamp with local time zone
+ c.set(2015, 18, 3, 3, m, 3)
+ checkEvaluation(
+ Minute(Literal(new Timestamp(c.getTimeInMillis)), timeZoneId),
+ c.get(Calendar.MINUTE))
- // validate timestamp without time zone
- val localDateTime = LocalDateTime.of(2015, 1, 3, 3, m, s)
- checkEvaluation(Minute(Literal(localDateTime), timeZoneId), m)
- }
+ // validate timestamp without time zone
+ val localDateTime = LocalDateTime.of(2015, 1, 3, 3, m, 3)
+ checkEvaluation(Minute(Literal(localDateTime), timeZoneId), m)
}
Seq(TimestampType, TimestampNTZType).foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org