You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2020/05/17 02:27:43 UTC
[spark] branch branch-3.0 updated: [SPARK-31725][CORE][SQL][TESTS]
Set America/Los_Angeles time zone and Locale.US in tests by default
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 4f5df2c [SPARK-31725][CORE][SQL][TESTS] Set America/Los_Angeles time zone and Locale.US in tests by default
4f5df2c is described below
commit 4f5df2c7b46b93687c634fbcd3c18c24aa7988a3
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Sun May 17 02:26:00 2020 +0000
[SPARK-31725][CORE][SQL][TESTS] Set America/Los_Angeles time zone and Locale.US in tests by default
### What changes were proposed in this pull request?
Set default time zone and locale in the default constructor of `SparkFunSuite`:
- Default time zone to `America/Los_Angeles`
- Default locale to `Locale.US`
### Why are the changes needed?
1. To deduplicate code by moving common time zone and locale settings to one place SparkFunSuite
2. To have the same default time zone and locale in all tests. This should prevent errors like https://github.com/apache/spark/pull/28538
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
by running all affected test suites
Closes #28548 from MaxGekk/timezone-settings-SparkFunSuite.
Authored-by: Max Gekk <ma...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 5539ecfdac6c06285c7449494196ea3b4eb4cf87)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../src/test/scala/org/apache/spark/SparkFunSuite.scala | 6 ++++++
.../expressions/SortOrderExpressionsSuite.scala | 11 +----------
.../src/test/scala/org/apache/spark/sql/QueryTest.scala | 7 +------
.../scala/org/apache/spark/sql/SQLQueryTestSuite.scala | 11 +----------
.../sql/hive/execution/HiveCompatibilitySuite.scala | 9 ---------
.../hive/execution/HiveWindowFunctionQuerySuite.scala | 17 -----------------
.../org/apache/spark/sql/hive/HiveInspectorSuite.scala | 6 ------
.../spark/sql/hive/execution/HiveQuerySuite.scala | 11 +----------
8 files changed, 10 insertions(+), 68 deletions(-)
diff --git a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
index cf4400e..ec641f8 100644
--- a/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkFunSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark
// scalastyle:off
import java.io.File
+import java.util.{Locale, TimeZone}
import org.apache.log4j.spi.LoggingEvent
@@ -63,6 +64,11 @@ abstract class SparkFunSuite
with Logging {
// scalastyle:on
+ // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
+ TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
+ // Add Locale setting
+ Locale.setDefault(Locale.US)
+
protected val enableAutoThreadAudit = true
protected override def beforeAll(): Unit = {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
index f269684..4e6976f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/SortOrderExpressionsSuite.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.catalyst.expressions
import java.sql.Timestamp
-import java.util.TimeZone
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.types._
@@ -35,15 +34,7 @@ class SortOrderExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
val l1 = Literal.create(20132983L, LongType)
val l2 = Literal.create(-20132983L, LongType)
val millis = 1524954911000L
- // Explicitly choose a time zone, since Date objects can create different values depending on
- // local time zone of the machine on which the test is running
- val oldDefaultTZ = TimeZone.getDefault
- val d1 = try {
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- Literal.create(new java.sql.Date(millis), DateType)
- } finally {
- TimeZone.setDefault(oldDefaultTZ)
- }
+ val d1 = Literal.create(new java.sql.Date(millis), DateType)
val t1 = Literal.create(new Timestamp(millis), TimestampType)
val f1 = Literal.create(0.7788229f, FloatType)
val f2 = Literal.create(-0.7788229f, FloatType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 4a21ae9..e52d226 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql
-import java.util.{Locale, TimeZone}
+import java.util.TimeZone
import scala.collection.JavaConverters._
@@ -35,11 +35,6 @@ abstract class QueryTest extends PlanTest {
protected def spark: SparkSession
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
-
/**
* Runs the plan and makes sure the answer contains all of the keywords.
*/
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 4a39d95..005cac7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql
import java.io.File
-import java.util.{Locale, TimeZone}
+import java.util.Locale
import java.util.regex.Pattern
import scala.collection.mutable.{ArrayBuffer, HashMap}
@@ -670,23 +670,14 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession {
session.sql("DROP TABLE IF EXISTS tenk1")
}
- private val originalTimeZone = TimeZone.getDefault
- private val originalLocale = Locale.getDefault
-
override def beforeAll(): Unit = {
super.beforeAll()
createTestTables(spark)
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
RuleExecutor.resetMetrics()
}
override def afterAll(): Unit = {
try {
- TimeZone.setDefault(originalTimeZone)
- Locale.setDefault(originalLocale)
removeTestTables(spark)
// For debugging dump some statistics about how much time was spent in various optimizer rules
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index 29825e5..d941c4b 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.hive.execution
import java.io.File
-import java.util.{Locale, TimeZone}
import org.scalatest.BeforeAndAfter
@@ -36,8 +35,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
private lazy val hiveQueryDir = TestHive.getHiveFile(
"ql/src/test/queries/clientpositive".split("/").mkString(File.separator))
- private val originalTimeZone = TimeZone.getDefault
- private val originalLocale = Locale.getDefault
private val originalColumnBatchSize = TestHive.conf.columnBatchSize
private val originalInMemoryPartitionPruning = TestHive.conf.inMemoryPartitionPruning
private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
@@ -51,10 +48,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
override def beforeAll(): Unit = {
super.beforeAll()
TestHive.setCacheTables(true)
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
// Set a relatively small column batch size for testing purposes
TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, 5)
// Enable in-memory partition pruning for testing purposes
@@ -73,8 +66,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
override def afterAll(): Unit = {
try {
TestHive.setCacheTables(false)
- TimeZone.setDefault(originalTimeZone)
- Locale.setDefault(originalLocale)
TestHive.setConf(SQLConf.COLUMN_BATCH_SIZE, originalColumnBatchSize)
TestHive.setConf(SQLConf.IN_MEMORY_PARTITION_PRUNING, originalInMemoryPartitionPruning)
TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
index ed23f65..2c0970c 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveWindowFunctionQuerySuite.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.hive.execution
import java.io.File
-import java.util.{Locale, TimeZone}
import org.scalatest.BeforeAndAfter
@@ -33,17 +32,11 @@ import org.apache.spark.util.Utils
* files, every `createQueryTest` calls should explicitly set `reset` to `false`.
*/
class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfter {
- private val originalTimeZone = TimeZone.getDefault
- private val originalLocale = Locale.getDefault
private val testTempDir = Utils.createTempDir()
override def beforeAll(): Unit = {
super.beforeAll()
TestHive.setCacheTables(true)
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
// Create the table used in windowing.q
sql("DROP TABLE IF EXISTS part")
@@ -103,8 +96,6 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
override def afterAll(): Unit = {
try {
TestHive.setCacheTables(false)
- TimeZone.setDefault(originalTimeZone)
- Locale.setDefault(originalLocale)
TestHive.reset()
} finally {
super.afterAll()
@@ -747,17 +738,11 @@ class HiveWindowFunctionQuerySuite extends HiveComparisonTest with BeforeAndAfte
class HiveWindowFunctionQueryFileSuite
extends HiveCompatibilitySuite with BeforeAndAfter {
- private val originalTimeZone = TimeZone.getDefault
- private val originalLocale = Locale.getDefault
private val testTempDir = Utils.createTempDir()
override def beforeAll(): Unit = {
super.beforeAll()
TestHive.setCacheTables(true)
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
// The following settings are used for generating golden files with Hive.
// We have to use kryo to correctly let Hive serialize plans with window functions.
@@ -772,8 +757,6 @@ class HiveWindowFunctionQueryFileSuite
override def afterAll(): Unit = {
try {
TestHive.setCacheTables(false)
- TimeZone.setDefault(originalTimeZone)
- Locale.setDefault(originalLocale)
TestHive.reset()
} finally {
super.afterAll()
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
index 5912992..13c48f3 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.hive
import java.util
-import java.util.{Locale, TimeZone}
import org.apache.hadoop.hive.ql.udf.UDAFPercentile
import org.apache.hadoop.hive.serde2.io.DoubleWritable
@@ -74,11 +73,6 @@ class HiveInspectorSuite extends SparkFunSuite with HiveInspectors {
.get())
}
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
-
val data =
Literal(true) ::
Literal(null) ::
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 1e89db2..63b985f 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.hive.execution
import java.io.File
import java.net.URI
import java.sql.Timestamp
-import java.util.{Locale, TimeZone}
+import java.util.Locale
import scala.util.Try
@@ -47,9 +47,6 @@ case class TestData(a: Int, b: String)
* included in the hive distribution.
*/
class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAndAfter {
- private val originalTimeZone = TimeZone.getDefault
- private val originalLocale = Locale.getDefault
-
import org.apache.spark.sql.hive.test.TestHive.implicits._
private val originalCrossJoinEnabled = TestHive.conf.crossJoinEnabled
@@ -59,10 +56,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
override def beforeAll(): Unit = {
super.beforeAll()
TestHive.setCacheTables(true)
- // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*)
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
- // Add Locale setting
- Locale.setDefault(Locale.US)
// Ensures that cross joins are enabled so that we can test them
TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, true)
}
@@ -70,8 +63,6 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
override def afterAll(): Unit = {
try {
TestHive.setCacheTables(false)
- TimeZone.setDefault(originalTimeZone)
- Locale.setDefault(originalLocale)
sql("DROP TEMPORARY FUNCTION IF EXISTS udtf_count2")
TestHive.setConf(SQLConf.CROSS_JOINS_ENABLED, originalCrossJoinEnabled)
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org