You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ph...@apache.org on 2024/06/07 09:53:29 UTC
(incubator-gluten) branch main updated: [VL] Update to_utc_timestamp and from_utc_timestamp tests (#5358)
This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new dc853f6af [VL] Update to_utc_timestamp and from_utc_timestamp tests (#5358)
dc853f6af is described below
commit dc853f6af7ed569dce16d1f44f374642755c6207
Author: Ankita Victor <an...@microsoft.com>
AuthorDate: Fri Jun 7 15:23:24 2024 +0530
[VL] Update to_utc_timestamp and from_utc_timestamp tests (#5358)
---
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
.../gluten/utils/velox/VeloxTestSettings.scala | 8 ---
.../spark/sql/GlutenDateFunctionsSuite.scala | 76 ----------------------
8 files changed, 336 deletions(-)
diff --git a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 5762855c2..5df53953e 100644
--- a/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark32/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -265,14 +265,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDataFrameFunctionsSuite]
// blocked by Velox-5768
.exclude("aggregate function - array for primitive type containing null")
diff --git a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index 154e67ae3..8d1f7320d 100644
--- a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++ b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -248,80 +248,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 898fc2b39..79357e9e2 100644
--- a/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark33/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1070,14 +1070,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
--- a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++ b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index bd437bbe8..c532355df 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1073,14 +1073,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
--- a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++ b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index af8d0dead..104c22dbe 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -1090,14 +1090,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("to_timestamp")
// Legacy mode is not supported, assuming this mode is not commonly used.
.exclude("SPARK-30668: use legacy timestamp parser in to_timestamp")
- // Replaced by another test.
- .exclude("to_utc_timestamp with literal zone")
- // Replaced by another test.
- .exclude("to_utc_timestamp with column zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with literal zone")
- // Replaced by another test
- .exclude("from_utc_timestamp with column zone")
enableSuite[GlutenDeprecatedAPISuite]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOff]
enableSuite[GlutenDynamicPartitionPruningV1SuiteAEOn]
diff --git a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
index fbbfdf45d..a946e6de4 100644
--- a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
+++ b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenDateFunctionsSuite.scala
@@ -246,80 +246,4 @@ class GlutenDateFunctionsSuite extends DateFunctionsSuite with GlutenSQLTestsTra
}
}
}
-
- testGluten("to_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-25 07:00:00")))
- )
- }
-
- testGluten("to_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "America/Los_Angeles"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "Europe/Paris")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(to_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- checkAnswer(
- df.select(to_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 07:00:00")),
- Row(Timestamp.valueOf("2015-07-24 22:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with literal zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00")
- ).toDF("a", "b")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), "America/Los_Angeles")),
- Seq(
- Row(Timestamp.valueOf("2015-07-23 17:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
-
- testGluten("from_utc_timestamp with column zone") {
- val df = Seq(
- (Timestamp.valueOf("2015-07-24 00:00:00"), "2015-07-24 00:00:00", "Europe/Paris"),
- (Timestamp.valueOf("2015-07-25 00:00:00"), "2015-07-25 00:00:00", "America/Los_Angeles")
- ).toDF("a", "b", "c")
- checkAnswer(
- df.select(from_utc_timestamp(col("a"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- checkAnswer(
- df.select(from_utc_timestamp(col("b"), col("c"))),
- Seq(
- Row(Timestamp.valueOf("2015-07-24 02:00:00")),
- Row(Timestamp.valueOf("2015-07-24 17:00:00")))
- )
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org