You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2025/07/21 08:14:26 UTC
(spark) branch master updated: [SPARK-52885][SPARK-52886][SPARK-52887][SQL] Implement the hour, minute, and second functions in Scala for the TIME type
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ddea7dd2be71 [SPARK-52885][SPARK-52886][SPARK-52887][SQL] Implement the hour, minute, and second functions in Scala for the TIME type
ddea7dd2be71 is described below
commit ddea7dd2be7144492d7e72d32d33274e2f36c819
Author: Uros Bojanic <ur...@databricks.com>
AuthorDate: Mon Jul 21 10:14:13 2025 +0200
[SPARK-52885][SPARK-52886][SPARK-52887][SQL] Implement the hour, minute, and second functions in Scala for the TIME type
### What changes were proposed in this pull request?
Add support for TIME type for the `hour`, `minute` and `second` functions in Scala API.
### Why are the changes needed?
Expand API support for the `hour`, `minute` and `second` expressions / functions for the TIME data type.
### Does this PR introduce _any_ user-facing change?
Yes, the support for TIME data type is now available with these functions in Scala API.
### How was this patch tested?
Added appropriate Scala functions tests in TimeFunctionsSuite.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51576 from uros-db/scala-hr_min_sec.
Authored-by: Uros Bojanic <ur...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../scala/org/apache/spark/sql/functions.scala | 6 +-
.../apache/spark/sql/TimeFunctionsSuiteBase.scala | 139 +++++++++++++++++++++
2 files changed, 142 insertions(+), 3 deletions(-)
diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
index ce5c76807b5c..c431b093599e 100644
--- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala
@@ -5400,7 +5400,7 @@ object functions {
def dayofyear(e: Column): Column = Column.fn("dayofyear", e)
/**
- * Extracts the hours as an integer from a given date/timestamp/string.
+ * Extracts the hours as an integer from a given date/time/timestamp/string.
* @return
* An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
@@ -5473,7 +5473,7 @@ object functions {
def last_day(e: Column): Column = Column.fn("last_day", e)
/**
- * Extracts the minutes as an integer from a given date/timestamp/string.
+ * Extracts the minutes as an integer from a given date/time/timestamp/string.
* @return
* An integer, or null if the input was a string that could not be cast to a date
* @group datetime_funcs
@@ -5579,7 +5579,7 @@ object functions {
Column.fn("next_day", date, dayOfWeek)
/**
- * Extracts the seconds as an integer from a given date/timestamp/string.
+ * Extracts the seconds as an integer from a given date/time/timestamp/string.
* @return
* An integer, or null if the input was a string that could not be cast to a timestamp
* @group datetime_funcs
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimeFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimeFunctionsSuiteBase.scala
new file mode 100644
index 000000000000..fb80368b3e4f
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TimeFunctionsSuiteBase.scala
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql
+
+import java.time.LocalTime
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.functions._
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.sql.types._
+
+abstract class TimeFunctionsSuiteBase extends QueryTest with SharedSparkSession {
+ import testImplicits._
+
+ test("SPARK-52885: hour function") {
+ // Input data for the function.
+ val schema = StructType(Seq(
+ StructField("time", TimeType(), nullable = false)
+ ))
+ val data = Seq(
+ Row(LocalTime.parse("00:00:00")),
+ Row(LocalTime.parse("01:02:03.4")),
+ Row(LocalTime.parse("23:59:59.999999"))
+ )
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
+
+ // Test the function using both `selectExpr` and `select`.
+ val result1 = df.selectExpr(
+ "hour(time)"
+ )
+ val result2 = df.select(
+ hour(col("time"))
+ )
+ // Check that both methods produce the same result.
+ checkAnswer(result1, result2)
+
+ // Expected output of the function.
+ val expected = Seq(
+ 0,
+ 1,
+ 23
+ ).toDF("hour").select(col("hour"))
+ // Check that the results match the expected output.
+ checkAnswer(result1, expected)
+ checkAnswer(result2, expected)
+ }
+
+ test("SPARK-52886: minute function") {
+ // Input data for the function.
+ val schema = StructType(Seq(
+ StructField("time", TimeType(), nullable = false)
+ ))
+ val data = Seq(
+ Row(LocalTime.parse("00:00:00")),
+ Row(LocalTime.parse("01:02:03.4")),
+ Row(LocalTime.parse("23:59:59.999999"))
+ )
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
+
+ // Test the function using both `selectExpr` and `select`.
+ val result1 = df.selectExpr(
+ "minute(time)"
+ )
+ val result2 = df.select(
+ minute(col("time"))
+ )
+ // Check that both methods produce the same result.
+ checkAnswer(result1, result2)
+
+ // Expected output of the function.
+ val expected = Seq(
+ 0,
+ 2,
+ 59
+ ).toDF("minute").select(col("minute"))
+ // Check that the results match the expected output.
+ checkAnswer(result1, expected)
+ checkAnswer(result2, expected)
+ }
+
+ test("SPARK-52887: second function") {
+ // Input data for the function.
+ val schema = StructType(Seq(
+ StructField("time", TimeType(), nullable = false)
+ ))
+ val data = Seq(
+ Row(LocalTime.parse("00:00:00")),
+ Row(LocalTime.parse("01:02:03.4")),
+ Row(LocalTime.parse("23:59:59.999999"))
+ )
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
+
+ // Test the function using both `selectExpr` and `select`.
+ val result1 = df.selectExpr(
+ "second(time)"
+ )
+ val result2 = df.select(
+ second(col("time"))
+ )
+ // Check that both methods produce the same result.
+ checkAnswer(result1, result2)
+
+ // Expected output of the function.
+ val expected = Seq(
+ 0,
+ 3,
+ 59
+ ).toDF("second").select(col("second"))
+ // Check that the results match the expected output.
+ checkAnswer(result1, expected)
+ checkAnswer(result2, expected)
+ }
+}
+
+// This class is used to run the same tests with ANSI mode enabled explicitly.
+class TimeFunctionsAnsiOnSuite extends TimeFunctionsSuiteBase {
+ override def sparkConf: SparkConf = super.sparkConf.set(SQLConf.ANSI_ENABLED.key, "true")
+}
+
+// This class is used to run the same tests with ANSI mode disabled explicitly.
+class TimeFunctionsAnsiOffSuite extends TimeFunctionsSuiteBase {
+ override def sparkConf: SparkConf = super.sparkConf.set(SQLConf.ANSI_ENABLED.key, "false")
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org