You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2021/06/12 18:45:50 UTC
[spark] branch master updated: [SPARK-35734][SQL] Format day-time
intervals using type fields
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 80f7989 [SPARK-35734][SQL] Format day-time intervals using type fields
80f7989 is described below
commit 80f7989d9a761eff1a0b3c64ec3aabb81506953d
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Sat Jun 12 21:45:12 2021 +0300
[SPARK-35734][SQL] Format day-time intervals using type fields
### What changes were proposed in this pull request?
This PR add a feature which formats day-time interval to strings using the start and end fields of `DayTimeIntervalType`.
### Why are the changes needed?
Currently, they are ignored, and any `DayTimeIntervalType` is formatted as `INTERVAL DAY TO SECOND.`
### Does this PR introduce _any_ user-facing change?
Yes. The format of day-time intervals is determined the start and end fields.
### How was this patch tested?
New test.
Closes #32891 from sarutak/interval-format.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../spark/sql/catalyst/util/IntervalUtils.scala | 58 +++++++++++++--
.../sql/catalyst/util/IntervalUtilsSuite.scala | 84 ++++++++++++++++++++++
2 files changed, 138 insertions(+), 4 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index c18cca9..dda5581 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -21,6 +21,7 @@ import java.time.{Duration, Period}
import java.time.temporal.ChronoUnit
import java.util.concurrent.TimeUnit
+import scala.collection.mutable
import scala.util.control.NonFatal
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -28,7 +29,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE, HIVE_STYLE, IntervalStyle}
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.Decimal
+import org.apache.spark.sql.types.{DayTimeIntervalType, Decimal}
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
// The style of textual representation of intervals
@@ -960,18 +961,34 @@ object IntervalUtils {
def toDayTimeIntervalString(
micros: Long,
style: IntervalStyle,
- // TODO(SPARK-35734): Format day-time intervals using type fields
startField: Byte,
endField: Byte): String = {
var sign = ""
var rest = micros
+ val from = DayTimeIntervalType.fieldToString(startField).toUpperCase
+ val to = DayTimeIntervalType.fieldToString(endField).toUpperCase
if (micros < 0) {
if (micros == Long.MinValue) {
// Especial handling of minimum `Long` value because negate op overflows `Long`.
// seconds = 106751991 * (24 * 60 * 60) + 4 * 60 * 60 + 54 = 9223372036854
// microseconds = -9223372036854000000L-775808 == Long.MinValue
val minIntervalString = style match {
- case ANSI_STYLE => "INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND"
+ case ANSI_STYLE =>
+ val baseStr = "-106751991 04:00:54.775808"
+ val fromPos = startField match {
+ case DayTimeIntervalType.DAY => 0
+ case DayTimeIntervalType.HOUR => 11
+ case DayTimeIntervalType.MINUTE => 14
+ case DayTimeIntervalType.SECOND => 17
+ }
+ val toPos = endField match {
+ case DayTimeIntervalType.DAY => 10
+ case DayTimeIntervalType.HOUR => 13
+ case DayTimeIntervalType.MINUTE => 16
+ case DayTimeIntervalType.SECOND => baseStr.length
+ }
+ val postfix = if (startField == endField) from else s"$from TO $to"
+ s"INTERVAL '${baseStr.substring(fromPos, toPos)}' $postfix"
case HIVE_STYLE => "-106751991 04:00:54.775808000"
}
return minIntervalString
@@ -992,7 +1009,40 @@ object IntervalUtils {
val secStr = java.math.BigDecimal.valueOf(secondsWithFraction, 6)
.stripTrailingZeros()
.toPlainString()
- f"INTERVAL '$sign$days $hours%02d:$minutes%02d:$leadSecZero$secStr' DAY TO SECOND"
+ val formatBuilder = new StringBuilder("INTERVAL '")
+ if (startField == endField) {
+ startField match {
+ case DayTimeIntervalType.DAY => formatBuilder.append(s"$sign$days' ")
+ case DayTimeIntervalType.HOUR => formatBuilder.append(f"$hours%02d' ")
+ case DayTimeIntervalType.MINUTE => formatBuilder.append(f"$minutes%02d' ")
+ case DayTimeIntervalType.SECOND => formatBuilder.append(s"$leadSecZero$secStr' ")
+ }
+ formatBuilder.append(from).toString
+ } else {
+ val formatArgs = new mutable.ArrayBuffer[Long]
+ if (startField <= DayTimeIntervalType.DAY && DayTimeIntervalType.DAY < endField) {
+ formatBuilder.append(s"$sign$days ")
+ }
+ if (startField <= DayTimeIntervalType.HOUR && DayTimeIntervalType.HOUR < endField) {
+ formatBuilder.append("%02d:")
+ formatArgs.append(hours)
+ }
+ if (startField <= DayTimeIntervalType.MINUTE && DayTimeIntervalType.MINUTE < endField) {
+ formatBuilder.append("%02d:")
+ formatArgs.append(minutes)
+ }
+ endField match {
+ case DayTimeIntervalType.HOUR =>
+ formatBuilder.append("%02d' ")
+ formatArgs.append(hours)
+ case DayTimeIntervalType.MINUTE =>
+ formatBuilder.append("%02d' ")
+ formatArgs.append(minutes)
+ case DayTimeIntervalType.SECOND =>
+ formatBuilder.append(s"$leadSecZero$secStr' ")
+ }
+ formatBuilder.append(s"$from TO $to").toString.format(formatArgs.toSeq: _*)
+ }
case HIVE_STYLE =>
val seconds = secondsWithFraction / MICROS_PER_SECOND
val nanos = (secondsWithFraction % MICROS_PER_SECOND) * NANOS_PER_MICROS
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 8db63e7..0d496c9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -534,4 +534,88 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
assert(toDayTimeIntervalString(micros, HIVE_STYLE, DAY, SECOND) === hiveIntervalStr)
}
}
+
+ test("SPARK-35734: Format day-time intervals using type fields") {
+ import DayTimeIntervalType._
+ Seq(
+ 0L ->
+ ("INTERVAL '0 00:00:00' DAY TO SECOND",
+ "INTERVAL '0 00:00' DAY TO MINUTE",
+ "INTERVAL '0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00' MINUTE TO SECOND",
+ "INTERVAL '0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00' SECOND"),
+ -1L ->
+ ("INTERVAL '-0 00:00:00.000001' DAY TO SECOND",
+ "INTERVAL '-0 00:00' DAY TO MINUTE",
+ "INTERVAL '-0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00.000001' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00.000001' MINUTE TO SECOND",
+ "INTERVAL '-0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00.000001' SECOND"),
+ 10 * MICROS_PER_MILLIS ->
+ ("INTERVAL '0 00:00:00.01' DAY TO SECOND",
+ "INTERVAL '0 00:00' DAY TO MINUTE",
+ "INTERVAL '0 00' DAY TO HOUR",
+ "INTERVAL '00:00:00.01' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:00.01' MINUTE TO SECOND",
+ "INTERVAL '0' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '00.01' SECOND"),
+ (-123 * MICROS_PER_DAY - 3 * MICROS_PER_SECOND) ->
+ ("INTERVAL '-123 00:00:03' DAY TO SECOND",
+ "INTERVAL '-123 00:00' DAY TO MINUTE",
+ "INTERVAL '-123 00' DAY TO HOUR",
+ "INTERVAL '00:00:03' HOUR TO SECOND",
+ "INTERVAL '00:00' HOUR TO MINUTE",
+ "INTERVAL '00:03' MINUTE TO SECOND",
+ "INTERVAL '-123' DAY",
+ "INTERVAL '00' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '03' SECOND"),
+ Long.MinValue ->
+ ("INTERVAL '-106751991 04:00:54.775808' DAY TO SECOND",
+ "INTERVAL '-106751991 04:00' DAY TO MINUTE",
+ "INTERVAL '-106751991 04' DAY TO HOUR",
+ "INTERVAL '04:00:54.775808' HOUR TO SECOND",
+ "INTERVAL '04:00' HOUR TO MINUTE",
+ "INTERVAL '00:54.775808' MINUTE TO SECOND",
+ "INTERVAL '-106751991' DAY",
+ "INTERVAL '04' HOUR",
+ "INTERVAL '00' MINUTE",
+ "INTERVAL '54.775808' SECOND")
+ ).foreach {
+ case (
+ micros, (
+ dayToSec,
+ dayToMinute,
+ dayToHour,
+ hourToSec,
+ hourToMinute,
+ minuteToSec,
+ day,
+ hour,
+ minute,
+ sec)) =>
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, SECOND) === dayToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, MINUTE) === dayToMinute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, HOUR) === dayToHour)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, SECOND) === hourToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, MINUTE) === hourToMinute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, MINUTE, SECOND) === minuteToSec)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, DAY, DAY) === day)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, HOUR, HOUR) === hour)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, MINUTE, MINUTE) === minute)
+ assert(toDayTimeIntervalString(micros, ANSI_STYLE, SECOND, SECOND) === sec)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org