You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2021/07/08 09:23:19 UTC
[spark] branch branch-3.2 updated: [SPARK-36021][SQL][FOLLOWUP]
DT/YM func use field byte to keep consistence
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 2776e8a [SPARK-36021][SQL][FOLLOWUP] DT/YM func use field byte to keep consistence
2776e8a is described below
commit 2776e8aa4792a7b2e95d5bbbd76cea0f9554503c
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Thu Jul 8 12:22:04 2021 +0300
[SPARK-36021][SQL][FOLLOWUP] DT/YM func use field byte to keep consistence
### What changes were proposed in this pull request?
With more thought, all DT/YM function use field byte to keep consistence is better
### Why are the changes needed?
Keep code consistence
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Not need
Closes #33252 from AngersZhuuuu/SPARK-36021-FOLLOWUP.
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
(cherry picked from commit 89aa16b4a838246cfb7bdc9318461485016f6252)
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../spark/sql/catalyst/parser/AstBuilder.scala | 17 ++++-----------
.../spark/sql/catalyst/util/IntervalUtils.scala | 24 ++++++++++------------
.../catalyst/parser/ExpressionParserSuite.scala | 3 ++-
.../sql/catalyst/util/IntervalUtilsSuite.scala | 13 ++++--------
4 files changed, 21 insertions(+), 36 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index d6363b5..4f1e53f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -40,7 +40,6 @@ import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.trees.CurrentOrigin
import org.apache.spark.sql.catalyst.util.{CharVarcharUtils, DateTimeUtils, IntervalUtils}
import org.apache.spark.sql.catalyst.util.DateTimeUtils.{convertSpecialDate, convertSpecialTimestamp, convertSpecialTimestampNTZ, getZoneId, stringToDate, stringToTimestamp, stringToTimestampWithoutTimeZone}
-import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit
import org.apache.spark.sql.connector.catalog.{SupportsNamespaces, TableCatalog}
import org.apache.spark.sql.connector.catalog.TableChange.ColumnPosition
import org.apache.spark.sql.connector.expressions.{ApplyTransform, BucketTransform, DaysTransform, Expression => V2Expression, FieldReference, HoursTransform, IdentityTransform, LiteralValue, MonthsTransform, Transform, YearsTransform}
@@ -2487,18 +2486,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
(from, to) match {
case ("year", "month") =>
IntervalUtils.fromYearMonthString(value)
- case ("day", "hour") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.HOUR)
- case ("day", "minute") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.MINUTE)
- case ("day", "second") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.DAY, IntervalUnit.SECOND)
- case ("hour", "minute") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.HOUR, IntervalUnit.MINUTE)
- case ("hour", "second") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.HOUR, IntervalUnit.SECOND)
- case ("minute", "second") =>
- IntervalUtils.fromDayTimeString(value, IntervalUnit.MINUTE, IntervalUnit.SECOND)
+ case ("day", "hour") | ("day", "minute") | ("day", "second") | ("hour", "minute") |
+ ("hour", "second") | ("minute", "second") =>
+ IntervalUtils.fromDayTimeString(value,
+ DayTimeIntervalType.stringToField(from), DayTimeIntervalType.stringToField(to))
case _ =>
throw QueryParsingErrors.fromToIntervalUnsupportedError(from, to, ctx)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 24bcad8..7579a28 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -458,7 +458,7 @@ object IntervalUtils {
* adapted from HiveIntervalDayTime.valueOf
*/
def fromDayTimeString(s: String): CalendarInterval = {
- fromDayTimeString(s, DAY, SECOND)
+ fromDayTimeString(s, DT.DAY, DT.SECOND)
}
/**
@@ -470,13 +470,12 @@ object IntervalUtils {
* - HOUR TO (HOUR|MINUTE|SECOND)
* - MINUTE TO (MINUTE|SECOND)
*/
- def fromDayTimeString(input: String, from: IntervalUnit, to: IntervalUnit): CalendarInterval = {
+ def fromDayTimeString(input: String, from: Byte, to: Byte): CalendarInterval = {
require(input != null, "Interval day-time string must be not null")
if (SQLConf.get.getConf(SQLConf.LEGACY_FROM_DAYTIME_STRING)) {
parseDayTimeLegacy(input, from, to)
} else {
- castDayTimeStringToInterval(
- input, DT.stringToField(from.toString), DT.stringToField(to.toString))
+ castDayTimeStringToInterval(input, from, to)
}
}
@@ -500,8 +499,8 @@ object IntervalUtils {
*/
private def parseDayTimeLegacy(
input: String,
- from: IntervalUnit,
- to: IntervalUnit): CalendarInterval = {
+ from: Byte,
+ to: Byte): CalendarInterval = {
assert(input.length == input.trim.length)
val m = dayTimePatternLegacy.pattern.matcher(input)
require(m.matches, s"Interval string must match day-time format of 'd h:m:s.n': $input, " +
@@ -517,7 +516,7 @@ object IntervalUtils {
var hours: Long = 0L
var minutes: Long = 0L
var seconds: Long = 0L
- if (m.group(5) != null || from == MINUTE) { // 'HH:mm:ss' or 'mm:ss minute'
+ if (m.group(5) != null || from == DT.MINUTE) { // 'HH:mm:ss' or 'mm:ss minute'
hours = toLongWithRange(HOUR, m.group(5), 0, 23)
minutes = toLongWithRange(MINUTE, m.group(6), 0, 59)
seconds = toLongWithRange(SECOND, m.group(7), 0, 59)
@@ -531,18 +530,17 @@ object IntervalUtils {
// Hive allow nanosecond precision interval
var secondsFraction = parseNanos(m.group(9), seconds < 0)
to match {
- case HOUR =>
+ case DT.HOUR =>
minutes = 0
seconds = 0
secondsFraction = 0
- case MINUTE =>
+ case DT.MINUTE =>
seconds = 0
secondsFraction = 0
- case SECOND =>
+ case DT.SECOND =>
// No-op
- case _ =>
- throw new IllegalArgumentException(
- s"Cannot support (interval '$input' $from to $to) expression")
+ case _ => throw new IllegalArgumentException(s"Cannot support (" +
+ s"interval '$input' ${DT.fieldToString(from)} to ${DT.fieldToString(to)}) expression")
}
var micros = secondsFraction
micros = Math.addExact(micros, Math.multiplyExact(hours, MICROS_PER_HOUR))
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
index 818afc4..aa1a451 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala
@@ -770,7 +770,8 @@ class ExpressionParserSuite extends AnalysisTest {
"0:0:0",
"0:0:1")
hourTimeValues.foreach { value =>
- val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND))
+ val result = Literal(IntervalUtils.fromDayTimeString(
+ value, DayTimeIntervalType.HOUR, DayTimeIntervalType.SECOND))
checkIntervals(s"'$value' hour to second", result)
}
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index 93e3ead..62059c0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
import org.apache.spark.sql.catalyst.util.IntervalStringStyles.{ANSI_STYLE, HIVE_STYLE}
import org.apache.spark.sql.catalyst.util.IntervalUtils._
-import org.apache.spark.sql.catalyst.util.IntervalUtils.IntervalUnit._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.DayTimeIntervalType
import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -203,8 +202,6 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
failFuncWithInvalidInput("5 30:12:20", "hour 30 outside range", fromDayTimeString)
failFuncWithInvalidInput("5 30-12", "must match day-time format", fromDayTimeString)
- failFuncWithInvalidInput("5 1:12:20", "Cannot support (interval",
- s => fromDayTimeString(s, HOUR, MICROSECOND))
}
}
@@ -314,13 +311,14 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
}
test("from day-time string") {
- def check(input: String, from: IntervalUnit, to: IntervalUnit, expected: String): Unit = {
+ import org.apache.spark.sql.types.DayTimeIntervalType._
+ def check(input: String, from: Byte, to: Byte, expected: String): Unit = {
withClue(s"from = $from, to = $to") {
val expectedUtf8 = UTF8String.fromString(expected)
assert(fromDayTimeString(input, from, to) === safeStringToInterval(expectedUtf8))
}
}
- def checkFail(input: String, from: IntervalUnit, to: IntervalUnit, errMsg: String): Unit = {
+ def checkFail(input: String, from: Byte, to: Byte, errMsg: String): Unit = {
failFuncWithInvalidInput(input, errMsg, s => fromDayTimeString(s, from, to))
}
@@ -358,10 +356,7 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
checkFail("5 30:12:20", DAY, SECOND, "hour 30 outside range")
checkFail("5 30-12", DAY, SECOND, "Interval string does not match day-time format")
- withClue("Expected to throw an exception for the invalid input") {
- val e = intercept[NoSuchElementException](fromDayTimeString("5 1:12:20", HOUR, MICROSECOND))
- assert(e.getMessage.contains("key not found: microsecond"))
- }
+
// whitespaces
check("\t +5 12:40\t ", DAY, MINUTE, "5 days 12 hours 40 minutes")
checkFail("+5\t 12:40", DAY, MINUTE, "Interval string does not match day-time format")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org