You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ya...@apache.org on 2022/03/25 10:14:09 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2207] Support newly added spark data types: DayTimeIntervalType/YearMonthIntervalType
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d0c92ca [KYUUBI #2207] Support newly added spark data types: DayTimeIntervalType/YearMonthIntervalType
d0c92ca is described below
commit d0c92caa2528e7d359026dcb2e838f2e92738537
Author: Nick Song <ch...@163.com>
AuthorDate: Fri Mar 25 18:13:57 2022 +0800
[KYUUBI #2207] Support newly added spark data types: DayTimeIntervalType/YearMonthIntervalType
### _Why are the changes needed?_
#2207
Support newly added datatypes
spark3.2 : DayTimeIntervalType/YearMonthIntervalType
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2212 from Nick-0723/2207.
Closes #2207
c08ee994 [Nick Song] revert DayTimeIntervalType's part
abc56be5 [Nick Song] support newly data types DayTimeIntervalType/YearMonthIntervalType/TimestampNTZType
Authored-by: Nick Song <ch...@163.com>
Signed-off-by: Kent Yao <ya...@apache.org>
---
.../engine/spark/schema/IntervalQualifier.scala | 66 ------------------
.../apache/kyuubi/engine/spark/schema/RowSet.scala | 16 ++---
.../kyuubi/engine/spark/schema/SchemaHelper.scala | 4 +-
.../scala/org/apache/kyuubi/util/RowSetUtils.scala | 30 +++++++-
.../apache/kyuubi/operation/SparkQueryTests.scala | 80 +++++++++++++++++++---
5 files changed, 105 insertions(+), 91 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/IntervalQualifier.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/IntervalQualifier.scala
deleted file mode 100644
index 13f5ff3..0000000
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/IntervalQualifier.scala
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kyuubi.engine.spark.schema
-
-import java.time.Duration
-import java.util.concurrent.TimeUnit
-
-object IntervalQualifier extends Enumeration {
-
- type IntervalQualifier = Value
-
- val DAY = new Val(0, "interval day")
- val HOUR = new Val(1, "interval day to hour")
- val MINUTE = new Val(2, "interval day to minute")
- val SECOND = new Val(3, "interval day to second")
- final private val SECOND_PER_MINUTE: Long = 60L
- final private val SECOND_PER_HOUR: Long = SECOND_PER_MINUTE * 60L
- final private val SECOND_PER_DAY: Long = SECOND_PER_HOUR * 24L
-
- def toDayTimeIntervalString(d: Duration, iq: IntervalQualifier): String = {
- var sign = ""
- var rest = d.getSeconds
- if (d.getSeconds < 0) {
- sign = "-"
- rest = -rest
- }
- iq match {
- case DAY =>
- val days = TimeUnit.SECONDS.toDays(rest)
- s"$sign$days 00:00:00.000000000"
- case HOUR =>
- val days = TimeUnit.SECONDS.toDays(rest)
- val hours = TimeUnit.SECONDS.toHours(rest % SECOND_PER_DAY)
- f"$sign$days $hours%02d:00:00.000000000"
- case MINUTE =>
- val days = TimeUnit.SECONDS.toDays(rest)
- rest %= SECOND_PER_DAY
- val hours = TimeUnit.SECONDS.toHours(rest)
- val minutes = TimeUnit.SECONDS.toMinutes(rest % SECOND_PER_HOUR)
- f"$sign$days $hours%02d:$minutes%02d:00.000000000"
- case SECOND =>
- val days = TimeUnit.SECONDS.toDays(rest)
- rest %= SECOND_PER_DAY
- val hours = TimeUnit.SECONDS.toHours(rest)
- rest %= SECOND_PER_HOUR
- val minutes = TimeUnit.SECONDS.toMinutes(rest)
- val seconds = rest % SECOND_PER_MINUTE
- f"$sign$days $hours%02d:$minutes%02d:$seconds%02d.${d.getNano}%09d"
- }
- }
-}
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
index 57367a8..33d1268 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
@@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.schema
import java.nio.ByteBuffer
import java.nio.charset.StandardCharsets
import java.sql.Timestamp
-import java.time.{Duration, Instant, LocalDate, ZoneId}
+import java.time._
import java.util.Date
import scala.collection.JavaConverters._
@@ -29,7 +29,6 @@ import org.apache.hive.service.rpc.thrift._
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
-import org.apache.kyuubi.engine.spark.schema.IntervalQualifier.{DAY, HOUR, MINUTE, SECOND}
import org.apache.kyuubi.util.RowSetUtils._
object RowSet {
@@ -255,16 +254,9 @@ object RowSet {
// Only match string in nested type values
"\"" + s + "\""
- case (d: Duration, dt) =>
- if (dt.simpleString == DAY.toString()) {
- IntervalQualifier.toDayTimeIntervalString(d, DAY)
- } else if (dt.simpleString == HOUR.toString()) {
- IntervalQualifier.toDayTimeIntervalString(d, HOUR)
- } else if (dt.simpleString == MINUTE.toString()) {
- IntervalQualifier.toDayTimeIntervalString(d, MINUTE)
- } else {
- IntervalQualifier.toDayTimeIntervalString(d, SECOND)
- }
+ case (d: Duration, _) => toDayTimeIntervalString(d)
+
+ case (p: Period, _) => toYearMonthIntervalString(p)
case (seq: scala.collection.Seq[_], ArrayType(typ, _)) =>
seq.map(v => (v, typ)).map(e => toHiveString(e, timeZone)).mkString("[", ",", "]")
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
index 62cb7b1..055b3ff 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
@@ -41,8 +41,10 @@ object SchemaHelper {
case TimestampType => TTypeId.TIMESTAMP_TYPE
case BinaryType => TTypeId.BINARY_TYPE
case CalendarIntervalType => TTypeId.STRING_TYPE
- case dt if dt.simpleString.startsWith("interval day") =>
+ case dt if dt.getClass.getSimpleName.equals("DayTimeIntervalType") =>
TTypeId.INTERVAL_DAY_TIME_TYPE
+ case ym if ym.getClass.getSimpleName.equals("YearMonthIntervalType") =>
+ TTypeId.INTERVAL_YEAR_MONTH_TYPE
case _: ArrayType => TTypeId.ARRAY_TYPE
case _: MapType => TTypeId.MAP_TYPE
case _: StructType => TTypeId.STRUCT_TYPE
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
index 38db271..6eb6b62 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala
@@ -19,12 +19,13 @@ package org.apache.kyuubi.util
import java.nio.ByteBuffer
import java.sql.Timestamp
-import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
+import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneId}
import java.time.chrono.IsoChronology
import java.time.format.DateTimeFormatter
import java.time.format.DateTimeFormatterBuilder
import java.time.temporal.ChronoField
import java.util.{Date, Locale}
+import java.util.concurrent.TimeUnit
import scala.language.implicitConversions
@@ -32,6 +33,10 @@ import org.apache.commons.lang3.time.FastDateFormat
private[kyuubi] object RowSetUtils {
+ final private val SECOND_PER_MINUTE: Long = 60L
+ final private val SECOND_PER_HOUR: Long = SECOND_PER_MINUTE * 60L
+ final private val SECOND_PER_DAY: Long = SECOND_PER_HOUR * 24L
+
private lazy val dateFormatter = {
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
.toFormatter(Locale.US)
@@ -79,4 +84,27 @@ private[kyuubi] object RowSetUtils {
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
ByteBuffer.wrap(bitSet.toByteArray)
}
+
+ def toDayTimeIntervalString(d: Duration): String = {
+ var rest = d.getSeconds
+ var sign = ""
+ if (d.getSeconds < 0) {
+ sign = "-"
+ rest = -rest
+ }
+ val days = TimeUnit.SECONDS.toDays(rest)
+ rest %= SECOND_PER_DAY
+ val hours = TimeUnit.SECONDS.toHours(rest)
+ rest %= SECOND_PER_HOUR
+ val minutes = TimeUnit.SECONDS.toMinutes(rest)
+ val seconds = rest % SECOND_PER_MINUTE
+ f"$sign$days $hours%02d:$minutes%02d:$seconds%02d.${d.getNano}%09d"
+ }
+
+ def toYearMonthIntervalString(d: Period): String = {
+ val years = d.getYears
+ val months = d.getMonths
+ val sign = if (years < 0 || months < 0) "-" else ""
+ s"$sign${Math.abs(years)}-${Math.abs(months)}"
+ }
}
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
index ed4fe76..96fd8d6 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala
@@ -24,10 +24,12 @@ import scala.collection.JavaConverters._
import org.apache.commons.lang3.StringUtils
import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsReq, TOpenSessionReq, TStatusCode}
-import org.apache.kyuubi.KYUUBI_VERSION
+import org.apache.kyuubi.{KYUUBI_VERSION, Utils}
trait SparkQueryTests extends HiveJDBCTestHelper {
+ protected lazy val SPARK_ENGINE_MAJOR_MINOR_VERSION: (Int, Int) = sparkEngineMajorMinorVersion
+
test("execute statement - select null") {
withJdbcStatement() { statement =>
val resultSet = statement.executeQuery("SELECT NULL AS col")
@@ -175,25 +177,70 @@ trait SparkQueryTests extends HiveJDBCTestHelper {
test("execute statement - select daytime interval") {
withJdbcStatement() { statement =>
Map(
- "-interval 2 day" -> "-2 00:00:00.000000000",
- "-interval 200 day" -> "-200 00:00:00.000000000",
- "interval 1 day 1 hour" -> "1 01:00:00.000000000",
- "interval 1 day 1 hour -60 minutes" -> "1 00:00:00.000000000",
- "interval 1 day 1 hour -60 minutes 30 seconds" -> "1 00:00:30.000000000",
+ "interval 1 day 1 hour -60 minutes 30 seconds" ->
+ Tuple2("1 00:00:30.000000000", "1 days 30 seconds"),
+ "interval 30 seconds 12345 milliseconds" ->
+ Tuple2("0 00:00:42.345000000", "42.345 seconds"),
+ "interval 1 hour 59 minutes 30 seconds 12345 milliseconds" ->
+ Tuple2("0 01:59:42.345000000", "1 hours 59 minutes 42.345 seconds"),
+ "-interval 2 day" -> Tuple2("-2 00:00:00.000000000", "-2 days"),
+ "interval 59 minutes 30 seconds 12345 milliseconds" ->
+ Tuple2("0 00:59:42.345000000", "59 minutes 42.345 seconds"),
+ "interval 25 hour" -> Tuple2("1 01:00:00.000000000", "25 hours"),
+ "interval 1 hour 62 minutes" -> Tuple2("0 02:02:00.000000000", "2 hours 2 minutes"),
"interval 1 day 1 hour 59 minutes 30 seconds 12345 milliseconds" ->
- "1 01:59:42.345000000").foreach { kv => // value -> result pair
+ Tuple2("1 01:59:42.345000000", "1 days 1 hours 59 minutes 42.345 seconds"),
+ "interval 1 day 1 hour -60 minutes" -> Tuple2("1 00:00:00.000000000", "1 days"),
+ "INTERVAL 30 SECONDS" -> Tuple2("0 00:00:30.000000000", "30 seconds"),
+ "interval -60 minutes 30 seconds" ->
+ Tuple2("-0 00:59:30.000000000", "-59 minutes -30 seconds"),
+ "-interval 200 day" -> Tuple2("-200 00:00:00.000000000", "-200 days"),
+ "interval 1 hour -60 minutes 30 seconds" -> Tuple2("0 00:00:30.000000000", "30 seconds"),
+ "interval 62 minutes" -> Tuple2("0 01:02:00.000000000", "1 hours 2 minutes"),
+ "interval 1 day 1 hour" -> Tuple2("1 01:00:00.000000000", "1 days 1 hours")).foreach {
+ kv => // value -> result pair
+ val resultSet = statement.executeQuery(s"SELECT ${kv._1} AS col")
+ assert(resultSet.next())
+ val result = resultSet.getString("col")
+ val metaData = resultSet.getMetaData
+ if (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3
+ && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 < 2) {
+ // for spark 3.1 and backwards
+ assert(result === kv._2._2)
+ assert(metaData.getPrecision(1) === Int.MaxValue)
+ assert(resultSet.getMetaData.getColumnType(1) === java.sql.Types.VARCHAR)
+ } else {
+ assert(result === kv._2._1)
+ assert(metaData.getPrecision(1) === 29)
+ assert(resultSet.getMetaData.getColumnType(1) === java.sql.Types.OTHER)
+ }
+ assert(metaData.getScale(1) === 0)
+ }
+ }
+ }
+
+ test("execute statement - select year/month interval") {
+ withJdbcStatement() { statement =>
+ Map(
+ "INTERVAL 2022 YEAR" -> Tuple2("2022-0", "2022 years"),
+ "INTERVAL '2021-07' YEAR TO MONTH" -> Tuple2("2021-7", "2021 years 7 months"),
+ "INTERVAL 3 MONTH" -> Tuple2("0-3", "3 months"),
+ "INTERVAL 241 MONTH" -> Tuple2("20-1", "20 years 1 months"),
+ "INTERVAL -1 year -25 MONTH" -> Tuple2("-3-1", "-3 years -1 months"),
+ "INTERVAL 3 year -25 MONTH" -> Tuple2("0-11", "11 months")).foreach { kv =>
val resultSet = statement.executeQuery(s"SELECT ${kv._1} AS col")
assert(resultSet.next())
val result = resultSet.getString("col")
val metaData = resultSet.getMetaData
- if (result.contains("days")) {
+ if (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3
+ && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 < 2) {
// for spark 3.1 and backwards
- assert(result.split("days").head.trim === kv._2.split(" ").head)
+ assert(result === kv._2._2)
assert(metaData.getPrecision(1) === Int.MaxValue)
assert(resultSet.getMetaData.getColumnType(1) === java.sql.Types.VARCHAR)
} else {
- assert(result === kv._2)
- assert(metaData.getPrecision(1) === 29)
+ assert(result === kv._2._1)
+ assert(metaData.getPrecision(1) === 11)
assert(resultSet.getMetaData.getColumnType(1) === java.sql.Types.OTHER)
}
assert(metaData.getScale(1) === 0)
@@ -545,4 +592,15 @@ trait SparkQueryTests extends HiveJDBCTestHelper {
assert(foundOperationLangItem)
}
}
+
+ def sparkEngineMajorMinorVersion: (Int, Int) = {
+ var sparkRuntimeVer = ""
+ withJdbcStatement() { stmt =>
+ val result = stmt.executeQuery("SELECT version()")
+ assert(result.next())
+ sparkRuntimeVer = result.getString(1)
+ assert(!result.next())
+ }
+ Utils.majorMinorVersion(sparkRuntimeVer)
+ }
}