You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ya...@apache.org on 2022/03/31 07:41:36 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2207] Fix DayTimeIntervalType/YearMonthIntervalType Column Size
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d77882a [KYUUBI #2207] Fix DayTimeIntervalType/YearMonthIntervalType Column Size
d77882a is described below
commit d77882a8b5abac16cf25faf42e5c9da2882c0125
Author: Nick Song <ch...@163.com>
AuthorDate: Thu Mar 31 15:41:24 2022 +0800
[KYUUBI #2207] Fix DayTimeIntervalType/YearMonthIntervalType Column Size
### _Why are the changes needed?_
Data type DayTimeIntervalType and YearMonthIntervalType have default column size ,but now is null.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2251 from Nick-0723/fix_interval_columnSize.
Closes #2207
1903da28 [Nick Song] fix spark3.3.0 interval type column size
Authored-by: Nick Song <ch...@163.com>
Signed-off-by: Kent Yao <ya...@apache.org>
---
.../kyuubi/engine/spark/schema/SchemaHelper.scala | 18 +++++++++++++++---
.../engine/spark/operation/SparkOperationSuite.scala | 10 +++++++---
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
index 02103eb..4111757 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala
@@ -31,6 +31,16 @@ object SchemaHelper {
*/
final val TIMESTAMP_NTZ = "TimestampNTZType$"
+ /**
+ * Spark 3.2.0 DataType DayTimeIntervalType's class name.
+ */
+ final val DAY_TIME_INTERVAL = "DayTimeIntervalType"
+
+ /**
+ * Spark 3.2.0 DataType YearMonthIntervalType's class name.
+ */
+ final val YEAR_MONTH_INTERVAL = "YearMonthIntervalType"
+
def toTTypeId(typ: DataType): TTypeId = typ match {
case NullType => TTypeId.NULL_TYPE
case BooleanType => TTypeId.BOOLEAN_TYPE
@@ -47,9 +57,9 @@ object SchemaHelper {
case ntz if ntz.getClass.getSimpleName.equals(TIMESTAMP_NTZ) => TTypeId.TIMESTAMP_TYPE
case BinaryType => TTypeId.BINARY_TYPE
case CalendarIntervalType => TTypeId.STRING_TYPE
- case dt if dt.getClass.getSimpleName.equals("DayTimeIntervalType") =>
+ case dt if dt.getClass.getSimpleName.equals(DAY_TIME_INTERVAL) =>
TTypeId.INTERVAL_DAY_TIME_TYPE
- case ym if ym.getClass.getSimpleName.equals("YearMonthIntervalType") =>
+ case ym if ym.getClass.getSimpleName.equals(YEAR_MONTH_INTERVAL) =>
TTypeId.INTERVAL_YEAR_MONTH_TYPE
case _: ArrayType => TTypeId.ARRAY_TYPE
case _: MapType => TTypeId.MAP_TYPE
@@ -125,7 +135,9 @@ object SchemaHelper {
* For array, map, string, and binaries, the column size is variable, return null as unknown.
*/
def getColumnSize(sparkType: DataType): Option[Int] = sparkType match {
- case ntz if ntz.getClass.getSimpleName.equals(TIMESTAMP_NTZ) => Some(ntz.defaultSize)
+ case dt
+ if Array(TIMESTAMP_NTZ, DAY_TIME_INTERVAL, YEAR_MONTH_INTERVAL)
+ .contains(dt.getClass.getSimpleName) => Some(dt.defaultSize)
case dt @ (BooleanType | _: NumericType | DateType | TimestampType |
CalendarIntervalType | NullType) =>
Some(dt.defaultSize)
diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
index 41b2b66..51a5414 100644
--- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
+++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala
@@ -57,7 +57,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with
test("get columns operation") {
val tableName = "spark_get_col_operation"
- val schema = new StructType()
+ var schema = new StructType()
.add("c0", "boolean", nullable = false, "0")
.add("c1", "tinyint", nullable = true, "1")
.add("c2", "smallint", nullable = false, "2")
@@ -80,7 +80,9 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with
// since spark3.3.0
if (SPARK_ENGINE_MAJOR_MINOR_VERSION._1 > 3 ||
(SPARK_ENGINE_MAJOR_MINOR_VERSION._1 == 3 && SPARK_ENGINE_MAJOR_MINOR_VERSION._2 >= 3)) {
- schema.add("c18", "timestamp_ntz", nullable = true, "18")
+ schema = schema.add("c18", "timestamp_ntz", nullable = true, "18")
+ .add("c19", "interval day", nullable = true, "19")
+ .add("c20", "interval year", nullable = true, "20")
}
val ddl =
@@ -118,7 +120,9 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with
STRUCT,
BINARY,
STRUCT,
- TIMESTAMP)
+ TIMESTAMP,
+ OTHER,
+ OTHER)
var pos = 0