You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/12/22 06:22:31 UTC
[spark] branch branch-3.0 updated: [SPARK-33860][SQL] Make
CatalystTypeConverters.convertToCatalyst match special Array value
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7af54fd [SPARK-33860][SQL] Make CatalystTypeConverters.convertToCatalyst match special Array value
7af54fd is described below
commit 7af54fd76ade27b1cabe90bce1b959c2224f31ad
Author: ulysses-you <ul...@gmail.com>
AuthorDate: Tue Dec 22 15:10:46 2020 +0900
[SPARK-33860][SQL] Make CatalystTypeConverters.convertToCatalyst match special Array value
### What changes were proposed in this pull request?
Add some case to match Array whose element type is primitive.
### Why are the changes needed?
We will get exception when use `Literal.create(Array(1, 2, 3), ArrayType(IntegerType))` .
```
Exception in thread "main" java.lang.IllegalArgumentException: requirement failed: Literal must have a corresponding value to array<int>, but class int[] found.
at scala.Predef$.require(Predef.scala:281)
at org.apache.spark.sql.catalyst.expressions.Literal$.validateLiteralValue(literals.scala:215)
at org.apache.spark.sql.catalyst.expressions.Literal.<init>(literals.scala:292)
at org.apache.spark.sql.catalyst.expressions.Literal$.create(literals.scala:140)
```
And same problem with other array whose element is primitive.
### Does this PR introduce _any_ user-facing change?
Yes.
### How was this patch tested?
Add test.
Closes #30868 from ulysses-you/SPARK-33860.
Authored-by: ulysses-you <ul...@gmail.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit 1dd63dccd893162f8ef969e42273a794ad73e49c)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../apache/spark/sql/catalyst/CatalystTypeConverters.scala | 4 +++-
.../sql/catalyst/expressions/LiteralExpressionSuite.scala | 11 +++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
index 34d2f45..2b467243 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/CatalystTypeConverters.scala
@@ -456,7 +456,9 @@ object CatalystTypeConverters {
case d: JavaBigDecimal => new DecimalConverter(DecimalType(d.precision, d.scale)).toCatalyst(d)
case seq: Seq[Any] => new GenericArrayData(seq.map(convertToCatalyst).toArray)
case r: Row => InternalRow(r.toSeq.map(convertToCatalyst): _*)
- case arr: Array[Any] => new GenericArrayData(arr.map(convertToCatalyst))
+ case arr: Array[Byte] => arr
+ case arr: Array[Char] => StringConverter.toCatalyst(arr)
+ case arr: Array[_] => new GenericArrayData(arr.map(convertToCatalyst))
case map: Map[_, _] =>
ArrayBasedMapData(
map,
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
index bb86135..794e8c0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala
@@ -317,4 +317,15 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
assert(literalStr === expected)
}
}
+
+ test("SPARK-33860: Make CatalystTypeConverters.convertToCatalyst match special Array value") {
+ assert(Literal(Array(1, 2, 3)) == Literal.create(Array(1, 2, 3), ArrayType(IntegerType)))
+ assert(Literal(Array(1L, 2L, 3L)) == Literal.create(Array(1L, 2L, 3L), ArrayType(LongType)))
+ assert(Literal(Array(1D, 2D, 3D)) == Literal.create(Array(1D, 2D, 3D), ArrayType(DoubleType)))
+ assert(Literal("123") == Literal.create(Array('1', '2', '3'), StringType))
+ assert(Literal(Array(1.toByte, 2.toByte, 3.toByte)) ==
+ Literal.create(Array(1.toByte, 2.toByte, 3.toByte), BinaryType))
+ assert(Literal(Array("1", "2", "3")) ==
+ Literal.create(Array("1", "2", "3"), ArrayType(StringType)))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org