You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2024/08/08 13:55:27 UTC
(spark) branch master updated: [SPARK-49138][SQL] Fix CollationTypeCasts of several expressions
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 01a6991a338c [SPARK-49138][SQL] Fix CollationTypeCasts of several expressions
01a6991a338c is described below
commit 01a6991a338cc3f0eac61e97470ef9dbdb170971
Author: Mihailo Milosevic <mi...@databricks.com>
AuthorDate: Thu Aug 8 21:55:12 2024 +0800
[SPARK-49138][SQL] Fix CollationTypeCasts of several expressions
### What changes were proposed in this pull request?
Fix for CreateMap and ArrayAppend expressions.
### Why are the changes needed?
While adding TypeCoercion for collations these two expressions were missed out, so this PR adds them to the rules so that they follow expected behaviour.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added testcases in CollationSuite.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #47649 from mihailom-db/fixtypecoercion.
Authored-by: Mihailo Milosevic <mi...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/catalyst/analysis/CollationTypeCasts.scala | 9 ++++-
.../org/apache/spark/sql/CollationSuite.scala | 38 +++++++++++++++++++++-
2 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
index 276062ce211d..9c7b5aaecd78 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CollationTypeCasts.scala
@@ -83,10 +83,17 @@ object CollationTypeCasts extends TypeCoercionRule {
val Seq(newInput, newDefault) = collateToSingleType(Seq(input, default))
framelessOffsetWindow.withNewChildren(Seq(newInput, offset, newDefault))
+ case mapCreate : CreateMap if mapCreate.children.size % 2 == 0 =>
+ // We only take in mapCreate if it has even number of children, as otherwise it should fail
+ // with wrong number of arguments
+ val newKeys = collateToSingleType(mapCreate.keys)
+ val newValues = collateToSingleType(mapCreate.values)
+ mapCreate.withNewChildren(newKeys.zip(newValues).flatMap(pair => Seq(pair._1, pair._2)))
+
case otherExpr @ (
_: In | _: InSubquery | _: CreateArray | _: ArrayJoin | _: Concat | _: Greatest | _: Least |
_: Coalesce | _: ArrayContains | _: ArrayExcept | _: ConcatWs | _: Mask | _: StringReplace |
- _: StringTranslate | _: StringTrim | _: StringTrimLeft | _: StringTrimRight |
+ _: StringTranslate | _: StringTrim | _: StringTrimLeft | _: StringTrimRight | _: ArrayAppend |
_: ArrayIntersect | _: ArrayPosition | _: ArrayRemove | _: ArrayUnion | _: ArraysOverlap |
_: Contains | _: EndsWith | _: EqualNullSafe | _: EqualTo | _: FindInSet | _: GreaterThan |
_: GreaterThanOrEqual | _: LessThan | _: LessThanOrEqual | _: StartsWith | _: StringInstr |
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
index e9e3432195a4..dd678ac48c68 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CollationSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAg
import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec
import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.internal.{SqlApiConf, SQLConf}
-import org.apache.spark.sql.types.{MapType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, MapType, StringType, StructField, StructType}
class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
protected val v2Source = classOf[FakeV2ProviderWithCustomSchema].getName
@@ -579,6 +579,42 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
}
}
+ test("SPARK-49138: ArrayAppend and CreateMap coercion testing") {
+ val df_array_append = sql(s"SELECT array_append(array('a', 'b'), 'c' COLLATE UNICODE)")
+ // array_append expression
+ checkAnswer(df_array_append, Seq(Row(Seq("a", "b", "c"))))
+ assert(df_array_append.schema.head.dataType == ArrayType(StringType("UNICODE"), true))
+
+ // make sure we fail this query even when collations are in
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("select map('a' COLLATE UTF8_LCASE, 'b', 'c')")
+ },
+ errorClass = "WRONG_NUM_ARGS.WITHOUT_SUGGESTION",
+ parameters = Map("functionName" -> "`map`", "expectedNum" -> "2n (n > 0)",
+ "actualNum" -> "3", "docroot" -> "https://spark.apache.org/docs/latest")
+ )
+
+ // make sure we fail this query even when collations are in
+ checkError(
+ exception = intercept[AnalysisException] {
+ sql("select map('a' COLLATE UTF8_LCASE, 'b', 'c' COLLATE UNICODE, 'c')")
+ },
+ errorClass = "COLLATION_MISMATCH.EXPLICIT",
+ sqlState = "42P21",
+ parameters = Map(
+ "explicitTypes" ->
+ s"`string collate UTF8_LCASE`, `string collate UNICODE`"
+ )
+ )
+
+ // map creation keys respects proper collation
+ val df_create_map = sql("select map('a' COLLATE UTF8_LCASE, 'b', 'c', 'c')")
+ checkAnswer(df_create_map, Seq(Row(Map("a" -> "b", "c" -> "c"))))
+ assert(df_create_map.schema.head.dataType ==
+ MapType(StringType("UTF8_LCASE"), StringType("UTF8_BINARY"), false))
+ }
+
test("SPARK-47692: Parameter marker with EXECUTE IMMEDIATE implicit casting") {
sql(s"DECLARE stmtStr1 = 'SELECT collation(:var1 || :var2)';")
sql(s"DECLARE stmtStr2 = 'SELECT collation(:var1 || (\\\'a\\\' COLLATE UNICODE))';")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org