(spark) branch master updated: [SPARK-51576][SQL] Prohibit ANSI cast from `array` to `array<_, containsNull = false>`

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 084143b977a9 [SPARK-51576][SQL] Prohibit ANSI cast from `array<variant, containsNull = false>` to `array<_, containsNull = false>`
084143b977a9 is described below

commit 084143b977a9c1a8eee50aea77125f29fa5267d0
Author: Harsh Motwani <ha...@databricks.com>
AuthorDate: Mon Mar 24 21:29:58 2025 +0800

    [SPARK-51576][SQL] Prohibit ANSI cast from `array<variant, containsNull = false>` to `array<_, containsNull = false>`
    
    ### What changes were proposed in this pull request?
    
    This PR prohibits ANSI cast from `ArrayType(VariantType, containsNull = false)` to `ArrayType(_, containsNull = false)`, or `StructField(VariantType, nullable = false)` to `StructField(_, nullable = true)` or `MapType(_, VariantType, valueContainsNull = false)` to `MapType(_, _, valueContainsNull = false)`.
    
    ### Why are the changes needed?
    
    Variant is the only data type which can be ANSI cast from a non-null value to a null value. More specifically, variant nulls (generated using `parse_json('null')`) when cast to string result in `null`.
    However, `array<variant, containsNull = false>` can be cast to `array<string, containsNull = true>`. This should not be allowed since `array(parse_json('null'))` would give `array(null)` after this cast.
    Here is a demonstration of this problem where we create an `array<string, containsNull = true>` which in fact does not contain nulls:
    ```
    >>> from pyspark.sql.functions import col
    >>> from pyspark.sql.types import StringType, VariantType, ArrayType
    >>> 
    >>> df = spark.sql("select array(parse_json('null')) arr")
    >>> df.printSchema()
    root
     |-- arr: array (nullable = false)
     |    |-- element: variant (containsNull = false)
    
    >>> df2 = df.select(col('arr').cast(ArrayType(StringType(), False)))
    >>> df2.selectExpr("arr[0] is null").show()
    +----------------+
    |(arr[0] IS NULL)|
    +----------------+
    |            true|
    +----------------+
    
    >>> df2.printSchema()
    root
     |-- arr: array (nullable = false)
     |    |-- element: string (containsNull = false)
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, users can no longer ANSI cast from `array<variant, containsNull = false>` to `array<_, containsNull = false>`. However, it is unlikely that users were hardcoding `containsNull = false`. The same is the case for struct field and maps.
    
    ### How was this patch tested?
    
    Unit tests making sure that only `containsNull = false` cases fail.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #50343 from harshmotw-db/harsh-motwani_data/variant_cast_fix.
    
    Authored-by: Harsh Motwani <ha...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/catalyst/expressions/Cast.scala      | 10 ++--
 .../catalyst/expressions/CastWithAnsiOnSuite.scala | 54 +++++++++++++++++++++-
 2 files changed, 60 insertions(+), 4 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index f35e9e42927e..7a4145933fc7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -134,19 +134,23 @@ object Cast extends QueryErrorsBase {
     // to convert data of these types to Variant Objects.
     case (_, VariantType) => variant.VariantGet.checkDataType(from, allowStructsAndMaps = false)
 
+    // non-null variants can generate nulls even in ANSI mode
     case (ArrayType(fromType, fn), ArrayType(toType, tn)) =>
-      canAnsiCast(fromType, toType) && resolvableNullability(fn, tn)
+      canAnsiCast(fromType, toType) && resolvableNullability(fn || (fromType == VariantType), tn)
 
+    // non-null variants can generate nulls even in ANSI mode
     case (MapType(fromKey, fromValue, fn), MapType(toKey, toValue, tn)) =>
       canAnsiCast(fromKey, toKey) && canAnsiCast(fromValue, toValue) &&
-        resolvableNullability(fn, tn)
+        resolvableNullability(fn || (fromValue == VariantType), tn)
 
+    // non-null variants can generate nulls even in ANSI mode
     case (StructType(fromFields), StructType(toFields)) =>
       fromFields.length == toFields.length &&
         fromFields.zip(toFields).forall {
           case (fromField, toField) =>
             canAnsiCast(fromField.dataType, toField.dataType) &&
-              resolvableNullability(fromField.nullable, toField.nullable)
+              resolvableNullability(fromField.nullable || (fromField.dataType == VariantType),
+                toField.nullable)
         }
 
     case (udt1: UserDefinedType[_], udt2: UserDefinedType[_]) if udt2.acceptsType(udt1) => true
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala
index 340a5f30112e..b62b8c1302cc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastWithAnsiOnSuite.scala
@@ -23,13 +23,14 @@ import java.time.DateTimeException
 import org.apache.spark.{SparkArithmeticException, SparkRuntimeException}
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.MILLIS_PER_SECOND
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils
 import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, UTC}
 import org.apache.spark.sql.errors.QueryErrorsBase
 import org.apache.spark.sql.types._
-import org.apache.spark.unsafe.types.UTF8String
+import org.apache.spark.unsafe.types.{UTF8String, VariantVal}
 
 /**
  * Test suite for data type casting expression [[Cast]] with ANSI mode enabled.
@@ -182,6 +183,57 @@ class CastWithAnsiOnSuite extends CastSuiteBase with QueryErrorsBase {
     }
   }
 
+  test("ANSI mode: disallow variant cast to non-nullable types") {
+    // Array
+    val variantVal = new VariantVal(Array[Byte](12, 3), Array[Byte](1, 0, 0))
+    val sourceArrayType = ArrayType(VariantType, containsNull = false)
+    val targetArrayType = ArrayType(StringType, containsNull = false)
+    val variantArray = Literal.create(Seq(variantVal), sourceArrayType)
+    assert(cast(variantArray, targetArrayType).checkInputDataTypes() == DataTypeMismatch(
+      errorSubClass = "CAST_WITHOUT_SUGGESTION",
+      messageParameters = Map(
+        "srcType" -> toSQLType(sourceArrayType),
+        "targetType" -> toSQLType(targetArrayType)
+      )
+    ))
+    // make sure containsNull = true works
+    val targetArrayType2 = ArrayType(StringType, containsNull = true)
+    assert(cast(variantArray, targetArrayType2).checkInputDataTypes() ==
+      TypeCheckResult.TypeCheckSuccess)
+
+    // Struct
+    val sourceStructType = StructType(Array(StructField("v", VariantType, nullable = false)))
+    val targetStructType = StructType(Array(StructField("v", StringType, nullable = false)))
+    val variantStruct = Literal.create(Row(variantVal), sourceStructType)
+    assert(cast(variantStruct, targetStructType).checkInputDataTypes() == DataTypeMismatch(
+      errorSubClass = "CAST_WITHOUT_SUGGESTION",
+      messageParameters = Map(
+        "srcType" -> toSQLType(sourceStructType),
+        "targetType" -> toSQLType(targetStructType)
+      )
+    ))
+    // make sure nullable = true works
+    val targetStructType2 = StructType(Array(StructField("v", StringType, nullable = true)))
+    assert(cast(variantStruct, targetStructType2).checkInputDataTypes() ==
+      TypeCheckResult.TypeCheckSuccess)
+
+    // Map
+    val sourceMapType = MapType(StringType, VariantType, valueContainsNull = false)
+    val targetMapType = MapType(StringType, StringType, valueContainsNull = false)
+    val variantMap = Literal.create(Map("k" -> variantVal), sourceMapType)
+    assert(cast(variantMap, targetMapType).checkInputDataTypes() == DataTypeMismatch(
+      errorSubClass = "CAST_WITHOUT_SUGGESTION",
+      messageParameters = Map(
+        "srcType" -> toSQLType(sourceMapType),
+        "targetType" -> toSQLType(targetMapType)
+      )
+    ))
+    // make sure valueContainsNull = true works
+    val targetMapType2 = MapType(StringType, StringType, valueContainsNull = true)
+    assert(cast(variantMap, targetMapType2).checkInputDataTypes() ==
+      TypeCheckResult.TypeCheckSuccess)
+  }
+
   test("ANSI mode: disallow type conversions between Datatime types and Boolean types") {
     val timestampLiteral = Literal(1L, TimestampType)
     val checkResult1 = cast(timestampLiteral, BooleanType).checkInputDataTypes()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org