(spark) branch master updated: [SPARK-48955][SQL] `ArrayCompact`'s datatype should be `containsNull = false`

[gu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new df510ae1a021 [SPARK-48955][SQL] `ArrayCompact`'s datatype should be `containsNull = false`
df510ae1a021 is described below

commit df510ae1a02191d7ca2d9fc299299e4fcbb269cd
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Mon Jul 22 08:52:55 2024 +0900

    [SPARK-48955][SQL] `ArrayCompact`'s datatype should be `containsNull = false`
    
    ### What changes were proposed in this pull request?
    `ArrayCompact`'s datatype should be `containsNull = false`
    
    ### Why are the changes needed?
    `ArrayCompact` - Removes null values from the array
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Added test
    
    before:
    ```
    scala> val df = spark.range(1).select(lit(Array(1,2,3)).alias("a"))
    val df: org.apache.spark.sql.DataFrame = [a: array<int>]
    
    scala> df.printSchema
    warning: 1 deprecation (since 2.13.3); for details, enable `:setting -deprecation` or `:replay -deprecation`
    root
     |-- a: array (nullable = false)
     |    |-- element: integer (containsNull = true)
    
    scala> df.select(array_compact(col("a"))).printSchema
    warning: 1 deprecation (since 2.13.3); for details, enable `:setting -deprecation` or `:replay -deprecation`
    root
     |-- array_compact(a): array (nullable = false)
     |    |-- element: integer (containsNull = true)
    ```
    
    after
    ```
    scala> df.select(array_compact(col("a"))).printSchema
    warning: 1 deprecation (since 2.13.3); for details, enable `:setting -deprecation` or `:replay -deprecation`
    root
     |-- array_compact(a): array (nullable = false)
     |    |-- element: integer (containsNull = false)
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #47430 from zhengruifeng/sql_array_compact_data_type.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 .../sql/catalyst/expressions/collectionOperations.scala   |  3 +++
 .../org/apache/spark/sql/DataFrameFunctionsSuite.scala    | 15 ++++++++++++---
 2 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index ea117f876550..708a060ceda5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -5206,6 +5206,9 @@ case class ArrayCompact(child: Expression)
 
   override def prettyName: String = "array_compact"
 
+  override def dataType: ArrayType =
+    child.dataType.asInstanceOf[ArrayType].copy(containsNull = false)
+
   override protected def withNewChildInternal(newChild: Expression): ArrayCompact =
     copy(child = newChild)
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index df1bb39f1874..830becc84c60 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -5811,14 +5811,23 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
       (Array[Integer](null, null, null), null, null)
     ).toDF("a", "b", "c")
 
-    checkAnswer(
-      df.select(array_compact($"a"),
-        array_compact($"b"), array_compact($"c")),
+    val df2 = df.select(
+      array_compact($"a").alias("a"),
+      array_compact($"b").alias("b"),
+      array_compact($"c").alias("c"))
+
+    checkAnswer(df2,
       Seq(Row(Seq(1, 2, 3, 4), Seq("a", "b", "c", "d"), Seq("", "")),
         Row(Seq.empty[Integer], Seq("1.0", "2.2", "3.0"), Seq.empty[String]),
         Row(Seq.empty[Integer], null, null))
     )
 
+    val expectedSchema = StructType(
+      StructField("a", ArrayType(IntegerType, containsNull = false), true) ::
+        StructField("b", ArrayType(StringType, containsNull = false), true) ::
+        StructField("c", ArrayType(StringType, containsNull = false), true) :: Nil)
+    assert(df2.schema === expectedSchema)
+
     checkAnswer(
       OneRowRelation().selectExpr("array_compact(array(1.0D, 2.0D, null))"),
       Seq(Row(Seq(1.0, 2.0)))


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org