[spark] branch master updated: [SPARK-33889][SQL] Fix NPE from `SHOW PARTITIONS` on V2 tables

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 303df64  [SPARK-33889][SQL] Fix NPE from `SHOW PARTITIONS` on V2 tables
303df64 is described below

commit 303df64b466b7734b3c497955d1cca3e34fb663e
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Wed Dec 23 14:34:01 2020 +0000

    [SPARK-33889][SQL] Fix NPE from `SHOW PARTITIONS` on V2 tables
    
    ### What changes were proposed in this pull request?
    At `ShowPartitionsExec.run()`, check that a row returned by `listPartitionIdentifiers()` contains a `null` field, and convert it to `"null"`.
    
    ### Why are the changes needed?
    Because `SHOW PARTITIONS` throws NPE on V2 table with `null` partition values.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes
    
    ### How was this patch tested?
    Added new UT to `v2.ShowPartitionsSuite`.
    
    Closes #30904 from MaxGekk/fix-npe-show-partitions.
    
    Authored-by: Max Gekk <ma...@gmail.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../execution/datasources/v2/ShowPartitionsExec.scala    |  6 +++---
 .../sql/execution/command/v2/ShowPartitionsSuite.scala   | 16 +++++++++++++++-
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
index 416dce6..ac24094 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
@@ -53,9 +53,9 @@ case class ShowPartitionsExec(
       var i = 0
       while (i < len) {
         val dataType = schema(i).dataType
-        val partValue = row.get(i, dataType)
-        val partValueStr = Cast(Literal(partValue, dataType), StringType, Some(timeZoneId))
-          .eval().toString
+        val partValueUTF8String =
+          Cast(Literal(row.get(i, dataType), dataType), StringType, Some(timeZoneId)).eval()
+        val partValueStr = if (partValueUTF8String == null) "null" else partValueUTF8String.toString
         partitions(i) = escapePathName(schema(i).name) + "=" + escapePathName(partValueStr)
         i += 1
       }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala
index e52c60d..ed0a7df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowPartitionsSuite.scala
@@ -17,7 +17,7 @@
 
 package org.apache.spark.sql.execution.command.v2
 
-import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.{AnalysisException, Row, SaveMode}
 import org.apache.spark.sql.execution.command
 
 class ShowPartitionsSuite extends command.ShowPartitionsSuiteBase with CommandSuiteBase {
@@ -34,4 +34,18 @@ class ShowPartitionsSuite extends command.ShowPartitionsSuiteBase with CommandSu
         "SHOW PARTITIONS cannot run for a table which does not support partitioning"))
     }
   }
+
+  test("SPARK-33889: null and empty string as partition values") {
+    import testImplicits._
+    withNamespaceAndTable("ns", "tbl") { t =>
+      val df = Seq((0, ""), (1, null)).toDF("a", "part")
+      df.write
+        .partitionBy("part")
+        .format("parquet")
+        .mode(SaveMode.Overwrite)
+        .saveAsTable(t)
+
+      runShowPartitionsSql(s"SHOW PARTITIONS $t", Row("part=") :: Row("part=null") :: Nil)
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org