[spark] branch master updated: [SPARK-33777][SQL] Sort output of V2 SHOW PARTITIONS

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new f156718  [SPARK-33777][SQL] Sort output of V2 SHOW PARTITIONS
f156718 is described below

commit f156718587fc33b9bf8e5abc4ae1f6fa0a5da887
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Mon Dec 14 14:28:47 2020 -0800

    [SPARK-33777][SQL] Sort output of V2 SHOW PARTITIONS
    
    ### What changes were proposed in this pull request?
    List partitions returned by the V2 `SHOW PARTITIONS` command in alphabetical order.
    
    ### Why are the changes needed?
    To have the same behavior as:
    1. V1 in-memory catalog, see https://github.com/apache/spark/blob/a28ed86a387b286745b30cd4d90b3d558205a5a7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/InMemoryCatalog.scala#L546
    2. V1 Hive catalogs, see https://github.com/apache/spark/blob/fab2995972761503563fa2aa547c67047c51bd33/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala#L715
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, after the changes, V2 SHOW PARTITIONS sorts its output.
    
    ### How was this patch tested?
    Added new UT to the base trait `ShowPartitionsSuiteBase` which contains tests for V1 and V2.
    
    Closes #30764 from MaxGekk/sort-show-partitions.
    
    Authored-by: Max Gekk <ma...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../execution/datasources/v2/ShowPartitionsExec.scala   |  5 +++--
 .../sql/execution/command/ShowPartitionsSuiteBase.scala | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
index c4b6aa8..416dce6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowPartitionsExec.scala
@@ -49,7 +49,7 @@ case class ShowPartitionsExec(
     val len = schema.length
     val partitions = new Array[String](len)
     val timeZoneId = SQLConf.get.sessionLocalTimeZone
-    partitionIdentifiers.map { row =>
+    val output = partitionIdentifiers.map { row =>
       var i = 0
       while (i < len) {
         val dataType = schema(i).dataType
@@ -59,7 +59,8 @@ case class ShowPartitionsExec(
         partitions(i) = escapePathName(schema(i).name) + "=" + escapePathName(partValueStr)
         i += 1
       }
-      InternalRow(UTF8String.fromString(partitions.mkString("/")))
+      partitions.mkString("/")
     }
+    output.sorted.map(p => InternalRow(UTF8String.fromString(p)))
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala
index b695dec..56c6e5a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowPartitionsSuiteBase.scala
@@ -173,4 +173,21 @@ trait ShowPartitionsSuiteBase extends QueryTest with SQLTestUtils {
       }
     }
   }
+
+  test("SPARK-33777: sorted output") {
+    withNamespace(s"$catalog.ns") {
+      sql(s"CREATE NAMESPACE $catalog.ns")
+      val table = s"$catalog.ns.dateTable"
+      withTable(table) {
+        sql(s"""
+          |CREATE TABLE $table (id int, part string)
+          |$defaultUsing
+          |PARTITIONED BY (part)""".stripMargin)
+        sql(s"ALTER TABLE $table ADD PARTITION(part = 'b')")
+        sql(s"ALTER TABLE $table ADD PARTITION(part = 'a')")
+        val partitions = sql(s"show partitions $table")
+        assert(partitions.first().getString(0) === "part=a")
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org