[spark] branch branch-3.0 updated: [SPARK-32163][SQL][3.0] Nested pruning should work even with cosmetic variations

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new b49756f  [SPARK-32163][SQL][3.0] Nested pruning should work even with cosmetic variations
b49756f is described below

commit b49756f9dc28b1246fa15694c6348589a47f724a
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Tue Jul 7 19:18:32 2020 -0700

    [SPARK-32163][SQL][3.0] Nested pruning should work even with cosmetic variations
    
    ### What changes were proposed in this pull request?
    
    This patch proposes to deal with cosmetic variations when processing nested column extractors in `NestedColumnAliasing`. Currently if cosmetic variations are in the nested column extractors, the query is not optimized.
    
    This backports #28988 to branch-3.0.
    
    ### Why are the changes needed?
    
    If the expressions extracting nested fields have cosmetic variations like qualifier difference, currently nested column pruning cannot work well.
    
    For example, two attributes which are semantically the same, are referred in a query, but the nested column extractors of them are treated differently when we deal with nested column pruning.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, fixing a bug in nested column pruning.
    
    ### How was this patch tested?
    
    Unit test.
    
    Closes #29027 from viirya/SPARK-32163-3.0.
    
    Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../spark/sql/catalyst/optimizer/NestedColumnAliasing.scala  | 11 ++++++++---
 .../spark/sql/execution/datasources/SchemaPruningSuite.scala | 12 ++++++++++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
index 9814bc2..becd7d1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NestedColumnAliasing.scala
@@ -118,10 +118,12 @@ object NestedColumnAliasing {
         case _ => false
       }
 
+    // Note that when we group by extractors with their references, we should remove
+    // cosmetic variations.
     val exclusiveAttrSet = AttributeSet(exclusiveAttrs ++ otherRootReferences)
     val aliasSub = nestedFieldReferences.asInstanceOf[Seq[ExtractValue]]
       .filter(!_.references.subsetOf(exclusiveAttrSet))
-      .groupBy(_.references.head)
+      .groupBy(_.references.head.canonicalized.asInstanceOf[Attribute])
       .flatMap { case (attr, nestedFields: Seq[ExtractValue]) =>
         // Each expression can contain multiple nested fields.
         // Note that we keep the original names to deliver to parquet in a case-sensitive way.
@@ -132,9 +134,12 @@ object NestedColumnAliasing {
 
         // If all nested fields of `attr` are used, we don't need to introduce new aliases.
         // By default, ColumnPruning rule uses `attr` already.
+        // Note that we need to remove cosmetic variations first, so we only count a
+        // nested field once.
         if (nestedFieldToAlias.nonEmpty &&
-            nestedFieldToAlias
-              .map { case (nestedField, _) => totalFieldNum(nestedField.dataType) }
+            nestedFields.map(_.canonicalized)
+              .distinct
+              .map { nestedField => totalFieldNum(nestedField.dataType) }
               .sum < totalFieldNum(attr.dataType)) {
           Some(attr.exprId -> nestedFieldToAlias)
         } else {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
index a3d4905..37ec6e29 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/SchemaPruningSuite.scala
@@ -301,6 +301,18 @@ abstract class SchemaPruningSuite
     checkAnswer(query, Row("Y.", 1) :: Row("X.", 1) :: Row(null, 2) :: Row(null, 2) :: Nil)
   }
 
+  testSchemaPruning("SPARK-32163: nested pruning should work even with cosmetic variations") {
+    withTempView("contact_alias") {
+      sql("select * from contacts")
+        .select(explode(col("friends.first")), col("friends"))
+        .createOrReplaceTempView("contact_alias")
+
+      val query = sql("select friends.middle, col from contact_alias")
+      checkScan(query, "struct<friends:array<struct<first:string,middle:string>>>")
+      checkAnswer(query, Row(Array("Z."), "Susan") :: Nil)
+    }
+  }
+
   protected def testSchemaPruning(testName: String)(testThunk: => Unit): Unit = {
     test(s"Spark vectorized reader - without partition data column - $testName") {
       withSQLConf(vectorizedReaderEnabledKey -> "true") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org