You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/12/07 03:43:22 UTC

[spark] branch branch-3.1 updated: [SPARK-33680][SQL][TESTS] Fix PrunePartitionSuiteBase/BucketedReadWithHiveSupportSuite not to depend on the default conf

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
     new 873b634  [SPARK-33680][SQL][TESTS] Fix PrunePartitionSuiteBase/BucketedReadWithHiveSupportSuite not to depend on the default conf
873b634 is described below

commit 873b634006edec62f813ac2c2a24e904643ff90b
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Dec 6 19:34:54 2020 -0800

    [SPARK-33680][SQL][TESTS] Fix PrunePartitionSuiteBase/BucketedReadWithHiveSupportSuite not to depend on the default conf
    
    ### What changes were proposed in this pull request?
    
    This PR updates `PrunePartitionSuiteBase/BucketedReadWithHiveSupportSuite` to have the require conf explicitly.
    
    ### Why are the changes needed?
    
    The unit test should not depend on the default configurations.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    According to https://github.com/apache/spark/pull/30628 , this seems to be the only ones.
    
    Pass the CIs.
    
    Closes #30631 from dongjoon-hyun/SPARK-CONF-AGNO.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit 73412ffb3a857acda5dab41d7be3f7ae627f6eaf)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../hive/execution/PrunePartitionSuiteBase.scala   | 81 +++++++++++-----------
 .../sources/BucketedReadWithHiveSupportSuite.scala |  4 +-
 2 files changed, 45 insertions(+), 40 deletions(-)

diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala
index 8e35cd0..bc170fc 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PrunePartitionSuiteBase.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BinaryOperator, Expression, IsNotNull, Literal}
 import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan}
 import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf.ADAPTIVE_EXECUTION_ENABLED
 import org.apache.spark.sql.test.SQLTestUtils
 
 abstract class PrunePartitionSuiteBase extends QueryTest with SQLTestUtils with TestHiveSingleton {
@@ -28,48 +29,50 @@ abstract class PrunePartitionSuiteBase extends QueryTest with SQLTestUtils with
   protected def format: String
 
   test("SPARK-28169: Convert scan predicate condition to CNF") {
-    withTempView("temp") {
-      withTable("t") {
-        sql(
-          s"""
-             |CREATE TABLE t(i INT, p STRING)
-             |USING $format
-             |PARTITIONED BY (p)""".stripMargin)
-
-        spark.range(0, 1000, 1).selectExpr("id as col")
-          .createOrReplaceTempView("temp")
-
-        for (part <- Seq(1, 2, 3, 4)) {
+    withSQLConf(ADAPTIVE_EXECUTION_ENABLED.key -> "false") {
+      withTempView("temp") {
+        withTable("t") {
           sql(
             s"""
-               |INSERT OVERWRITE TABLE t PARTITION (p='$part')
-               |SELECT col FROM temp""".stripMargin)
-        }
+               |CREATE TABLE t(i INT, p STRING)
+               |USING $format
+               |PARTITIONED BY (p)""".stripMargin)
 
-        assertPrunedPartitions(
-          "SELECT * FROM t WHERE p = '1' OR (p = '2' AND i = 1)", 2,
-          "((`p` = '1') || (`p` = '2'))")
-        assertPrunedPartitions(
-          "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (i = 1 OR p = '2')", 4,
-          "")
-        assertPrunedPartitions(
-          "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '3' AND i = 3 )", 2,
-          "((`p` = '1') || (`p` = '3'))")
-        assertPrunedPartitions(
-          "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '2' OR p = '3')", 3,
-          "((`p` = '1') || ((`p` = '2') || (`p` = '3')))")
-        assertPrunedPartitions(
-          "SELECT * FROM t", 4,
-          "")
-        assertPrunedPartitions(
-          "SELECT * FROM t WHERE p = '1' AND i = 2", 1,
-          "(`p` = '1')")
-        assertPrunedPartitions(
-          """
-            |SELECT i, COUNT(1) FROM (
-            |SELECT * FROM t WHERE  p = '1' OR (p = '2' AND i = 1)
-            |) tmp GROUP BY i
-          """.stripMargin, 2, "((`p` = '1') || (`p` = '2'))")
+          spark.range(0, 1000, 1).selectExpr("id as col")
+            .createOrReplaceTempView("temp")
+
+          for (part <- Seq(1, 2, 3, 4)) {
+            sql(
+              s"""
+                 |INSERT OVERWRITE TABLE t PARTITION (p='$part')
+                 |SELECT col FROM temp""".stripMargin)
+          }
+
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE p = '1' OR (p = '2' AND i = 1)", 2,
+            "((`p` = '1') || (`p` = '2'))")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (i = 1 OR p = '2')", 4,
+            "")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '3' AND i = 3 )", 2,
+            "((`p` = '1') || (`p` = '3'))")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE (p = '1' AND i = 2) OR (p = '2' OR p = '3')", 3,
+            "((`p` = '1') || ((`p` = '2') || (`p` = '3')))")
+          assertPrunedPartitions(
+            "SELECT * FROM t", 4,
+            "")
+          assertPrunedPartitions(
+            "SELECT * FROM t WHERE p = '1' AND i = 2", 1,
+            "(`p` = '1')")
+          assertPrunedPartitions(
+            """
+              |SELECT i, COUNT(1) FROM (
+              |SELECT * FROM t WHERE  p = '1' OR (p = '2' AND i = 1)
+              |) tmp GROUP BY i
+            """.stripMargin, 2, "((`p` = '1') || (`p` = '2'))")
+        }
       }
     }
   }
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
index 35dab79..0790135 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/BucketedReadWithHiveSupportSuite.scala
@@ -17,10 +17,12 @@
 
 package org.apache.spark.sql.sources
 
+import org.apache.spark.sql.execution.adaptive.DisableAdaptiveExecutionSuite
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION
 
-class BucketedReadWithHiveSupportSuite extends BucketedReadSuite with TestHiveSingleton {
+class BucketedReadWithHiveSupportSuite
+  extends BucketedReadSuite with DisableAdaptiveExecutionSuite with TestHiveSingleton {
   protected override def beforeAll(): Unit = {
     super.beforeAll()
     assert(spark.sparkContext.conf.get(CATALOG_IMPLEMENTATION) == "hive")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org