You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/02/25 10:08:57 UTC
[spark] branch branch-3.1 updated: [SPARK-34436][SQL] DPP support
LIKE ANY/ALL expression
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8842faf [SPARK-34436][SQL] DPP support LIKE ANY/ALL expression
8842faf is described below
commit 8842faf4329fdd1a66e846f492c091f3126f4bf2
Author: Yuming Wang <yu...@apache.org>
AuthorDate: Thu Feb 25 18:07:39 2021 +0800
[SPARK-34436][SQL] DPP support LIKE ANY/ALL expression
### What changes were proposed in this pull request?
This pr make DPP support LIKE ANY/ALL expression:
```sql
SELECT date_id, product_id FROM fact_sk f
JOIN dim_store s
ON f.store_id = s.store_id WHERE s.country LIKE ANY ('%D%E%', '%A%B%')
```
### Why are the changes needed?
Improve query performance.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit test.
Closes #31563 from wangyum/SPARK-34436.
Lead-authored-by: Yuming Wang <yu...@apache.org>
Co-authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 4a3200b08ac3e7733b5a3dc7271d35e6872c5967)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../execution/dynamicpruning/PartitionPruning.scala | 1 +
.../spark/sql/DynamicPartitionPruningSuite.scala | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
index e30f9b6..7fac91a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/dynamicpruning/PartitionPruning.scala
@@ -163,6 +163,7 @@ object PartitionPruning extends Rule[LogicalPlan] with PredicateHelper {
case _: BinaryComparison => true
case _: In | _: InSet => true
case _: StringPredicate => true
+ case _: MultiLikeBase => true
case _ => false
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
index 55437aa..db7b0dd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DynamicPartitionPruningSuite.scala
@@ -1361,6 +1361,26 @@ abstract class DynamicPartitionPruningSuiteBase
checkAnswer(df, Nil)
}
}
+
+ test("SPARK-34436: DPP support LIKE ANY/ALL expression") {
+ withSQLConf(SQLConf.DYNAMIC_PARTITION_PRUNING_ENABLED.key -> "true") {
+ val df = sql(
+ """
+ |SELECT date_id, product_id FROM fact_sk f
+ |JOIN dim_store s
+ |ON f.store_id = s.store_id WHERE s.country LIKE ANY ('%D%E%', '%A%B%')
+ """.stripMargin)
+
+ checkPartitionPruningPredicate(df, false, true)
+
+ checkAnswer(df,
+ Row(1030, 2) ::
+ Row(1040, 2) ::
+ Row(1050, 2) ::
+ Row(1060, 2) :: Nil
+ )
+ }
+ }
}
class DynamicPartitionPruningSuiteAEOff extends DynamicPartitionPruningSuiteBase {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org