You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/04/20 01:51:24 UTC
spark git commit: [SPARK-20350] Add optimization rules to apply
Complementation Laws.
Repository: spark
Updated Branches:
refs/heads/master 4fea7848c -> 63824b2c8
[SPARK-20350] Add optimization rules to apply Complementation Laws.
## What changes were proposed in this pull request?
Apply Complementation Laws during boolean expression simplification.
## How was this patch tested?
Tested using unit tests, integration tests, and manual tests.
Author: ptkool <mi...@shopify.com>
Author: Michael Styles <mi...@shopify.com>
Closes #17650 from ptkool/apply_complementation_laws.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/63824b2c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/63824b2c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/63824b2c
Branch: refs/heads/master
Commit: 63824b2c8e010ba03013be498def236c654d4fed
Parents: 4fea784
Author: ptkool <mi...@shopify.com>
Authored: Thu Apr 20 09:51:13 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Apr 20 09:51:13 2017 +0800
----------------------------------------------------------------------
.../sql/catalyst/optimizer/expressions.scala | 5 +++++
.../optimizer/BooleanSimplificationSuite.scala | 19 +++++++++++++++++++
2 files changed, 24 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/63824b2c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
index ea2c5d2..34382bd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/expressions.scala
@@ -154,6 +154,11 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
case TrueLiteral Or _ => TrueLiteral
case _ Or TrueLiteral => TrueLiteral
+ case a And b if Not(a).semanticEquals(b) => FalseLiteral
+ case a Or b if Not(a).semanticEquals(b) => TrueLiteral
+ case a And b if a.semanticEquals(Not(b)) => FalseLiteral
+ case a Or b if a.semanticEquals(Not(b)) => TrueLiteral
+
case a And b if a.semanticEquals(b) => a
case a Or b if a.semanticEquals(b) => a
http://git-wip-us.apache.org/repos/asf/spark/blob/63824b2c/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
index 935bff7..c275f99 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/BooleanSimplificationSuite.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules._
import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.Row
class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
@@ -42,6 +43,16 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
val testRelation = LocalRelation('a.int, 'b.int, 'c.int, 'd.string)
+ val testRelationWithData = LocalRelation.fromExternalRows(
+ testRelation.output, Seq(Row(1, 2, 3, "abc"))
+ )
+
+ private def checkCondition(input: Expression, expected: LogicalPlan): Unit = {
+ val plan = testRelationWithData.where(input).analyze
+ val actual = Optimize.execute(plan)
+ comparePlans(actual, expected)
+ }
+
private def checkCondition(input: Expression, expected: Expression): Unit = {
val plan = testRelation.where(input).analyze
val actual = Optimize.execute(plan)
@@ -160,4 +171,12 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {
testRelation.where('a > 2 || ('b > 3 && 'b < 5)))
comparePlans(actual, expected)
}
+
+ test("Complementation Laws") {
+ checkCondition('a && !'a, testRelation)
+ checkCondition(!'a && 'a, testRelation)
+
+ checkCondition('a || !'a, testRelationWithData)
+ checkCondition(!'a || 'a, testRelationWithData)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org