You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/24 21:06:13 UTC
spark git commit: [MINOR][SQL] add new unit test to LimitPushdown
Repository: spark
Updated Branches:
refs/heads/master bc9641d90 -> 6f0ba8472
[MINOR][SQL] add new unit test to LimitPushdown
## What changes were proposed in this pull request?
This PR is repaired as follows
1、update y -> x in "left outer join" test case ,maybe is mistake.
2、add a new test case："left outer join and left sides are limited"
3、add a new test case："left outer join and right sides are limited"
4、add a new test case: "right outer join and right sides are limited"
5、add a new test case: "right outer join and left sides are limited"
6、Remove annotations without code implementation
## How was this patch tested?
add new unit test case.
Author: caoxuewen <ca...@zte.com.cn>
Closes #20381 from heary-cao/LimitPushdownSuite.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6f0ba847
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6f0ba847
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6f0ba847
Branch: refs/heads/master
Commit: 6f0ba8472d1128551fa8090deebcecde0daebc53
Parents: bc9641d
Author: caoxuewen <ca...@zte.com.cn>
Authored: Wed Jan 24 13:06:09 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Wed Jan 24 13:06:09 2018 -0800
----------------------------------------------------------------------
.../sql/catalyst/optimizer/Optimizer.scala | 1 -
.../catalyst/optimizer/LimitPushdownSuite.scala | 30 +++++++++++++++++++-
2 files changed, 29 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6f0ba847/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 0f9daa5..8d20770 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -352,7 +352,6 @@ object LimitPushDown extends Rule[LogicalPlan] {
// on both sides if it is applied multiple times. Therefore:
// - If one side is already limited, stack another limit on top if the new limit is smaller.
// The redundant limit will be collapsed by the CombineLimits rule.
- // - If neither side is limited, limit the side that is estimated to be bigger.
case LocalLimit(exp, join @ Join(left, right, joinType, _)) =>
val newJoin = joinType match {
case RightOuter => join.copy(right = maybePushLocalLimit(exp, right))
http://git-wip-us.apache.org/repos/asf/spark/blob/6f0ba847/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
index cc98d23..17fb9fc 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/LimitPushdownSuite.scala
@@ -93,7 +93,21 @@ class LimitPushdownSuite extends PlanTest {
test("left outer join") {
val originalQuery = x.join(y, LeftOuter).limit(1)
val optimized = Optimize.execute(originalQuery.analyze)
- val correctAnswer = Limit(1, LocalLimit(1, y).join(y, LeftOuter)).analyze
+ val correctAnswer = Limit(1, LocalLimit(1, x).join(y, LeftOuter)).analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("left outer join and left sides are limited") {
+ val originalQuery = x.limit(2).join(y, LeftOuter).limit(1)
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = Limit(1, LocalLimit(1, x).join(y, LeftOuter)).analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("left outer join and right sides are limited") {
+ val originalQuery = x.join(y.limit(2), LeftOuter).limit(1)
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = Limit(1, LocalLimit(1, x).join(Limit(2, y), LeftOuter)).analyze
comparePlans(optimized, correctAnswer)
}
@@ -104,6 +118,20 @@ class LimitPushdownSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+ test("right outer join and right sides are limited") {
+ val originalQuery = x.join(y.limit(2), RightOuter).limit(1)
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = Limit(1, x.join(LocalLimit(1, y), RightOuter)).analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
+ test("right outer join and left sides are limited") {
+ val originalQuery = x.limit(2).join(y, RightOuter).limit(1)
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer = Limit(1, Limit(2, x).join(LocalLimit(1, y), RightOuter)).analyze
+ comparePlans(optimized, correctAnswer)
+ }
+
test("larger limits are not pushed on top of smaller ones in right outer join") {
val originalQuery = x.join(y.limit(5), RightOuter).limit(10)
val optimized = Optimize.execute(originalQuery.analyze)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org