You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/30 07:44:16 UTC
spark git commit: [SPARK-7225][SQL] CombineLimits optimizer does not
work
Repository: spark
Updated Branches:
refs/heads/master ba49eb162 -> 445951449
[SPARK-7225][SQL] CombineLimits optimizer does not work
SQL
```
select key from (select key from src limit 100) t2 limit 10
```
Optimized Logical Plan before modifying
```
== Optimized Logical Plan ==
Limit 10
Limit 100
Project key#3
MetastoreRelation default, src, None
```
Optimized Logical Plan after modifying
```
== Optimized Logical Plan ==
Limit 10
Project [key#1]
MetastoreRelation default, src, None
```
Author: Zhongshuai Pei <79...@qq.com>
Author: DoingDone9 <79...@qq.com>
Closes #5770 from DoingDone9/limitOptimizer and squashes the following commits:
c68eaa7 [Zhongshuai Pei] Update CombiningLimitsSuite.scala
97e18cf [Zhongshuai Pei] Update Optimizer.scala
19ab875 [Zhongshuai Pei] Update CombiningLimitsSuite.scala
7db4566 [Zhongshuai Pei] Update CombiningLimitsSuite.scala
e2a491d [Zhongshuai Pei] Update Optimizer.scala
f03fe7f [Zhongshuai Pei] Merge pull request #12 from apache/master
f12fa50 [Zhongshuai Pei] Merge pull request #10 from apache/master
f61210c [Zhongshuai Pei] Merge pull request #9 from apache/master
34b1a9a [Zhongshuai Pei] Merge pull request #8 from apache/master
802261c [DoingDone9] Merge pull request #7 from apache/master
d00303b [DoingDone9] Merge pull request #6 from apache/master
98b134f [DoingDone9] Merge pull request #5 from apache/master
161cae3 [DoingDone9] Merge pull request #4 from apache/master
c87e8b6 [DoingDone9] Merge pull request #3 from apache/master
cb1852d [DoingDone9] Merge pull request #2 from apache/master
c3f046f [DoingDone9] Merge pull request #1 from apache/master
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/44595144
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/44595144
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/44595144
Branch: refs/heads/master
Commit: 4459514497eb76e6f2465d071857854390453805
Parents: ba49eb1
Author: Zhongshuai Pei <79...@qq.com>
Authored: Wed Apr 29 22:44:14 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Apr 29 22:44:14 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/optimizer/Optimizer.scala | 15 +++++++--------
.../optimizer/CombiningLimitsSuite.scala | 19 +++++++++++++++++++
2 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/44595144/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 2d03fbf..485761d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -36,7 +36,13 @@ object DefaultOptimizer extends Optimizer {
// SubQueries are only needed for analysis and can be removed before execution.
Batch("Remove SubQueries", FixedPoint(100),
EliminateSubQueries) ::
- Batch("Combine Limits", FixedPoint(100),
+ Batch("Operator Reordering", FixedPoint(100),
+ UnionPushdown,
+ CombineFilters,
+ PushPredicateThroughProject,
+ PushPredicateThroughJoin,
+ PushPredicateThroughGenerate,
+ ColumnPruning,
CombineLimits) ::
Batch("ConstantFolding", FixedPoint(100),
NullPropagation,
@@ -49,13 +55,6 @@ object DefaultOptimizer extends Optimizer {
OptimizeIn) ::
Batch("Decimal Optimizations", FixedPoint(100),
DecimalAggregates) ::
- Batch("Filter Pushdown", FixedPoint(100),
- UnionPushdown,
- CombineFilters,
- PushPredicateThroughProject,
- PushPredicateThroughJoin,
- PushPredicateThroughGenerate,
- ColumnPruning) ::
Batch("LocalRelation", FixedPoint(100),
ConvertToLocalRelation) :: Nil
}
http://git-wip-us.apache.org/repos/asf/spark/blob/44595144/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
index 2d16d66..a30052b 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala
@@ -27,6 +27,8 @@ class CombiningLimitsSuite extends PlanTest {
object Optimize extends RuleExecutor[LogicalPlan] {
val batches =
+ Batch("Filter Pushdown", FixedPoint(100),
+ ColumnPruning) ::
Batch("Combine Limit", FixedPoint(10),
CombineLimits) ::
Batch("Constant Folding", FixedPoint(10),
@@ -69,4 +71,21 @@ class CombiningLimitsSuite extends PlanTest {
comparePlans(optimized, correctAnswer)
}
+
+ test("limits: combines two limits after ColumnPruning") {
+ val originalQuery =
+ testRelation
+ .select('a)
+ .limit(2)
+ .select('a)
+ .limit(5)
+
+ val optimized = Optimize.execute(originalQuery.analyze)
+ val correctAnswer =
+ testRelation
+ .select('a)
+ .limit(2).analyze
+
+ comparePlans(optimized, correctAnswer)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org