You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/14 14:26:53 UTC
spark git commit: [SPARK-23021][SQL] AnalysisBarrier should override
innerChildren to print correct explain output
Repository: spark
Updated Branches:
refs/heads/master 66738d29c -> 990f05c80
[SPARK-23021][SQL] AnalysisBarrier should override innerChildren to print correct explain output
## What changes were proposed in this pull request?
`AnalysisBarrier` in the current master cuts off explain results for parsed logical plans;
```
scala> Seq((1, 1)).toDF("a", "b").groupBy("a").count().sample(0.1).explain(true)
== Parsed Logical Plan ==
Sample 0.0, 0.1, false, -7661439431999668039
+- AnalysisBarrier Aggregate [a#5], [a#5, count(1) AS count#14L]
```
To fix this, `AnalysisBarrier` needs to override `innerChildren` and this pr changed the output to;
```
== Parsed Logical Plan ==
Sample 0.0, 0.1, false, -5086223488015741426
+- AnalysisBarrier
+- Aggregate [a#5], [a#5, count(1) AS count#14L]
+- Project [_1#2 AS a#5, _2#3 AS b#6]
+- LocalRelation [_1#2, _2#3]
```
## How was this patch tested?
Added tests in `DataFrameSuite`.
Author: Takeshi Yamamuro <ya...@apache.org>
Closes #20247 from maropu/SPARK-23021-2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/990f05c8
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/990f05c8
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/990f05c8
Branch: refs/heads/master
Commit: 990f05c80347c6eec2ee06823cff587c9ea90b49
Parents: 66738d2
Author: Takeshi Yamamuro <ya...@apache.org>
Authored: Sun Jan 14 22:26:21 2018 +0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Sun Jan 14 22:26:21 2018 +0800
----------------------------------------------------------------------
.../plans/logical/basicLogicalOperators.scala | 1 +
.../sql/hive/execution/HiveExplainSuite.scala | 17 +++++++++++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/990f05c8/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 95e099c..a4fca79 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -903,6 +903,7 @@ case class Deduplicate(
* This analysis barrier will be removed at the end of analysis stage.
*/
case class AnalysisBarrier(child: LogicalPlan) extends LeafNode {
+ override protected def innerChildren: Seq[LogicalPlan] = Seq(child)
override def output: Seq[Attribute] = child.output
override def isStreaming: Boolean = child.isStreaming
override def doCanonicalize(): LogicalPlan = child.canonicalized
http://git-wip-us.apache.org/repos/asf/spark/blob/990f05c8/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
index dfabf1e..a4273de 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveExplainSuite.scala
@@ -171,4 +171,21 @@ class HiveExplainSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
sql("EXPLAIN EXTENDED CODEGEN SELECT 1")
}
}
+
+ test("SPARK-23021 AnalysisBarrier should not cut off explain output for parsed logical plans") {
+ val df = Seq((1, 1)).toDF("a", "b").groupBy("a").count().limit(1)
+ val outputStream = new java.io.ByteArrayOutputStream()
+ Console.withOut(outputStream) {
+ df.explain(true)
+ }
+ assert(outputStream.toString.replaceAll("""#\d+""", "#0").contains(
+ s"""== Parsed Logical Plan ==
+ |GlobalLimit 1
+ |+- LocalLimit 1
+ | +- AnalysisBarrier
+ | +- Aggregate [a#0], [a#0, count(1) AS count#0L]
+ | +- Project [_1#0 AS a#0, _2#0 AS b#0]
+ | +- LocalRelation [_1#0, _2#0]
+ |""".stripMargin))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org