You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/12/17 14:52:44 UTC
[spark] branch branch-2.4 updated: [SPARK-26352][SQL][FOLLOWUP-2.4]
Fix missing sameOutput in branch-2.4
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 0a69787 [SPARK-26352][SQL][FOLLOWUP-2.4] Fix missing sameOutput in branch-2.4
0a69787 is described below
commit 0a69787fd05b41cd5272ba95072310dba5be3978
Author: Kris Mok <re...@gmail.com>
AuthorDate: Mon Dec 17 22:48:59 2018 +0800
[SPARK-26352][SQL][FOLLOWUP-2.4] Fix missing sameOutput in branch-2.4
## What changes were proposed in this pull request?
After https://github.com/apache/spark/pull/23303 was merged to branch-2.3/2.4, the builds on those branches were broken due to missing a `LogicalPlan.sameOutput` function which came from https://github.com/apache/spark/pull/22713 only available on master.
This PR is to follow-up with the broken 2.3/2.4 branches and make a copy of the new `LogicalPlan.sameOutput` into `ReorderJoin` to make it locally available.
## How was this patch tested?
Fix the build of 2.3/2.4.
Closes #23330 from rednaxelafx/clean-build-2.4.
Authored-by: Kris Mok <re...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../apache/spark/sql/catalyst/optimizer/joins.scala | 17 ++++++++++++++++-
.../sql/catalyst/optimizer/JoinReorderSuite.scala | 19 +++++++++++++++++--
2 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
index 0b64712..2feb472 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/joins.scala
@@ -100,7 +100,7 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
createOrderedJoin(input, conditions)
}
- if (p.sameOutput(reordered)) {
+ if (sameOutput(p, reordered)) {
reordered
} else {
// Reordering the joins have changed the order of the columns.
@@ -108,6 +108,21 @@ object ReorderJoin extends Rule[LogicalPlan] with PredicateHelper {
Project(p.output, reordered)
}
}
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
}
/**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
index c94a8b9..38a70f0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/JoinReorderSuite.scala
@@ -291,8 +291,8 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
val optimized = Optimize.execute(analyzed)
val expected = groundTruthBestPlan.analyze
- assert(analyzed.sameOutput(expected)) // if this fails, the expected plan itself is incorrect
- assert(analyzed.sameOutput(optimized))
+ assert(sameOutput(analyzed, expected)) // if this fails, the expected plan itself is incorrect
+ assert(sameOutput(analyzed, optimized))
compareJoinOrder(optimized, expected)
}
@@ -300,4 +300,19 @@ class JoinReorderSuite extends PlanTest with StatsEstimationTestBase {
private def outputsOf(plans: LogicalPlan*): Seq[Attribute] = {
plans.map(_.output).reduce(_ ++ _)
}
+
+ /**
+ * Returns true iff output of both plans are semantically the same, ie.:
+ * - they contain the same number of `Attribute`s;
+ * - references are the same;
+ * - the order is equal too.
+ * NOTE: this is copied over from SPARK-25691 from master.
+ */
+ def sameOutput(plan1: LogicalPlan, plan2: LogicalPlan): Boolean = {
+ val output1 = plan1.output
+ val output2 = plan2.output
+ output1.length == output2.length && output1.zip(output2).forall {
+ case (a1, a2) => a1.semanticEquals(a2)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org