You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/12/14 22:05:54 UTC
spark git commit: [SPARK-12275] [SQL] No plan for BroadcastHint in
some condition - 1.5 backport
Repository: spark
Updated Branches:
refs/heads/branch-1.5 e4cf12118 -> 0fdf5542b
[SPARK-12275] [SQL] No plan for BroadcastHint in some condition - 1.5 backport
backport #10265 to branch 1.5.
When SparkStrategies.BasicOperators's "case BroadcastHint(child) => apply(child)" is hit,
it only recursively invokes BasicOperators.apply with this "child".
It makes many strategies have no change to process this plan,
which probably leads to "No plan" issue, so we use planLater to go through all strategies.
https://issues.apache.org/jira/browse/SPARK-12275
Author: yucai <yu...@intel.com>
Closes #10291 from yucai/backport_1.5_no_plan_for_broadcasthint and squashes the following commits:
b09715c [yucai] [SPARK-12275][SQL] No plan for BroadcastHint in some condition - 1.5 backport
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0fdf5542
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0fdf5542
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0fdf5542
Branch: refs/heads/branch-1.5
Commit: 0fdf5542bfa1627bf084735d6b9184ef12291941
Parents: e4cf121
Author: yucai <yu...@intel.com>
Authored: Mon Dec 14 13:05:48 2015 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Mon Dec 14 13:05:48 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/execution/SparkStrategies.scala | 2 +-
.../test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0fdf5542/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
index 4df5368..2b514e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala
@@ -388,7 +388,7 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
case e @ EvaluatePython(udf, child, _) =>
BatchPythonEvaluation(udf, e.output, planLater(child)) :: Nil
case LogicalRDD(output, rdd) => PhysicalRDD(output, rdd, "PhysicalRDD") :: Nil
- case BroadcastHint(child) => apply(child)
+ case BroadcastHint(child) => planLater(child) :: Nil
case _ => Nil
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0fdf5542/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
index e2716d7..718e158 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameJoinSuite.scala
@@ -107,5 +107,12 @@ class DataFrameJoinSuite extends QueryTest with SharedSQLContext {
// planner should not crash without a join
broadcast(df1).queryExecution.executedPlan
+
+ // SPARK-12275: no physical plan for BroadcastHint in some condition
+ withTempPath { path =>
+ df1.write.parquet(path.getCanonicalPath)
+ val pf1 = sqlContext.read.parquet(path.getCanonicalPath)
+ assert(df1.join(broadcast(pf1)).count() === 4)
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org