You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2019/01/16 12:58:53 UTC
[spark] 02/02: Revert "[SPARK-26576][SQL] Broadcast hint not
applied to partitioned table"
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 18c138bf01cd43edc6db7115b90c4e7ae7126392
Author: Takeshi Yamamuro <ya...@apache.org>
AuthorDate: Wed Jan 16 21:56:39 2019 +0900
Revert "[SPARK-26576][SQL] Broadcast hint not applied to partitioned table"
This reverts commit 87c2c11e742a8b35699f68ec2002f817c56bef87.
---
.../apache/spark/sql/catalyst/planning/patterns.scala | 3 +++
.../execution/PruneFileSourcePartitionsSuite.scala | 19 +------------------
2 files changed, 4 insertions(+), 18 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
index a91063b..cc391aa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala
@@ -65,6 +65,9 @@ object PhysicalOperation extends PredicateHelper {
val substitutedCondition = substitute(aliases)(condition)
(fields, filters ++ splitConjunctivePredicates(substitutedCondition), other, aliases)
+ case h: ResolvedHint =>
+ collectProjectsAndFilters(h.child)
+
case other =>
(None, Nil, other, Map.empty)
}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
index 8a9adf7..9438418 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/PruneFileSourcePartitionsSuite.scala
@@ -17,20 +17,15 @@
package org.apache.spark.sql.hive.execution
-import org.scalatest.Matchers._
-
import org.apache.spark.sql.QueryTest
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, ResolvedHint}
+import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, LogicalRelation, PruneFileSourcePartitions}
import org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat
-import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec
-import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.hive.test.TestHiveSingleton
-import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SQLTestUtils
import org.apache.spark.sql.types.StructType
@@ -96,16 +91,4 @@ class PruneFileSourcePartitionsSuite extends QueryTest with SQLTestUtils with Te
assert(size2 < tableStats.get.sizeInBytes)
}
}
-
- test("SPARK-26576 Broadcast hint not applied to partitioned table") {
- withTable("tbl") {
- withSQLConf(SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") {
- spark.range(10).selectExpr("id", "id % 3 as p").write.partitionBy("p").saveAsTable("tbl")
- val df = spark.table("tbl")
- val qe = df.join(broadcast(df), "p").queryExecution
- qe.optimizedPlan.collect { case _: ResolvedHint => } should have size 1
- qe.sparkPlan.collect { case j: BroadcastHashJoinExec => j } should have size 1
- }
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org