You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2016/06/25 00:21:23 UTC
spark git commit: Revert "[SPARK-16186] [SQL] Support partition batch
pruning with `IN` predicate in InMemoryTableScanExec"
Repository: spark
Updated Branches:
refs/heads/master a65bcbc27 -> 20768dade
Revert "[SPARK-16186] [SQL] Support partition batch pruning with `IN` predicate in InMemoryTableScanExec"
This reverts commit a65bcbc27dcd9b3053cb13c5d67251c8d48f4397.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/20768dad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/20768dad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/20768dad
Branch: refs/heads/master
Commit: 20768dade2fee5dfe967a4629cad477e3d3bce6e
Parents: a65bcbc
Author: Davies Liu <da...@gmail.com>
Authored: Fri Jun 24 17:21:18 2016 -0700
Committer: Davies Liu <da...@gmail.com>
Committed: Fri Jun 24 17:21:18 2016 -0700
----------------------------------------------------------------------
.../columnar/InMemoryTableScanExec.scala | 4 ----
.../columnar/PartitionBatchPruningSuite.scala | 23 +-------------------
2 files changed, 1 insertion(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/20768dad/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
index 67a410f..183e494 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryTableScanExec.scala
@@ -79,10 +79,6 @@ private[sql] case class InMemoryTableScanExec(
case IsNull(a: Attribute) => statsFor(a).nullCount > 0
case IsNotNull(a: Attribute) => statsFor(a).count - statsFor(a).nullCount > 0
-
- case In(a: AttributeReference, list: Seq[Expression]) if list.forall(_.isInstanceOf[Literal]) =>
- list.map(l => statsFor(a).lowerBound <= l.asInstanceOf[Literal] &&
- l.asInstanceOf[Literal] <= statsFor(a).upperBound).reduce(_ || _)
}
val partitionFilters: Seq[Expression] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/20768dad/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
index b99cd67..7ca8e04 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/columnar/PartitionBatchPruningSuite.scala
@@ -65,18 +65,11 @@ class PartitionBatchPruningSuite
}, 5).toDF()
pruningData.createOrReplaceTempView("pruningData")
spark.catalog.cacheTable("pruningData")
-
- val pruningStringData = sparkContext.makeRDD((100 to 200).map { key =>
- StringData(key.toString)
- }, 5).toDF()
- pruningStringData.createOrReplaceTempView("pruningStringData")
- spark.catalog.cacheTable("pruningStringData")
}
override protected def afterEach(): Unit = {
try {
spark.catalog.uncacheTable("pruningData")
- spark.catalog.uncacheTable("pruningStringData")
} finally {
super.afterEach()
}
@@ -117,23 +110,9 @@ class PartitionBatchPruningSuite
88 to 100
}
- // Support `IN` predicate
- checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1)", 1, 1)(Seq(1))
- checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 2)", 1, 1)(Seq(1, 2))
- checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 11)", 1, 2)(Seq(1, 11))
- checkBatchPruning("SELECT key FROM pruningData WHERE key IN (1, 21, 41, 61, 81)", 5, 5)(
- Seq(1, 21, 41, 61, 81))
- checkBatchPruning("SELECT CAST(s AS INT) FROM pruningStringData WHERE s = '100'", 1, 1)(Seq(100))
- checkBatchPruning("SELECT CAST(s AS INT) FROM pruningStringData WHERE s < '102'", 1, 1)(
- Seq(100, 101))
- checkBatchPruning(
- "SELECT CAST(s AS INT) FROM pruningStringData WHERE s IN ('99', '150', '201')", 1, 1)(
- Seq(150))
-
- // With unsupported `InSet` predicate
+ // With unsupported predicate
{
val seq = (1 to 30).mkString(", ")
- checkBatchPruning(s"SELECT key FROM pruningData WHERE key IN ($seq)", 5, 10)(1 to 30)
checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq))", 5, 10)(31 to 100)
checkBatchPruning(s"SELECT key FROM pruningData WHERE NOT (key IN ($seq)) AND key > 88", 1, 2) {
89 to 100
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org