You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/12/03 15:54:38 UTC
spark git commit: [SPARK-26181][SQL] the `hasMinMaxStats` method of
`ColumnStatsMap` is not correct
Repository: spark
Updated Branches:
refs/heads/master eebb940ed -> 8534d753e
[SPARK-26181][SQL] the `hasMinMaxStats` method of `ColumnStatsMap` is not correct
## What changes were proposed in this pull request?
For now the `hasMinMaxStats` will return the same as `hasCountStats`, which is obviously not as expected.
## How was this patch tested?
Existing tests.
Closes #23152 from adrian-wang/minmaxstats.
Authored-by: Daoyuan Wang <me...@daoyuan.wang>
Signed-off-by: Wenchen Fan <we...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8534d753
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8534d753
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8534d753
Branch: refs/heads/master
Commit: 8534d753ecb21ea64ffbaefb5eaca38ba0464c6d
Parents: eebb940
Author: Daoyuan Wang <me...@daoyuan.wang>
Authored: Mon Dec 3 23:54:26 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Mon Dec 3 23:54:26 2018 +0800
----------------------------------------------------------------------
.../statsEstimation/FilterEstimation.scala | 14 +++++++---
.../statsEstimation/FilterEstimationSuite.scala | 27 ++++++++++++++++++++
.../sql/hive/execution/SQLQuerySuite.scala | 14 ++++++++++
3 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8534d753/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
index 5a3eeef..2c5beef 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/FilterEstimation.scala
@@ -670,6 +670,14 @@ case class FilterEstimation(plan: Filter) extends Logging {
logDebug("[CBO] No range comparison statistics for String/Binary type " + attrLeft)
return None
case _ =>
+ if (!colStatsMap.hasMinMaxStats(attrLeft)) {
+ logDebug("[CBO] No min/max statistics for " + attrLeft)
+ return None
+ }
+ if (!colStatsMap.hasMinMaxStats(attrRight)) {
+ logDebug("[CBO] No min/max statistics for " + attrRight)
+ return None
+ }
}
val colStatLeft = colStatsMap(attrLeft)
@@ -879,13 +887,13 @@ case class ColumnStatsMap(originalMap: AttributeMap[ColumnStat]) {
}
def hasCountStats(a: Attribute): Boolean =
- get(a).map(_.hasCountStats).getOrElse(false)
+ get(a).exists(_.hasCountStats)
def hasDistinctCount(a: Attribute): Boolean =
- get(a).map(_.distinctCount.isDefined).getOrElse(false)
+ get(a).exists(_.distinctCount.isDefined)
def hasMinMaxStats(a: Attribute): Boolean =
- get(a).map(_.hasCountStats).getOrElse(false)
+ get(a).exists(_.hasMinMaxStats)
/**
* Gets column stat for the given attribute. Prefer the column stat in updatedMap than that in
http://git-wip-us.apache.org/repos/asf/spark/blob/8534d753/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
index 47bfa62..b0a47e7 100755
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/FilterEstimationSuite.scala
@@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral}
import org.apache.spark.sql.catalyst.plans.LeftOuter
import org.apache.spark.sql.catalyst.plans.logical._
+import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.{ColumnStatsMap, FilterEstimation}
import org.apache.spark.sql.catalyst.plans.logical.statsEstimation.EstimationUtils._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.types._
@@ -821,6 +822,32 @@ class FilterEstimationSuite extends StatsEstimationTestBase {
expectedRowCount = 3)
}
+ test("ColumnStatsMap tests") {
+ val attrNoDistinct = AttributeReference("att_without_distinct", IntegerType)()
+ val attrNoCount = AttributeReference("att_without_count", BooleanType)()
+ val attrNoMinMax = AttributeReference("att_without_min_max", DateType)()
+ val colStatNoDistinct = ColumnStat(distinctCount = None, min = Some(1), max = Some(10),
+ nullCount = Some(0), avgLen = Some(4), maxLen = Some(4))
+ val colStatNoCount = ColumnStat(distinctCount = Some(2), min = Some(false), max = Some(true),
+ nullCount = None, avgLen = Some(1), maxLen = Some(1))
+ val colStatNoMinMax = ColumnStat(distinctCount = Some(1), min = None, max = None,
+ nullCount = Some(1), avgLen = None, maxLen = None)
+ val columnStatsMap = ColumnStatsMap(AttributeMap(Seq(
+ attrNoDistinct -> colStatNoDistinct,
+ attrNoCount -> colStatNoCount,
+ attrNoMinMax -> colStatNoMinMax
+ )))
+ assert(!columnStatsMap.hasDistinctCount(attrNoDistinct))
+ assert(columnStatsMap.hasDistinctCount(attrNoCount))
+ assert(columnStatsMap.hasDistinctCount(attrNoMinMax))
+ assert(!columnStatsMap.hasCountStats(attrNoDistinct))
+ assert(!columnStatsMap.hasCountStats(attrNoCount))
+ assert(columnStatsMap.hasCountStats(attrNoMinMax))
+ assert(columnStatsMap.hasMinMaxStats(attrNoDistinct))
+ assert(columnStatsMap.hasMinMaxStats(attrNoCount))
+ assert(!columnStatsMap.hasMinMaxStats(attrNoMinMax))
+ }
+
private def childStatsTestPlan(outList: Seq[Attribute], tableRowCount: BigInt): StatsTestPlan = {
StatsTestPlan(
outputList = outList,
http://git-wip-us.apache.org/repos/asf/spark/blob/8534d753/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index dfcde8c..fab2a27 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2276,4 +2276,18 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
}
+
+ test("SPARK-26181 hasMinMaxStats method of ColumnStatsMap is not correct") {
+ withSQLConf(SQLConf.CBO_ENABLED.key -> "true") {
+ withTable("all_null") {
+ sql("create table all_null (attr1 int, attr2 int)")
+ sql("insert into all_null values (null, null)")
+ sql("analyze table all_null compute statistics for columns attr1, attr2")
+ // check if the stats can be calculated without Cast exception.
+ sql("select * from all_null where attr1 < 1").queryExecution.stringWithStats
+ sql("select * from all_null where attr1 < attr2").queryExecution.stringWithStats
+ }
+ }
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org