You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2019/05/23 20:39:57 UTC
[spark] branch branch-2.4 updated (d6ab7e6 -> fb60066)
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a change to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git.
from d6ab7e6 [SPARK-26045][BUILD] Leave avro, avro-ipc dependendencies as compile scope even for hadoop-provided usages
new ec6a08b Revert "Revert "[SPARK-27351][SQL] Wrong outputRows estimation after AggregateEstimation wit…""
new fb60066 Revert "Revert "[SPARK-27539][SQL] Fix inaccurate aggregate outputRows estimation with column containing null values""
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../statsEstimation/AggregateEstimation.scala | 12 +++++++++--
.../statsEstimation/AggregateEstimationSuite.scala | 23 +++++++++++++++++++++-
2 files changed, 32 insertions(+), 3 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 01/02: Revert "Revert "[SPARK-27351][SQL] Wrong outputRows estimation after AggregateEstimation wit…""
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
commit ec6a08b510118f32e3bb21859662aa34d10a36d1
Author: HyukjinKwon <gu...@apache.org>
AuthorDate: Fri May 24 05:39:06 2019 +0900
Revert "Revert "[SPARK-27351][SQL] Wrong outputRows estimation after AggregateEstimation wit…""
This reverts commit e69ad46c72ed26c8293da95dc19b6f31445c0df5.
---
.../plans/logical/statsEstimation/AggregateEstimation.scala | 12 ++++++++++--
.../catalyst/statsEstimation/AggregateEstimationSuite.scala | 12 +++++++++++-
2 files changed, 21 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
index 111c594..7ef22fa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
@@ -39,8 +39,16 @@ object AggregateEstimation {
// Multiply distinct counts of group-by columns. This is an upper bound, which assumes
// the data contains all combinations of distinct values of group-by columns.
var outputRows: BigInt = agg.groupingExpressions.foldLeft(BigInt(1))(
- (res, expr) => res *
- childStats.attributeStats(expr.asInstanceOf[Attribute]).distinctCount.get)
+ (res, expr) => {
+ val columnStat = childStats.attributeStats(expr.asInstanceOf[Attribute])
+ val distinctCount = columnStat.distinctCount.get
+ val distinctValue: BigInt = if (distinctCount == 0 && columnStat.nullCount.get > 0) {
+ 1
+ } else {
+ distinctCount
+ }
+ res * distinctValue
+ })
outputRows = if (agg.groupingExpressions.isEmpty) {
// If there's no group-by columns, the output is a single row containing values of aggregate
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
index 8213d56..6bdf8cd 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
@@ -38,7 +38,9 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
attr("key22") -> ColumnStat(distinctCount = Some(2), min = Some(10), max = Some(20),
nullCount = Some(0), avgLen = Some(4), maxLen = Some(4)),
attr("key31") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
- nullCount = Some(0), avgLen = Some(4), maxLen = Some(4))
+ nullCount = Some(0), avgLen = Some(4), maxLen = Some(4)),
+ attr("key32") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
+ nullCount = Some(4), avgLen = Some(4), maxLen = Some(4))
))
private val nameToAttr: Map[String, Attribute] = columnInfo.map(kv => kv._1.name -> kv._1)
@@ -92,6 +94,14 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
expectedOutputRowCount = 0)
}
+ test("group-by column with only null value") {
+ checkAggStats(
+ tableColumns = Seq("key22", "key32"),
+ tableRowCount = 6,
+ groupByColumns = Seq("key22", "key32"),
+ expectedOutputRowCount = nameToColInfo("key22")._2.distinctCount.get)
+ }
+
test("non-cbo estimation") {
val attributes = Seq("key12").map(nameToAttr)
val child = StatsTestPlan(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 02/02: Revert "Revert "[SPARK-27539][SQL] Fix inaccurate
aggregate outputRows estimation with column containing null values""
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
commit fb60066027b4ce4281acfd2755538566da2664aa
Author: HyukjinKwon <gu...@apache.org>
AuthorDate: Fri May 24 05:39:13 2019 +0900
Revert "Revert "[SPARK-27539][SQL] Fix inaccurate aggregate outputRows estimation with column containing null values""
This reverts commit e0e8a6de1345e6e716bb8c6e35a98e981feb3bab.
---
.../plans/logical/statsEstimation/AggregateEstimation.scala | 4 ++--
.../catalyst/statsEstimation/AggregateEstimationSuite.scala | 13 ++++++++++++-
2 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
index 7ef22fa..b9e72c2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statsEstimation/AggregateEstimation.scala
@@ -42,8 +42,8 @@ object AggregateEstimation {
(res, expr) => {
val columnStat = childStats.attributeStats(expr.asInstanceOf[Attribute])
val distinctCount = columnStat.distinctCount.get
- val distinctValue: BigInt = if (distinctCount == 0 && columnStat.nullCount.get > 0) {
- 1
+ val distinctValue: BigInt = if (columnStat.nullCount.get > 0) {
+ distinctCount + 1
} else {
distinctCount
}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
index 6bdf8cd..d89b9df 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/statsEstimation/AggregateEstimationSuite.scala
@@ -40,7 +40,9 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
attr("key31") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
nullCount = Some(0), avgLen = Some(4), maxLen = Some(4)),
attr("key32") -> ColumnStat(distinctCount = Some(0), min = None, max = None,
- nullCount = Some(4), avgLen = Some(4), maxLen = Some(4))
+ nullCount = Some(4), avgLen = Some(4), maxLen = Some(4)),
+ attr("key33") -> ColumnStat(distinctCount = Some(2), min = None, max = None,
+ nullCount = Some(2), avgLen = Some(4), maxLen = Some(4))
))
private val nameToAttr: Map[String, Attribute] = columnInfo.map(kv => kv._1.name -> kv._1)
@@ -102,6 +104,15 @@ class AggregateEstimationSuite extends StatsEstimationTestBase with PlanTest {
expectedOutputRowCount = nameToColInfo("key22")._2.distinctCount.get)
}
+ test("group-by column with null value") {
+ checkAggStats(
+ tableColumns = Seq("key21", "key33"),
+ tableRowCount = 6,
+ groupByColumns = Seq("key21", "key33"),
+ expectedOutputRowCount = nameToColInfo("key21")._2.distinctCount.get *
+ (nameToColInfo("key33")._2.distinctCount.get + 1))
+ }
+
test("non-cbo estimation") {
val attributes = Seq("key12").map(nameToAttr)
val child = StatsTestPlan(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org