You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2019/06/20 05:19:27 UTC
[spark] branch master updated: [SPARK-23263][TEST] CTAS should
update stat if autoUpdate statistics is enabled
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4968f87 [SPARK-23263][TEST] CTAS should update stat if autoUpdate statistics is enabled
4968f87 is described below
commit 4968f871685c23b83690a9a1490ed886a3417d93
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Thu Jun 20 14:19:10 2019 +0900
[SPARK-23263][TEST] CTAS should update stat if autoUpdate statistics is enabled
## What changes were proposed in this pull request?
The [SPARK-27403](https://issues.apache.org/jira/browse/SPARK-27403) fixed CTAS cannot update statistics even if `spark.sql.statistics.size.autoUpdate.enabled` is enabled, as mentioned in [SPARK-23263](https://issues.apache.org/jira/browse/SPARK-23263).
This pr adds tests for that fix.
## How was this patch tested?
N/A
Closes #20430 from wangyum/SPARK-23263.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../apache/spark/sql/hive/StatisticsSuite.scala | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
index 483bd37..7a8e257 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/StatisticsSuite.scala
@@ -1431,4 +1431,26 @@ class StatisticsSuite extends StatisticsCollectionTestBase with TestHiveSingleto
assert(catalogStats.rowCount.isEmpty)
}
}
+
+ test(s"CTAS should update statistics if ${SQLConf.AUTO_SIZE_UPDATE_ENABLED.key} is enabled") {
+ val tableName = "SPARK_23263"
+ Seq(false, true).foreach { isConverted =>
+ Seq(false, true).foreach { updateEnabled =>
+ withSQLConf(
+ SQLConf.AUTO_SIZE_UPDATE_ENABLED.key -> updateEnabled.toString,
+ HiveUtils.CONVERT_METASTORE_PARQUET.key -> isConverted.toString) {
+ withTable(tableName) {
+ sql(s"CREATE TABLE $tableName STORED AS parquet AS SELECT 'a', 'b'")
+ val catalogTable = getCatalogTable(tableName)
+ // Hive serde tables always update statistics by Hive metastore
+ if (!isConverted || updateEnabled) {
+ assert(catalogTable.stats.nonEmpty)
+ } else {
+ assert(catalogTable.stats.isEmpty)
+ }
+ }
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org