You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2019/01/03 16:31:22 UTC
[spark] branch master updated: [SPARK-26517][SQL][TEST] Avoid
duplicate test in ParquetSchemaPruningSuite
This is an automated email from the ASF dual-hosted git repository.
srowen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 40711ee [SPARK-26517][SQL][TEST] Avoid duplicate test in ParquetSchemaPruningSuite
40711ee is described below
commit 40711eef168716c44b873359e17822fe6b3387f4
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Thu Jan 3 10:30:47 2019 -0600
[SPARK-26517][SQL][TEST] Avoid duplicate test in ParquetSchemaPruningSuite
## What changes were proposed in this pull request?
`testExactCaseQueryPruning` and `testMixedCaseQueryPruning` don't need to set up `PARQUET_VECTORIZED_READER_ENABLED` config. Because `withMixedCaseData` will run against both Spark vectorized reader and Parquet-mr reader.
## How was this patch tested?
Existing test.
Closes #23427 from viirya/fix-parquet-schema-pruning-test.
Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
Signed-off-by: Sean Owen <se...@databricks.com>
---
.../parquet/ParquetSchemaPruningSuite.scala | 23 +++++-----------------
1 file changed, 5 insertions(+), 18 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
index 434c441..9a02529 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetSchemaPruningSuite.scala
@@ -312,15 +312,8 @@ class ParquetSchemaPruningSuite
// schema's column and field names. N.B. this implies that `testThunk` should pass using either a
// case-sensitive or case-insensitive query parser
private def testExactCaseQueryPruning(testName: String)(testThunk: => Unit) {
- test(s"Spark vectorized reader - case-sensitive parser - mixed-case schema - $testName") {
- withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
- SQLConf.CASE_SENSITIVE.key -> "true") {
- withMixedCaseData(testThunk)
- }
- }
- test(s"Parquet-mr reader - case-sensitive parser - mixed-case schema - $testName") {
- withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
- SQLConf.CASE_SENSITIVE.key -> "true") {
+ test(s"Case-sensitive parser - mixed-case schema - $testName") {
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "true") {
withMixedCaseData(testThunk)
}
}
@@ -330,20 +323,14 @@ class ParquetSchemaPruningSuite
// Tests schema pruning for a query whose column and field names may differ in case from the table
// schema's column and field names
private def testMixedCaseQueryPruning(testName: String)(testThunk: => Unit) {
- test(s"Spark vectorized reader - case-insensitive parser - mixed-case schema - $testName") {
- withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "true",
- SQLConf.CASE_SENSITIVE.key -> "false") {
- withMixedCaseData(testThunk)
- }
- }
- test(s"Parquet-mr reader - case-insensitive parser - mixed-case schema - $testName") {
- withSQLConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key -> "false",
- SQLConf.CASE_SENSITIVE.key -> "false") {
+ test(s"Case-insensitive parser - mixed-case schema - $testName") {
+ withSQLConf(SQLConf.CASE_SENSITIVE.key -> "false") {
withMixedCaseData(testThunk)
}
}
}
+ // Tests given test function with Spark vectorized reader and Parquet-mr reader.
private def withMixedCaseData(testThunk: => Unit) {
withParquetTable(mixedCaseData, "mixedcase") {
testThunk
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org