You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/08/30 08:24:59 UTC
spark git commit: [SPARK-24716][TESTS][FOLLOW-UP] Test Hive metastore
schema and parquet schema are in different letter cases
Repository: spark
Updated Branches:
refs/heads/master 56bc70047 -> e9fce2a4c
[SPARK-24716][TESTS][FOLLOW-UP] Test Hive metastore schema and parquet schema are in different letter cases
## What changes were proposed in this pull request?
Since https://github.com/apache/spark/pull/21696. Spark uses Parquet schema instead of Hive metastore schema to do pushdown.
That change can avoid wrong records returned when Hive metastore schema and parquet schema are in different letter cases. This pr add a test case for it.
More details:
https://issues.apache.org/jira/browse/SPARK-25206
## How was this patch tested?
unit tests
Closes #22267 from wangyum/SPARK-24716-TESTS.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e9fce2a4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e9fce2a4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e9fce2a4
Branch: refs/heads/master
Commit: e9fce2a4c124cdd709e93065ea691169b5a25f7d
Parents: 56bc700
Author: Yuming Wang <yu...@ebay.com>
Authored: Thu Aug 30 16:24:47 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Aug 30 16:24:47 2018 +0800
----------------------------------------------------------------------
.../apache/spark/sql/hive/HiveParquetSuite.scala | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e9fce2a4/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
index 09c1547..e5c9df0 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveParquetSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hive
import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.execution.datasources.parquet.ParquetTest
import org.apache.spark.sql.hive.test.TestHiveSingleton
+import org.apache.spark.sql.internal.SQLConf
case class Cases(lower: String, UPPER: String)
@@ -76,4 +77,19 @@ class HiveParquetSuite extends QueryTest with ParquetTest with TestHiveSingleton
}
}
}
+
+ test("SPARK-25206: wrong records are returned by filter pushdown " +
+ "when Hive metastore schema and parquet schema are in different letter cases") {
+ withSQLConf(SQLConf.PARQUET_FILTER_PUSHDOWN_ENABLED.key -> true.toString) {
+ withTempPath { path =>
+ val data = spark.range(1, 10).toDF("id")
+ data.write.parquet(path.getCanonicalPath)
+ withTable("SPARK_25206") {
+ sql("CREATE TABLE SPARK_25206 (ID LONG) USING parquet LOCATION " +
+ s"'${path.getCanonicalPath}'")
+ checkAnswer(sql("select id from SPARK_25206 where id > 0"), data)
+ }
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org