You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/02/12 20:20:33 UTC
spark git commit: [SPARK-22002][SQL][FOLLOWUP][TEST] Add a test to
check if the original schema doesn't have metadata.
Repository: spark
Updated Branches:
refs/heads/master 5bb11411a -> 0c66fe4f2
[SPARK-22002][SQL][FOLLOWUP][TEST] Add a test to check if the original schema doesn't have metadata.
## What changes were proposed in this pull request?
This is a follow-up pr of #19231 which modified the behavior to remove metadata from JDBC table schema.
This pr adds a test to check if the schema doesn't have metadata.
## How was this patch tested?
Added a test and existing tests.
Author: Takuya UESHIN <ue...@databricks.com>
Closes #20585 from ueshin/issues/SPARK-22002/fup1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0c66fe4f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0c66fe4f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0c66fe4f
Branch: refs/heads/master
Commit: 0c66fe4f22f8af4932893134bb0fd56f00fabeae
Parents: 5bb1141
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Mon Feb 12 12:20:29 2018 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Feb 12 12:20:29 2018 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 22 ++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0c66fe4f/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index cb2df0a..5238adc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1168,4 +1168,26 @@ class JDBCSuite extends SparkFunSuite
val df3 = sql("SELECT * FROM test_sessionInitStatement")
assert(df3.collect() === Array(Row(21519, 1234)))
}
+
+ test("jdbc data source shouldn't have unnecessary metadata in its schema") {
+ val schema = StructType(Seq(
+ StructField("NAME", StringType, true), StructField("THEID", IntegerType, true)))
+
+ val df = spark.read.format("jdbc")
+ .option("Url", urlWithUserAndPass)
+ .option("DbTaBle", "TEST.PEOPLE")
+ .load()
+ assert(df.schema === schema)
+
+ withTempView("people_view") {
+ sql(
+ s"""
+ |CREATE TEMPORARY VIEW people_view
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (uRl '$url', DbTaBlE 'TEST.PEOPLE', User 'testUser', PassWord 'testPass')
+ """.stripMargin.replaceAll("\n", " "))
+
+ assert(sql("select * from people_view").schema === schema)
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org