You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2024/04/25 01:57:39 UTC
(spark) branch master updated: [SPARK-47979][SQL][TESTS] Use Hive tables explicitly for Hive table capability tests
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0fcced63be99 [SPARK-47979][SQL][TESTS] Use Hive tables explicitly for Hive table capability tests
0fcced63be99 is described below
commit 0fcced63be99302593591d29370c00e7c0d73cec
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Wed Apr 24 18:57:29 2024 -0700
[SPARK-47979][SQL][TESTS] Use Hive tables explicitly for Hive table capability tests
### What changes were proposed in this pull request?
This PR aims to use `Hive` tables explicitly for Hive table capability tests in `hive` and `hive-thriftserver` module.
### Why are the changes needed?
To make Hive test coverage robust by making it independent from Apache Spark configuration changes.
### Does this PR introduce _any_ user-facing change?
No, this is a test only change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #46211 from dongjoon-hyun/SPARK-47979.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala | 2 +-
.../scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala | 1 +
.../scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 9 +++------
.../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 6 +++---
.../spark/sql/hive/execution/command/ShowCreateTableSuite.scala | 4 ++++
5 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
index b552611b75d1..2b2cbec41d64 100644
--- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
+++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/UISeleniumSuite.scala
@@ -108,7 +108,7 @@ class UISeleniumSuite
val baseURL = s"http://$localhost:$uiPort"
val queries = Seq(
- "CREATE TABLE test_map(key INT, value STRING)",
+ "CREATE TABLE test_map (key INT, value STRING) USING HIVE",
s"LOAD DATA LOCAL INPATH '${TestData.smallKv}' OVERWRITE INTO TABLE test_map")
queries.foreach(statement.execute)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
index 0bc288501a01..b60adfb6f4cf 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
@@ -686,6 +686,7 @@ class HiveClientSuite(version: String) extends HiveVersionSuite(version) {
versionSpark.sql(
s"""
|CREATE TABLE tab(c1 string)
+ |USING HIVE
|location '${tmpDir.toURI.toString}'
""".stripMargin)
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
index 241fdd4b9ec5..965db22b78f1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala
@@ -216,7 +216,7 @@ class HiveDDLSuite
test("SPARK-22431: alter table tests with nested types") {
withTable("t1", "t2", "t3") {
- spark.sql("CREATE TABLE t1 (q STRUCT<col1:INT, col2:STRING>, i1 INT)")
+ spark.sql("CREATE TABLE t1 (q STRUCT<col1:INT, col2:STRING>, i1 INT) USING HIVE")
spark.sql("ALTER TABLE t1 ADD COLUMNS (newcol1 STRUCT<`col1`:STRING, col2:Int>)")
val newcol = spark.sql("SELECT * FROM t1").schema.fields(2).name
assert("newcol1".equals(newcol))
@@ -2614,7 +2614,7 @@ class HiveDDLSuite
"msg" -> "java.lang.UnsupportedOperationException: Unknown field type: void")
)
- sql("CREATE TABLE t3 AS SELECT NULL AS null_col")
+ sql("CREATE TABLE t3 USING HIVE AS SELECT NULL AS null_col")
checkAnswer(sql("SELECT * FROM t3"), Row(null))
}
@@ -2642,9 +2642,6 @@ class HiveDDLSuite
sql("CREATE TABLE t3 (v VOID) USING hive")
checkAnswer(sql("SELECT * FROM t3"), Seq.empty)
-
- sql("CREATE TABLE t4 (v VOID)")
- checkAnswer(sql("SELECT * FROM t4"), Seq.empty)
}
// Create table with void type using spark.catalog.createTable
@@ -3324,7 +3321,7 @@ class HiveDDLSuite
| INTERVAL '1-1' YEAR TO MONTH AS YM,
| INTERVAL '1 02:03:04.123456' DAY TO SECOND AS DT
|""".stripMargin,
- s"CREATE TABLE $tbl (dt INTERVAL HOUR TO MINUTE)"
+ s"CREATE TABLE $tbl (dt INTERVAL HOUR TO MINUTE) USING HIVE"
).foreach { sqlCmd =>
checkError(
exception = intercept[SparkUnsupportedOperationException] {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 86e6b01cb6ca..397da6c18b50 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -369,7 +369,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
test("SPARK-7270: consider dynamic partition when comparing table output") {
withTable("test_partition", "ptest") {
- sql(s"CREATE TABLE test_partition (a STRING) PARTITIONED BY (b BIGINT, c STRING)")
+ sql(s"CREATE TABLE test_partition (a STRING) USING HIVE PARTITIONED BY (b BIGINT, c STRING)")
sql(s"CREATE TABLE ptest (a STRING, b BIGINT, c STRING)")
val analyzedPlan = sql(
@@ -804,7 +804,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
}
test("ADD JAR command") {
- sql("CREATE TABLE alter1(a INT, b INT)")
+ sql("CREATE TABLE alter1(a INT, b INT) USING HIVE")
checkError(
exception = intercept[AnalysisException] {
sql(
@@ -1225,7 +1225,7 @@ class HiveQuerySuite extends HiveComparisonTest with SQLTestUtils with BeforeAnd
test("Partition spec validation") {
sql("DROP TABLE IF EXISTS dp_test")
- sql("CREATE TABLE dp_test(key INT, value STRING) PARTITIONED BY (dp INT, sp INT)")
+ sql("CREATE TABLE dp_test(key INT, value STRING) USING HIVE PARTITIONED BY (dp INT, sp INT)")
sql("SET hive.exec.dynamic.partition.mode=strict")
// Should throw when using strict dynamic partition mode without any static partition
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
index 3098015dc7da..3dc73e116152 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala
@@ -43,6 +43,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite
| c1 INT COMMENT 'bla',
| c2 STRING
|)
+ |USING HIVE
|TBLPROPERTIES (
| 'prop1' = 'value1',
| 'prop2' = 'value2'
@@ -67,6 +68,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite
| c1 INT COMMENT 'bla',
| c2 STRING
|)
+ |USING HIVE
|LOCATION '${dir.toURI}'
|TBLPROPERTIES (
| 'prop1' = 'value1',
@@ -94,6 +96,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite
| c1 INT COMMENT 'bla',
| c2 STRING
|)
+ |USING HIVE
|COMMENT 'bla'
|PARTITIONED BY (
| p1 BIGINT COMMENT 'bla',
@@ -193,6 +196,7 @@ class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuite
withNamespaceAndTable(ns, table) { t =>
sql(
s"""CREATE TABLE $t (a INT, b STRING)
+ |STORED AS TEXTFILE
|CLUSTERED BY (a)
|SORTED BY (b)
|INTO 2 BUCKETS
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org