You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2019/03/07 00:56:54 UTC
[spark] branch master updated: [SPARK-27078][SQL] Fix
NoSuchFieldError when read Hive materialized views
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 32848ee [SPARK-27078][SQL] Fix NoSuchFieldError when read Hive materialized views
32848ee is described below
commit 32848eecc55946ad91e62e231d2e310a0270a63d
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Wed Mar 6 16:56:32 2019 -0800
[SPARK-27078][SQL] Fix NoSuchFieldError when read Hive materialized views
## What changes were proposed in this pull request?
This pr fix `NoSuchFieldError` when reading Hive materialized views from Hive 2.3.4.
How to reproduce:
Hive side:
```sql
CREATE TABLE materialized_view_tbl (key INT);
CREATE MATERIALIZED VIEW view_1 DISABLE REWRITE AS SELECT * FROM materialized_view_tbl;
```
Spark side:
```java
bin/spark-sql --conf spark.sql.hive.metastore.version=2.3.4 --conf spark.sql.hive.metastore.jars=maven
spark-sql> select * from view_1;
19/03/05 19:55:37 ERROR SparkSQLDriver: Failed in [select * from view_1]
java.lang.NoSuchFieldError: INDEX_TABLE
at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$getTableOption$3(HiveClientImpl.scala:438)
at scala.Option.map(Option.scala:163)
at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$getTableOption$1(HiveClientImpl.scala:370)
at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:277)
at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:215)
at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:214)
at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:260)
at org.apache.spark.sql.hive.client.HiveClientImpl.getTableOption(HiveClientImpl.scala:368)
```
## How was this patch tested?
unit tests
Closes #23984 from wangyum/SPARK-24360.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../org/apache/spark/sql/hive/client/HiveClientImpl.scala | 5 +++--
.../org/apache/spark/sql/hive/client/VersionsSuite.scala | 11 +++++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
index 2ca54af..74237a9 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala
@@ -435,8 +435,9 @@ private[hive] class HiveClientImpl(
case HiveTableType.EXTERNAL_TABLE => CatalogTableType.EXTERNAL
case HiveTableType.MANAGED_TABLE => CatalogTableType.MANAGED
case HiveTableType.VIRTUAL_VIEW => CatalogTableType.VIEW
- case HiveTableType.INDEX_TABLE =>
- throw new AnalysisException("Hive index table is not supported.")
+ case unsupportedType =>
+ val tableTypeStr = unsupportedType.toString.toLowerCase(Locale.ROOT).replace("_", " ")
+ throw new AnalysisException(s"Hive $tableTypeStr is not supported.")
},
schema = schema,
partitionColumnNames = partCols.map(_.name),
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
index bdbcfe9..8f2365c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala
@@ -582,6 +582,17 @@ class VersionsSuite extends SparkFunSuite with Logging {
}
}
+ test(s"$version: sql read hive materialized view") {
+ // HIVE-14249 Since Hive 2.3.0, materialized view is supported.
+ // But skip Hive 3.1 because of SPARK-27074.
+ if (version == "2.3") {
+ client.runSqlHive("CREATE TABLE materialized_view_tbl (c1 INT)")
+ client.runSqlHive("CREATE MATERIALIZED VIEW mv1 AS SELECT * FROM materialized_view_tbl")
+ val e = intercept[AnalysisException](versionSpark.table("mv1").collect()).getMessage
+ assert(e.contains("Hive materialized view is not supported"))
+ }
+ }
+
///////////////////////////////////////////////////////////////////////////
// Miscellaneous API
///////////////////////////////////////////////////////////////////////////
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org