You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/10/30 17:19:38 UTC
spark git commit: [SPARK-22396][SQL] Better Error Message for
InsertIntoDir using Hive format without enabling Hive Support
Repository: spark
Updated Branches:
refs/heads/master 079a2609d -> 65338de5f
[SPARK-22396][SQL] Better Error Message for InsertIntoDir using Hive format without enabling Hive Support
## What changes were proposed in this pull request?
When Hive support is not on, users can hit unresolved plan node when trying to call `INSERT OVERWRITE DIRECTORY` using Hive format.
```
"unresolved operator 'InsertIntoDir true, Storage(Location: /private/var/folders/vx/j0ydl5rn0gd9mgrh1pljnw900000gn/T/spark-b4227606-9311-46a8-8c02-56355bf0e2bc, Serde Library: org.apache.hadoop.hive.ql.io.orc.OrcSerde, InputFormat: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat, OutputFormat: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat), hive, true;;
```
This PR is to issue a better error message.
## How was this patch tested?
Added a test case.
Author: gatorsmile <ga...@gmail.com>
Closes #19608 from gatorsmile/hivesupportInsertOverwrite.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/65338de5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/65338de5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/65338de5
Branch: refs/heads/master
Commit: 65338de5fbaf90774fb3f4c51321359d324ace56
Parents: 079a260
Author: gatorsmile <ga...@gmail.com>
Authored: Mon Oct 30 10:19:34 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Oct 30 10:19:34 2017 -0700
----------------------------------------------------------------------
.../spark/sql/execution/datasources/rules.scala | 3 +++
.../org/apache/spark/sql/sources/InsertSuite.scala | 16 ++++++++++++++++
2 files changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/65338de5/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
index 7a2c85e..60c430b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/rules.scala
@@ -404,6 +404,9 @@ object HiveOnlyCheck extends (LogicalPlan => Unit) {
plan.foreach {
case CreateTable(tableDesc, _, _) if DDLUtils.isHiveTable(tableDesc) =>
throw new AnalysisException("Hive support is required to CREATE Hive TABLE (AS SELECT)")
+ case i: InsertIntoDir if DDLUtils.isHiveTable(i.provider) =>
+ throw new AnalysisException(
+ "Hive support is required to INSERT OVERWRITE DIRECTORY with the Hive format")
case _ => // OK
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/65338de5/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 875b745..8b7e2e5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -408,6 +408,22 @@ class InsertSuite extends DataSourceTest with SharedSQLContext {
}
}
+ test("Insert overwrite directory using Hive serde without turning on Hive support") {
+ withTempDir { dir =>
+ val path = dir.toURI.getPath
+ val e = intercept[AnalysisException] {
+ sql(
+ s"""
+ |INSERT OVERWRITE LOCAL DIRECTORY '$path'
+ |STORED AS orc
+ |SELECT 1, 2
+ """.stripMargin)
+ }.getMessage
+ assert(e.contains(
+ "Hive support is required to INSERT OVERWRITE DIRECTORY with the Hive format"))
+ }
+ }
+
test("insert overwrite directory to data source not providing FileFormat") {
withTempDir { dir =>
val path = dir.toURI.getPath
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org