You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/08/08 14:26:51 UTC
spark git commit: [SPARK-16457][SQL] Fix Wrong Messages when CTAS
with a Partition By Clause
Repository: spark
Updated Branches:
refs/heads/master 94a9d11ed -> ab126909c
[SPARK-16457][SQL] Fix Wrong Messages when CTAS with a Partition By Clause
#### What changes were proposed in this pull request?
When doing a CTAS with a Partition By clause, we got a wrong error message.
For example,
```SQL
CREATE TABLE gen__tmp
PARTITIONED BY (key string)
AS SELECT key, value FROM mytable1
```
The error message we get now is like
```
Operation not allowed: Schema may not be specified in a Create Table As Select (CTAS) statement(line 2, pos 0)
```
However, based on the code, the message we should get is like
```
Operation not allowed: A Create Table As Select (CTAS) statement is not allowed to create a partitioned table using Hive's file formats. Please use the syntax of "CREATE TABLE tableName USING dataSource OPTIONS (...) PARTITIONED BY ...\" to create a partitioned table through a CTAS statement.(line 2, pos 0)
```
Currently, partitioning columns is part of the schema. This PR fixes the bug by changing the detection orders.
#### How was this patch tested?
Added test cases.
Author: gatorsmile <ga...@gmail.com>
Closes #14113 from gatorsmile/ctas.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ab126909
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ab126909
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ab126909
Branch: refs/heads/master
Commit: ab126909ce381842dbb057d480a1f9bee1b4f38e
Parents: 94a9d11
Author: gatorsmile <ga...@gmail.com>
Authored: Mon Aug 8 22:26:44 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Mon Aug 8 22:26:44 2016 +0800
----------------------------------------------------------------------
.../spark/sql/execution/SparkSqlParser.scala | 12 +++----
.../sql/hive/execution/SQLQuerySuite.scala | 36 ++++++++++++++------
2 files changed, 32 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ab126909/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index 2bb6862..c3e3b21 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -998,12 +998,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
selectQuery match {
case Some(q) =>
- // Just use whatever is projected in the select statement as our schema
- if (schema.nonEmpty) {
- operationNotAllowed(
- "Schema may not be specified in a Create Table As Select (CTAS) statement",
- ctx)
- }
// Hive does not allow to use a CTAS statement to create a partitioned table.
if (tableDesc.partitionColumnNames.nonEmpty) {
val errorMessage = "A Create Table As Select (CTAS) statement is not allowed to " +
@@ -1013,6 +1007,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
"CTAS statement."
operationNotAllowed(errorMessage, ctx)
}
+ // Just use whatever is projected in the select statement as our schema
+ if (schema.nonEmpty) {
+ operationNotAllowed(
+ "Schema may not be specified in a Create Table As Select (CTAS) statement",
+ ctx)
+ }
val hasStorageProperties = (ctx.createFileFormat != null) || (ctx.rowFormat != null)
if (conf.convertCTAS && !hasStorageProperties) {
http://git-wip-us.apache.org/repos/asf/spark/blob/ab126909/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index cba6aa5..b659325 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -642,19 +642,35 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
test("specifying the column list for CTAS") {
- Seq((1, "111111"), (2, "222222")).toDF("key", "value").createOrReplaceTempView("mytable1")
+ withTempView("mytable1") {
+ Seq((1, "111111"), (2, "222222")).toDF("key", "value").createOrReplaceTempView("mytable1")
+ withTable("gen__tmp") {
+ sql("create table gen__tmp as select key as a, value as b from mytable1")
+ checkAnswer(
+ sql("SELECT a, b from gen__tmp"),
+ sql("select key, value from mytable1").collect())
+ }
- sql("create table gen__tmp as select key as a, value as b from mytable1")
- checkAnswer(
- sql("SELECT a, b from gen__tmp"),
- sql("select key, value from mytable1").collect())
- sql("DROP TABLE gen__tmp")
+ withTable("gen__tmp") {
+ val e = intercept[AnalysisException] {
+ sql("create table gen__tmp(a int, b string) as select key, value from mytable1")
+ }.getMessage
+ assert(e.contains("Schema may not be specified in a Create Table As Select (CTAS)"))
+ }
- intercept[AnalysisException] {
- sql("create table gen__tmp(a int, b string) as select key, value from mytable1")
+ withTable("gen__tmp") {
+ val e = intercept[AnalysisException] {
+ sql(
+ """
+ |CREATE TABLE gen__tmp
+ |PARTITIONED BY (key string)
+ |AS SELECT key, value FROM mytable1
+ """.stripMargin)
+ }.getMessage
+ assert(e.contains("A Create Table As Select (CTAS) statement is not allowed to " +
+ "create a partitioned table using Hive's file formats"))
+ }
}
-
- sql("drop table mytable1")
}
test("command substitution") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org