You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/11/22 23:57:15 UTC
spark git commit: [SPARK-18533] Raise correct error upon
specification of schema for datasource tables created using CTAS
Repository: spark
Updated Branches:
refs/heads/master 9c42d4a76 -> 39a1d3063
[SPARK-18533] Raise correct error upon specification of schema for datasource tables created using CTAS
## What changes were proposed in this pull request?
Fixes the inconsistency of error raised between data source and hive serde
tables when schema is specified in CTAS scenario. In the process the grammar for
create table (datasource) is simplified.
**before:**
``` SQL
spark-sql> create table t2 (c1 int, c2 int) using parquet as select * from t1;
Error in query:
mismatched input 'as' expecting {<EOF>, '.', 'OPTIONS', 'CLUSTERED', 'PARTITIONED'}(line 1, pos 64)
== SQL ==
create table t2 (c1 int, c2 int) using parquet as select * from t1
----------------------------------------------------------------^^^
```
**After:**
```SQL
spark-sql> create table t2 (c1 int, c2 int) using parquet as select * from t1
> ;
Error in query:
Operation not allowed: Schema may not be specified in a Create Table As Select (CTAS) statement(line 1, pos 0)
== SQL ==
create table t2 (c1 int, c2 int) using parquet as select * from t1
^^^
```
## How was this patch tested?
Added a new test in CreateTableAsSelectSuite
Author: Dilip Biswal <db...@us.ibm.com>
Closes #15968 from dilipbiswal/ctas.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/39a1d306
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/39a1d306
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/39a1d306
Branch: refs/heads/master
Commit: 39a1d30636857715247c82d551b200e1c331ad69
Parents: 9c42d4a
Author: Dilip Biswal <db...@us.ibm.com>
Authored: Tue Nov 22 15:57:07 2016 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Tue Nov 22 15:57:07 2016 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 6 +----
.../spark/sql/execution/SparkSqlParser.scala | 24 ++++++++++++++++++--
.../sql/sources/CreateTableAsSelectSuite.scala | 9 ++++++++
3 files changed, 32 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/39a1d306/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index bd05855..4531fe4 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -71,11 +71,7 @@ statement
| createTableHeader ('(' colTypeList ')')? tableProvider
(OPTIONS tablePropertyList)?
(PARTITIONED BY partitionColumnNames=identifierList)?
- bucketSpec? #createTableUsing
- | createTableHeader tableProvider
- (OPTIONS tablePropertyList)?
- (PARTITIONED BY partitionColumnNames=identifierList)?
- bucketSpec? AS? query #createTableUsing
+ bucketSpec? (AS? query)? #createTableUsing
| createTableHeader ('(' columns=colTypeList ')')?
(COMMENT STRING)?
(PARTITIONED BY '(' partitionColumns=colTypeList ')')?
http://git-wip-us.apache.org/repos/asf/spark/blob/39a1d306/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
index df509a5..0300bfe 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala
@@ -322,7 +322,20 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
}
/**
- * Create a [[CreateTable]] logical plan.
+ * Create a data source table, returning a [[CreateTable]] logical plan.
+ *
+ * Expected format:
+ * {{{
+ * CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [db_name.]table_name
+ * USING table_provider
+ * [OPTIONS table_property_list]
+ * [PARTITIONED BY (col_name, col_name, ...)]
+ * [CLUSTERED BY (col_name, col_name, ...)
+ * [SORTED BY (col_name [ASC|DESC], ...)]
+ * INTO num_buckets BUCKETS
+ * ]
+ * [AS select_statement];
+ * }}}
*/
override def visitCreateTableUsing(ctx: CreateTableUsingContext): LogicalPlan = withOrigin(ctx) {
val (table, temp, ifNotExists, external) = visitCreateTableHeader(ctx.createTableHeader)
@@ -371,6 +384,12 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
operationNotAllowed("CREATE TEMPORARY TABLE ... USING ... AS query", ctx)
}
+ // Don't allow explicit specification of schema for CTAS
+ if (schema.nonEmpty) {
+ operationNotAllowed(
+ "Schema may not be specified in a Create Table As Select (CTAS) statement",
+ ctx)
+ }
CreateTable(tableDesc, mode, Some(query))
} else {
if (temp) {
@@ -1052,7 +1071,8 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
"CTAS statement."
operationNotAllowed(errorMessage, ctx)
}
- // Just use whatever is projected in the select statement as our schema
+
+ // Don't allow explicit specification of schema for CTAS.
if (schema.nonEmpty) {
operationNotAllowed(
"Schema may not be specified in a Create Table As Select (CTAS) statement",
http://git-wip-us.apache.org/repos/asf/spark/blob/39a1d306/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
index 5cc9467..61939fe 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala
@@ -249,4 +249,13 @@ class CreateTableAsSelectSuite
}
}
}
+
+ test("specifying the column list for CTAS") {
+ withTable("t") {
+ val e = intercept[ParseException] {
+ sql("CREATE TABLE t (a int, b int) USING parquet AS SELECT 1, 2")
+ }.getMessage
+ assert(e.contains("Schema may not be specified in a Create Table As Select (CTAS)"))
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org