You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2016/04/14 20:08:19 UTC
spark git commit: [SPARK-14592][SQL] Native support for CREATE TABLE
LIKE DDL command
Repository: spark
Updated Branches:
refs/heads/master c971aee40 -> 28efdd3fd
[SPARK-14592][SQL] Native support for CREATE TABLE LIKE DDL command
## What changes were proposed in this pull request?
JIRA: https://issues.apache.org/jira/browse/SPARK-14592
This patch adds native support for DDL command `CREATE TABLE LIKE`.
The SQL syntax is like:
CREATE TABLE table_name LIKE existing_table
CREATE TABLE IF NOT EXISTS table_name LIKE existing_table
## How was this patch tested?
`HiveDDLCommandSuite`. `HiveQuerySuite` already tests `CREATE TABLE LIKE`.
Author: Liang-Chi Hsieh <si...@tw.ibm.com>
This patch had conflicts when merged, resolved by
Committer: Andrew Or <an...@databricks.com>
Closes #12362 from viirya/create-table-like.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/28efdd3f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/28efdd3f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/28efdd3f
Branch: refs/heads/master
Commit: 28efdd3fd789fa2ebed5be03b36ca0f682e37669
Parents: c971aee
Author: Liang-Chi Hsieh <si...@tw.ibm.com>
Authored: Thu Apr 14 11:08:08 2016 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Apr 14 11:08:08 2016 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 7 ++--
.../spark/sql/execution/command/tables.scala | 40 +++++++++++++++++++-
.../hive/execution/HiveCompatibilitySuite.scala | 4 +-
.../sql/hive/execution/HiveSqlParser.scala | 13 ++++++-
.../spark/sql/hive/HiveDDLCommandSuite.scala | 24 +++++++++++-
5 files changed, 79 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/28efdd3f/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index a937ad1..9cf2dd2 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -55,6 +55,8 @@ statement
rowFormat? createFileFormat? locationSpec?
(TBLPROPERTIES tablePropertyList)?
(AS? query)? #createTable
+ | CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier
+ LIKE source=tableIdentifier #createTableLike
| ANALYZE TABLE tableIdentifier partitionSpec? COMPUTE STATISTICS
(identifier | FOR COLUMNS identifierSeq?)? #analyze
| ALTER (TABLE | VIEW) from=tableIdentifier
@@ -136,10 +138,7 @@ statement
;
hiveNativeCommands
- : createTableHeader LIKE tableIdentifier
- rowFormat? createFileFormat? locationSpec?
- (TBLPROPERTIES tablePropertyList)?
- | DELETE FROM tableIdentifier (WHERE booleanExpression)?
+ : DELETE FROM tableIdentifier (WHERE booleanExpression)?
| TRUNCATE TABLE tableIdentifier partitionSpec?
(COLUMNS identifierList)?
| SHOW COLUMNS (FROM | IN) tableIdentifier ((FROM|IN) identifier)?
http://git-wip-us.apache.org/repos/asf/spark/blob/28efdd3f/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
index e315598..0b41985 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala
@@ -17,9 +17,45 @@
package org.apache.spark.sql.execution.command
-import org.apache.spark.sql.{Row, SQLContext}
+import org.apache.spark.sql.{AnalysisException, Row, SQLContext}
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.CatalogTable
+import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogTableType}
+
+/**
+ * A command to create a table with the same definition of the given existing table.
+ *
+ * The syntax of using this command in SQL is:
+ * {{{
+ * CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
+ * LIKE [other_db_name.]existing_table_name
+ * }}}
+ */
+case class CreateTableLike(
+ targetTable: TableIdentifier,
+ sourceTable: TableIdentifier,
+ ifNotExists: Boolean) extends RunnableCommand {
+
+ override def run(sqlContext: SQLContext): Seq[Row] = {
+ val catalog = sqlContext.sessionState.catalog
+ if (!catalog.tableExists(sourceTable)) {
+ throw new AnalysisException(
+ s"Source table in CREATE TABLE LIKE does not exist: '$sourceTable'")
+ }
+ if (catalog.isTemporaryTable(sourceTable)) {
+ throw new AnalysisException(
+ s"Source table in CREATE TABLE LIKE cannot be temporary: '$sourceTable'")
+ }
+
+ val tableToCreate = catalog.getTableMetadata(sourceTable).copy(
+ identifier = targetTable,
+ tableType = CatalogTableType.MANAGED_TABLE,
+ createTime = System.currentTimeMillis,
+ lastAccessTime = -1).withNewStorage(locationUri = None)
+
+ catalog.createTable(tableToCreate, ifNotExists)
+ Seq.empty[Row]
+ }
+}
// TODO: move the rest of the table commands from ddl.scala to this file
http://git-wip-us.apache.org/repos/asf/spark/blob/28efdd3f/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
index a45d180..989e68a 100644
--- a/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
+++ b/sql/hive/compatibility/src/test/scala/org/apache/spark/sql/hive/execution/HiveCompatibilitySuite.scala
@@ -416,6 +416,9 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"skewjoinopt18",
"skewjoinopt9",
+ // This test tries to create a table like with TBLPROPERTIES clause, which we don't support.
+ "create_like_tbl_props",
+
// Index commands are not supported
"drop_index",
"drop_index_removes_partition_dirs",
@@ -537,7 +540,6 @@ class HiveCompatibilitySuite extends HiveQueryFileTest with BeforeAndAfter {
"count",
"cp_mj_rc",
"create_insert_outputformat",
- "create_like_tbl_props",
"create_nested_type",
"create_struct_table",
"create_view_translate",
http://git-wip-us.apache.org/repos/asf/spark/blob/28efdd3f/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
index 8c70707..a97b65e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/HiveSqlParser.scala
@@ -31,8 +31,10 @@ import org.apache.spark.sql.catalyst.parser._
import org.apache.spark.sql.catalyst.parser.SqlBaseParser._
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.execution.SparkSqlAstBuilder
-import org.apache.spark.sql.execution.command.CreateTable
+import org.apache.spark.sql.execution.command.{CreateTable, CreateTableLike}
import org.apache.spark.sql.hive.{CreateTableAsSelect => CTAS, CreateViewAsSelect => CreateView, HiveSerDe}
+import org.apache.spark.sql.hive.{HiveGenericUDTF, HiveMetastoreTypes, HiveSerDe}
+import org.apache.spark.sql.hive.HiveShim.HiveFunctionWrapper
/**
* Concrete parser for HiveQl statements.
@@ -232,6 +234,15 @@ class HiveSqlAstBuilder extends SparkSqlAstBuilder {
}
/**
+ * Create a [[CreateTableLike]] command.
+ */
+ override def visitCreateTableLike(ctx: CreateTableLikeContext): LogicalPlan = withOrigin(ctx) {
+ val targetTable = visitTableIdentifier(ctx.target)
+ val sourceTable = visitTableIdentifier(ctx.source)
+ CreateTableLike(targetTable, sourceTable, ctx.EXISTS != null)
+ }
+
+ /**
* Create or replace a view. This creates a [[CreateViewAsSelect]] command.
*
* For example:
http://git-wip-us.apache.org/repos/asf/spark/blob/28efdd3f/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
index b87f035..110c6d1 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveDDLCommandSuite.scala
@@ -29,7 +29,7 @@ import org.apache.spark.sql.catalyst.expressions.JsonTuple
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{Generate, ScriptTransformation}
-import org.apache.spark.sql.execution.command.CreateTable
+import org.apache.spark.sql.execution.command.{CreateTable, CreateTableLike}
import org.apache.spark.sql.hive.execution.{HiveNativeCommand, HiveSqlParser}
class HiveDDLCommandSuite extends PlanTest {
@@ -557,4 +557,26 @@ class HiveDDLCommandSuite extends PlanTest {
assertUnsupported("MSCK REPAIR TABLE tab1")
}
+ test("create table like") {
+ val v1 = "CREATE TABLE table1 LIKE table2"
+ val (target, source, exists) = parser.parsePlan(v1).collect {
+ case CreateTableLike(t, s, allowExisting) => (t, s, allowExisting)
+ }.head
+ assert(exists == false)
+ assert(target.database.isEmpty)
+ assert(target.table == "table1")
+ assert(source.database.isEmpty)
+ assert(source.table == "table2")
+
+ val v2 = "CREATE TABLE IF NOT EXISTS table1 LIKE table2"
+ val (target2, source2, exists2) = parser.parsePlan(v2).collect {
+ case CreateTableLike(t, s, allowExisting) => (t, s, allowExisting)
+ }.head
+ assert(exists2)
+ assert(target2.database.isEmpty)
+ assert(target2.table == "table1")
+ assert(source2.database.isEmpty)
+ assert(source2.table == "table2")
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org