You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/03/27 05:09:06 UTC
spark git commit: [SPARK-14157][SQL] Parse Drop Function DDL command
Repository: spark
Updated Branches:
refs/heads/master b547de8a6 -> bc925b73a
[SPARK-14157][SQL] Parse Drop Function DDL command
## What changes were proposed in this pull request?
JIRA: https://issues.apache.org/jira/browse/SPARK-14157
We only parse create function command. In order to support native drop function command, we need to parse it too.
>From Hive [manual](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+DDL#LanguageManualDDL-Create/Drop/ReloadFunction), the drop function command has syntax as:
DROP [TEMPORARY] FUNCTION [IF EXISTS] function_name;
## How was this patch tested?
Added test into `DDLCommandSuite`.
Author: Liang-Chi Hsieh <si...@tw.ibm.com>
Closes #11959 from viirya/parse-drop-func.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bc925b73
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bc925b73
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bc925b73
Branch: refs/heads/master
Commit: bc925b73a65ad57a636e6e6cb5648e4aed027af5
Parents: b547de8
Author: Liang-Chi Hsieh <si...@tw.ibm.com>
Authored: Sat Mar 26 20:09:01 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Sat Mar 26 20:09:01 2016 -0700
----------------------------------------------------------------------
.../apache/spark/sql/execution/SparkQl.scala | 38 ++++++++++++++++--
.../spark/sql/execution/command/ddl.scala | 16 +++++++-
.../sql/execution/command/DDLCommandSuite.scala | 42 +++++++++++++++++++-
.../org/apache/spark/sql/hive/HiveQl.scala | 1 -
4 files changed, 90 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bc925b73/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala
index b9542c7..c78b9b4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkQl.scala
@@ -165,11 +165,11 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly
case _ => parseFailed("Invalid CREATE FUNCTION command", node)
}
// If database name is specified, there are 3 tokens, otherwise 2.
- val (funcName, alias) = funcNameArgs match {
+ val (dbName, funcName, alias) = funcNameArgs match {
case Token(dbName, Nil) :: Token(fname, Nil) :: Token(aname, Nil) :: Nil =>
- (unquoteString(dbName) + "." + unquoteString(fname), unquoteString(aname))
+ (Some(unquoteString(dbName)), unquoteString(fname), unquoteString(aname))
case Token(fname, Nil) :: Token(aname, Nil) :: Nil =>
- (unquoteString(fname), unquoteString(aname))
+ (None, unquoteString(fname), unquoteString(aname))
case _ =>
parseFailed("Invalid CREATE FUNCTION command", node)
}
@@ -190,7 +190,37 @@ private[sql] class SparkQl(conf: ParserConf = SimpleParserConf()) extends Cataly
}
case _ => parseFailed("Invalid CREATE FUNCTION command", node)
}
- CreateFunction(funcName, alias, resources, temp.isDefined)(node.source)
+ CreateFunction(dbName, funcName, alias, resources, temp.isDefined)(node.source)
+
+ // DROP [TEMPORARY] FUNCTION [IF EXISTS] function_name;
+ case Token("TOK_DROPFUNCTION", args) =>
+ // Example format:
+ //
+ // TOK_DROPFUNCTION
+ // :- db_name
+ // :- func_name
+ // :- TOK_IFEXISTS
+ // +- TOK_TEMPORARY
+ val (funcNameArgs, otherArgs) = args.partition {
+ case Token("TOK_IFEXISTS", _) => false
+ case Token("TOK_TEMPORARY", _) => false
+ case Token(_, Nil) => true
+ case _ => parseFailed("Invalid DROP FUNCTION command", node)
+ }
+ // If database name is specified, there are 2 tokens, otherwise 1.
+ val (dbName, funcName) = funcNameArgs match {
+ case Token(dbName, Nil) :: Token(fname, Nil) :: Nil =>
+ (Some(unquoteString(dbName)), unquoteString(fname))
+ case Token(fname, Nil) :: Nil =>
+ (None, unquoteString(fname))
+ case _ =>
+ parseFailed("Invalid DROP FUNCTION command", node)
+ }
+
+ val Seq(ifExists, temp) = getClauses(Seq(
+ "TOK_IFEXISTS", "TOK_TEMPORARY"), otherArgs)
+
+ DropFunction(dbName, funcName, ifExists.isDefined, temp.isDefined)(node.source)
case Token("TOK_ALTERTABLE", alterTableArgs) =>
AlterTableCommandParser.parse(node)
http://git-wip-us.apache.org/repos/asf/spark/blob/bc925b73/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
index 373b557..a0f5b75 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/ddl.scala
@@ -19,7 +19,8 @@ package org.apache.spark.sql.execution.command
import org.apache.spark.internal.Logging
import org.apache.spark.sql.{Row, SQLContext}
-import org.apache.spark.sql.catalyst.TableIdentifier
+import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier}
+import org.apache.spark.sql.catalyst.catalog.CatalogFunction
import org.apache.spark.sql.catalyst.catalog.ExternalCatalog.TablePartitionSpec
import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference}
import org.apache.spark.sql.execution.datasources.BucketSpec
@@ -71,12 +72,25 @@ case class DropDatabase(
extends NativeDDLCommand(sql) with Logging
case class CreateFunction(
+ databaseName: Option[String],
functionName: String,
alias: String,
resources: Seq[(String, String)],
isTemp: Boolean)(sql: String)
extends NativeDDLCommand(sql) with Logging
+/**
+ * The DDL command that drops a function.
+ * ifExists: returns an error if the function doesn't exist, unless this is true.
+ * isTemp: indicates if it is a temporary function.
+ */
+case class DropFunction(
+ databaseName: Option[String],
+ functionName: String,
+ ifExists: Boolean,
+ isTemp: Boolean)(sql: String)
+ extends NativeDDLCommand(sql) with Logging
+
case class AlterTableRename(
oldName: TableIdentifier,
newName: TableIdentifier)(sql: String)
http://git-wip-us.apache.org/repos/asf/spark/blob/bc925b73/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
index a33175a..18f48ff 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLCommandSuite.scala
@@ -117,12 +117,14 @@ class DDLCommandSuite extends PlanTest {
val parsed1 = parser.parsePlan(sql1)
val parsed2 = parser.parsePlan(sql2)
val expected1 = CreateFunction(
+ None,
"helloworld",
"com.matthewrathbone.example.SimpleUDFExample",
Seq(("jar", "/path/to/jar1"), ("jar", "/path/to/jar2")),
isTemp = true)(sql1)
val expected2 = CreateFunction(
- "hello.world",
+ Some("hello"),
+ "world",
"com.matthewrathbone.example.SimpleUDFExample",
Seq(("archive", "/path/to/archive"), ("file", "/path/to/file")),
isTemp = false)(sql2)
@@ -130,6 +132,44 @@ class DDLCommandSuite extends PlanTest {
comparePlans(parsed2, expected2)
}
+ test("drop function") {
+ val sql1 = "DROP TEMPORARY FUNCTION helloworld"
+ val sql2 = "DROP TEMPORARY FUNCTION IF EXISTS helloworld"
+ val sql3 = "DROP FUNCTION hello.world"
+ val sql4 = "DROP FUNCTION IF EXISTS hello.world"
+
+ val parsed1 = parser.parsePlan(sql1)
+ val parsed2 = parser.parsePlan(sql2)
+ val parsed3 = parser.parsePlan(sql3)
+ val parsed4 = parser.parsePlan(sql4)
+
+ val expected1 = DropFunction(
+ None,
+ "helloworld",
+ ifExists = false,
+ isTemp = true)(sql1)
+ val expected2 = DropFunction(
+ None,
+ "helloworld",
+ ifExists = true,
+ isTemp = true)(sql2)
+ val expected3 = DropFunction(
+ Some("hello"),
+ "world",
+ ifExists = false,
+ isTemp = false)(sql3)
+ val expected4 = DropFunction(
+ Some("hello"),
+ "world",
+ ifExists = true,
+ isTemp = false)(sql4)
+
+ comparePlans(parsed1, expected1)
+ comparePlans(parsed2, expected2)
+ comparePlans(parsed3, expected3)
+ comparePlans(parsed4, expected4)
+ }
+
test("alter table: rename table") {
val sql = "ALTER TABLE table_name RENAME TO new_table_name"
val parsed = parser.parsePlan(sql)
http://git-wip-us.apache.org/repos/asf/spark/blob/bc925b73/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
index 6586b90..61fe098 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveQl.scala
@@ -102,7 +102,6 @@ private[hive] class HiveQl(conf: ParserConf) extends SparkQl(conf) with Logging
"TOK_DESCDATABASE",
- "TOK_DROPFUNCTION",
"TOK_DROPINDEX",
"TOK_DROPMACRO",
"TOK_DROPROLE",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org