You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2016/04/26 19:29:58 UTC
spark git commit: [SPARK-14910][SQL] Native DDL Command Support for
Describe Function in Non-identifier Format
Repository: spark
Updated Branches:
refs/heads/master b208229ba -> 162cf02ef
[SPARK-14910][SQL] Native DDL Command Support for Describe Function in Non-identifier Format
#### What changes were proposed in this pull request?
The existing `Describe Function` only support the function name in `identifier`. This is different from what Hive behaves. That is why many test cases `udf_abc` in `HiveCompatibilitySuite` are not using our native DDL support. For example,
- udf_not.q
- udf_bitwise_not.q
This PR is to resolve the issues. Now, we can support the command of `Describe Function` whose function names are in the following format:
- `qualifiedName` (e.g., `db.func1`)
- `STRING` (e.g., `'func1'`)
- `comparisonOperator` (e.g,. `<`)
- `arithmeticOperator` (e.g., `+`)
- `predicateOperator` (e.g., `or`)
Note, before this PR, we only have a native command support when the function name is in the format of `qualifiedName`.
#### How was this patch tested?
Added test cases in `DDLSuite.scala`. Also manually verified all the related test cases in `HiveCompatibilitySuite` passed.
Author: gatorsmile <ga...@gmail.com>
Closes #12679 from gatorsmile/descFunction.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/162cf02e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/162cf02e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/162cf02e
Branch: refs/heads/master
Commit: 162cf02efa025fdb32adc3eaabb8e4232fe90e08
Parents: b208229
Author: gatorsmile <ga...@gmail.com>
Authored: Tue Apr 26 19:29:34 2016 +0200
Committer: Herman van Hovell <hv...@questtec.nl>
Committed: Tue Apr 26 19:29:34 2016 +0200
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/parser/SqlBase.g4 | 18 ++++++-
.../catalyst/analysis/FunctionRegistry.scala | 1 +
.../spark/sql/catalyst/parser/AstBuilder.scala | 12 ++++-
.../spark/sql/execution/command/DDLSuite.scala | 51 ++++++++++++++++++++
4 files changed, 79 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
index fa4b8c4..64f68c9 100644
--- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
+++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
@@ -107,7 +107,7 @@ statement
| SHOW TBLPROPERTIES table=tableIdentifier
('(' key=tablePropertyKey ')')? #showTblProperties
| SHOW FUNCTIONS (LIKE? (qualifiedName | pattern=STRING))? #showFunctions
- | (DESC | DESCRIBE) FUNCTION EXTENDED? qualifiedName #describeFunction
+ | (DESC | DESCRIBE) FUNCTION EXTENDED? describeFuncName #describeFunction
| (DESC | DESCRIBE) option=(EXTENDED | FORMATTED)?
tableIdentifier partitionSpec? describeColName? #describeTable
| (DESC | DESCRIBE) DATABASE EXTENDED? identifier #describeDatabase
@@ -220,6 +220,14 @@ partitionVal
: identifier (EQ constant)?
;
+describeFuncName
+ : qualifiedName
+ | STRING
+ | comparisonOperator
+ | arithmeticOperator
+ | predicateOperator
+ ;
+
describeColName
: identifier ('.' (identifier | STRING))*
;
@@ -519,6 +527,14 @@ comparisonOperator
: EQ | NEQ | NEQJ | LT | LTE | GT | GTE | NSEQ
;
+arithmeticOperator
+ : PLUS | MINUS | ASTERISK | SLASH | PERCENT | DIV | TILDE | AMPERSAND | PIPE | HAT
+ ;
+
+predicateOperator
+ : OR | AND | IN | NOT
+ ;
+
booleanValue
: TRUE | FALSE
;
http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index dd35770..1bada2c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -363,6 +363,7 @@ object FunctionRegistry {
expression[Not]("not"),
expression[Or]("or"),
+ // comparison operators
expression[EqualNullSafe]("<=>"),
expression[EqualTo]("="),
expression[EqualTo]("=="),
http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
index 3d7b888..7f98c21 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
@@ -112,8 +112,16 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with Logging {
* Create a plan for a DESCRIBE FUNCTION command.
*/
override def visitDescribeFunction(ctx: DescribeFunctionContext): LogicalPlan = withOrigin(ctx) {
- val functionName = ctx.qualifiedName().identifier().asScala.map(_.getText).mkString(".")
- DescribeFunction(functionName, ctx.EXTENDED != null)
+ import ctx._
+ val functionName =
+ if (describeFuncName.STRING() != null) {
+ string(describeFuncName.STRING())
+ } else if (describeFuncName.qualifiedName() != null) {
+ describeFuncName.qualifiedName().identifier().asScala.map(_.getText).mkString(".")
+ } else {
+ describeFuncName.getText
+ }
+ DescribeFunction(functionName, EXTENDED != null)
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/162cf02e/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 9ffffa0..e601ff1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -716,4 +716,55 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach {
}
}
+ test("describe function") {
+ checkAnswer(
+ sql("DESCRIBE FUNCTION log"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.Logarithm") ::
+ Row("Function: log") ::
+ Row("Usage: log(b, x) - Returns the logarithm of x with base b.") :: Nil
+ )
+ // predicate operator
+ checkAnswer(
+ sql("DESCRIBE FUNCTION or"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.Or") ::
+ Row("Function: or") ::
+ Row("Usage: a or b - Logical OR.") :: Nil
+ )
+ checkAnswer(
+ sql("DESCRIBE FUNCTION !"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.Not") ::
+ Row("Function: !") ::
+ Row("Usage: ! a - Logical not") :: Nil
+ )
+ // arithmetic operators
+ checkAnswer(
+ sql("DESCRIBE FUNCTION +"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.Add") ::
+ Row("Function: +") ::
+ Row("Usage: a + b - Returns a+b.") :: Nil
+ )
+ // comparison operators
+ checkAnswer(
+ sql("DESCRIBE FUNCTION <"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.LessThan") ::
+ Row("Function: <") ::
+ Row("Usage: a < b - Returns TRUE if a is less than b.") :: Nil
+ )
+ // STRING
+ checkAnswer(
+ sql("DESCRIBE FUNCTION 'concat'"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.Concat") ::
+ Row("Function: concat") ::
+ Row("Usage: concat(str1, str2, ..., strN) " +
+ "- Returns the concatenation of str1, str2, ..., strN") :: Nil
+ )
+ // extended mode
+ checkAnswer(
+ sql("DESCRIBE FUNCTION EXTENDED ^"),
+ Row("Class: org.apache.spark.sql.catalyst.expressions.BitwiseXor") ::
+ Row("Extended Usage:\n> SELECT 3 ^ 5; 2") ::
+ Row("Function: ^") ::
+ Row("Usage: a ^ b - Bitwise exclusive OR.") :: Nil
+ )
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org