You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/04/17 16:50:25 UTC
spark git commit: [SPARK-20349][SQL] ListFunctions returns duplicate
functions after using persistent functions
Repository: spark
Updated Branches:
refs/heads/master 24f09b39c -> 01ff0350a
[SPARK-20349][SQL] ListFunctions returns duplicate functions after using persistent functions
### What changes were proposed in this pull request?
The session catalog caches some persistent functions in the `FunctionRegistry`, so there can be duplicates. Our Catalog API `listFunctions` does not handle it.
It would be better if `SessionCatalog` API can de-duplciate the records, instead of doing it by each API caller. In `FunctionRegistry`, our functions are identified by the unquoted string. Thus, this PR is try to parse it using our parser interface and then de-duplicate the names.
### How was this patch tested?
Added test cases.
Author: Xiao Li <ga...@gmail.com>
Closes #17646 from gatorsmile/showFunctions.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/01ff0350
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/01ff0350
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/01ff0350
Branch: refs/heads/master
Commit: 01ff0350a85b179715946c3bd4f003db7c5e3641
Parents: 24f09b3
Author: Xiao Li <ga...@gmail.com>
Authored: Mon Apr 17 09:50:20 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Mon Apr 17 09:50:20 2017 -0700
----------------------------------------------------------------------
.../sql/catalyst/catalog/SessionCatalog.scala | 21 +++++++++++++++-----
.../spark/sql/execution/command/functions.scala | 4 +---
.../spark/sql/hive/execution/HiveUDFSuite.scala | 17 ++++++++++++++++
3 files changed, 34 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/01ff0350/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
index 1417bcc..3fbf83f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala
@@ -22,6 +22,7 @@ import java.util.Locale
import javax.annotation.concurrent.GuardedBy
import scala.collection.mutable
+import scala.util.{Failure, Success, Try}
import com.google.common.cache.{Cache, CacheBuilder}
import org.apache.hadoop.conf.Configuration
@@ -1202,15 +1203,25 @@ class SessionCatalog(
def listFunctions(db: String, pattern: String): Seq[(FunctionIdentifier, String)] = {
val dbName = formatDatabaseName(db)
requireDbExists(dbName)
- val dbFunctions = externalCatalog.listFunctions(dbName, pattern)
- .map { f => FunctionIdentifier(f, Some(dbName)) }
- val loadedFunctions = StringUtils.filterPattern(functionRegistry.listFunction(), pattern)
- .map { f => FunctionIdentifier(f) }
+ val dbFunctions = externalCatalog.listFunctions(dbName, pattern).map { f =>
+ FunctionIdentifier(f, Some(dbName)) }
+ val loadedFunctions =
+ StringUtils.filterPattern(functionRegistry.listFunction(), pattern).map { f =>
+ // In functionRegistry, function names are stored as an unquoted format.
+ Try(parser.parseFunctionIdentifier(f)) match {
+ case Success(e) => e
+ case Failure(_) =>
+ // The names of some built-in functions are not parsable by our parser, e.g., %
+ FunctionIdentifier(f)
+ }
+ }
val functions = dbFunctions ++ loadedFunctions
+ // The session catalog caches some persistent functions in the FunctionRegistry
+ // so there can be duplicates.
functions.map {
case f if FunctionRegistry.functionSet.contains(f.funcName) => (f, "SYSTEM")
case f => (f, "USER")
- }
+ }.distinct
}
http://git-wip-us.apache.org/repos/asf/spark/blob/01ff0350/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
index e0d0029..5450823 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/functions.scala
@@ -207,8 +207,6 @@ case class ShowFunctionsCommand(
case (f, "USER") if showUserFunctions => f.unquotedString
case (f, "SYSTEM") if showSystemFunctions => f.unquotedString
}
- // The session catalog caches some persistent functions in the FunctionRegistry
- // so there can be duplicates.
- functionNames.distinct.sorted.map(Row(_))
+ functionNames.sorted.map(Row(_))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/01ff0350/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index 4bbf925..4446af2 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -573,6 +573,23 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
checkAnswer(testData.selectExpr("statelessUDF() as s").agg(max($"s")), Row(1))
}
}
+
+ test("Show persistent functions") {
+ val testData = spark.sparkContext.parallelize(StringCaseClass("") :: Nil).toDF()
+ withTempView("inputTable") {
+ testData.createOrReplaceTempView("inputTable")
+ withUserDefinedFunction("testUDFToListInt" -> false) {
+ val numFunc = spark.catalog.listFunctions().count()
+ sql(s"CREATE FUNCTION testUDFToListInt AS '${classOf[UDFToListInt].getName}'")
+ assert(spark.catalog.listFunctions().count() == numFunc + 1)
+ checkAnswer(
+ sql("SELECT testUDFToListInt(s) FROM inputTable"),
+ Seq(Row(Seq(1, 2, 3))))
+ assert(sql("show functions").count() == numFunc + 1)
+ assert(spark.catalog.listFunctions().count() == numFunc + 1)
+ }
+ }
+ }
}
class TestPair(x: Int, y: Int) extends Writable with Serializable {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org