You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2022/06/30 19:04:05 UTC
[spark] branch master updated: [SPARK-39641][SQL][TESTS] Unify v1 and v2 SHOW FUNCTIONS tests
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 91f95a7de9e [SPARK-39641][SQL][TESTS] Unify v1 and v2 SHOW FUNCTIONS tests
91f95a7de9e is described below
commit 91f95a7de9e524b9fd4730a3125c17622b67a760
Author: Max Gekk <ma...@gmail.com>
AuthorDate: Thu Jun 30 22:03:30 2022 +0300
[SPARK-39641][SQL][TESTS] Unify v1 and v2 SHOW FUNCTIONS tests
### What changes were proposed in this pull request?
1. Move `SHOW FUNCTIONS` parsing tests to `ShowFunctionsParserSuite`.
2. Put common v2 `SHOW FUNCTIONS` tests to `v2.ShowFunctionsSuite`.
3. Refactor existing tests of temporary/permanent functions in v1 catalog, and place them to `v1.ShowTempFunctionsSuite` (temporary) and `hive.execution.command.ShowFunctionsSuite` (permanent).
### Why are the changes needed?
1. The unification will allow to run common `SHOW FUNCTIONS` tests for both DSv1/Hive DSv1 and DSv2
2. We can detect missing features and differences between DSv1 and DSv2 implementations.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
By running the modified test suites:
```
$ build/sbt "testOnly *DDLParserSuite"
$ build/sbt "test:testOnly *DDLSuite"
$ build/sbt "test:testOnly *DataSourceV2FunctionSuite"
$ build/sbt "test:testOnly *SQLQuerySuite"
```
and new test suites:
```
$ build/sbt "sql/test:testOnly *ShowFunctionsParserSuite"
$ build/sbt -Phive-2.3 -Phive-thriftserver "test:testOnly *ShowFunctionsSuite"
$ build/sbt "test:testOnly *ShowTempFunctionsSuite"
```
Closes #37017 from MaxGekk/show-functions-v2.
Lead-authored-by: Max Gekk <ma...@gmail.com>
Co-authored-by: Maxim Gekk <ma...@gmail.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../spark/sql/catalyst/parser/DDLParserSuite.scala | 39 --------
.../sql/connector/catalog/InMemoryCatalog.scala | 4 +
.../scala/org/apache/spark/sql/SQLQuerySuite.scala | 38 --------
.../sql/connector/DataSourceV2FunctionSuite.scala | 9 --
.../spark/sql/execution/command/DDLSuite.scala | 19 +---
.../command/ShowFunctionsParserSuite.scala | 68 +++++++++++++
.../execution/command/ShowFunctionsSuiteBase.scala | 72 ++++++++++++++
.../execution/command/v1/ShowFunctionsSuite.scala | 107 +++++++++++++++++++++
.../execution/command/v2/CommandSuiteBase.scala | 19 +++-
.../execution/command/v2/ShowFunctionsSuite.scala | 40 ++++++++
.../spark/sql/hive/execution/HiveUDFSuite.scala | 27 ------
.../spark/sql/hive/execution/SQLQuerySuite.scala | 52 +---------
.../execution/command/ShowFunctionsSuite.scala | 71 ++++++++++++++
13 files changed, 382 insertions(+), 183 deletions(-)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
index 62195823bcc..f06377f36a8 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala
@@ -2042,45 +2042,6 @@ class DDLParserSuite extends AnalysisTest {
DescribeFunction(createFuncPlan(Seq("a", "b", "c")), true))
}
- test("SHOW FUNCTIONS") {
- val nsPlan = UnresolvedNamespace(Nil)
- comparePlans(
- parsePlan("SHOW FUNCTIONS"),
- ShowFunctions(nsPlan, true, true, None))
- comparePlans(
- parsePlan("SHOW USER FUNCTIONS"),
- ShowFunctions(nsPlan, true, false, None))
- comparePlans(
- parsePlan("SHOW user FUNCTIONS"),
- ShowFunctions(nsPlan, true, false, None))
- comparePlans(
- parsePlan("SHOW SYSTEM FUNCTIONS"),
- ShowFunctions(nsPlan, false, true, None))
- comparePlans(
- parsePlan("SHOW ALL FUNCTIONS"),
- ShowFunctions(nsPlan, true, true, None))
- comparePlans(
- parsePlan("SHOW FUNCTIONS 'funct*'"),
- ShowFunctions(nsPlan, true, true, Some("funct*")))
- comparePlans(
- parsePlan("SHOW FUNCTIONS LIKE 'funct*'"),
- ShowFunctions(nsPlan, true, true, Some("funct*")))
- comparePlans(
- parsePlan("SHOW FUNCTIONS IN db LIKE 'funct*'"),
- ShowFunctions(UnresolvedNamespace(Seq("db")), true, true, Some("funct*")))
-
- // The legacy syntax.
- comparePlans(
- parsePlan("SHOW FUNCTIONS a"),
- ShowFunctions(nsPlan, true, true, Some("a")))
- comparePlans(
- parsePlan("SHOW FUNCTIONS LIKE a"),
- ShowFunctions(nsPlan, true, true, Some("a")))
- comparePlans(
- parsePlan("SHOW FUNCTIONS LIKE a.b.c"),
- ShowFunctions(UnresolvedNamespace(Seq("a", "b")), true, true, Some("c")))
- }
-
test("REFRESH FUNCTION") {
def createFuncPlan(name: Seq[String]): UnresolvedFunc = {
UnresolvedFunc(name, "REFRESH FUNCTION", true, None)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
index be3baf92520..aa9914d760f 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryCatalog.scala
@@ -56,6 +56,10 @@ class InMemoryCatalog extends InMemoryTableCatalog with FunctionCatalog {
functions.put(ident, fn)
}
+ def dropFunction(ident: Identifier): Unit = {
+ functions.remove(ident)
+ }
+
def clearFunctions(): Unit = {
functions.clear()
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 056e59b3b86..edc8d3dae77 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -30,12 +30,10 @@ import org.apache.commons.io.FileUtils
import org.apache.spark.{AccumulatorSuite, SparkException}
import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart}
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.expressions.{GenericRow, Hex}
import org.apache.spark.sql.catalyst.expressions.aggregate.{Complete, Partial}
import org.apache.spark.sql.catalyst.optimizer.{ConvertToLocalRelation, NestedColumnAliasingSuite}
import org.apache.spark.sql.catalyst.plans.logical.{LocalLimit, Project, RepartitionByExpression, Sort}
-import org.apache.spark.sql.catalyst.util.StringUtils
import org.apache.spark.sql.execution.{CommandResultExec, UnionExec}
import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
import org.apache.spark.sql.execution.aggregate._
@@ -78,42 +76,6 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
}
}
- test("show functions") {
- def getFunctions(pattern: String): Seq[Row] = {
- StringUtils.filterPattern(
- spark.sessionState.catalog.listFunctions("default").map(_._1.funcName)
- ++ FunctionRegistry.builtinOperators.keys, pattern)
- .map(Row(_))
- }
-
- def createFunction(names: Seq[String]): Unit = {
- names.foreach { name =>
- spark.udf.register(name, (arg1: Int, arg2: String) => arg2 + arg1)
- }
- }
-
- def dropFunction(names: Seq[String]): Unit = {
- names.foreach { name =>
- spark.sessionState.catalog.dropTempFunction(name, false)
- }
- }
-
- val functions = Array("ilog", "logi", "logii", "logiii", "crc32i", "cubei", "cume_disti",
- "isize", "ispace", "to_datei", "date_addi", "current_datei")
-
- createFunction(functions)
-
- checkAnswer(sql("SHOW functions"), getFunctions("*"))
- assert(sql("SHOW functions").collect().size > 200)
-
- Seq("^c*", "*e$", "log*", "*date*").foreach { pattern =>
- // For the pattern part, only '*' and '|' are allowed as wildcards.
- // For '*', we need to replace it to '.*'.
- checkAnswer(sql(s"SHOW FUNCTIONS '$pattern'"), getFunctions(pattern))
- }
- dropFunction(functions)
- }
-
test("describe functions") {
checkKeywordsExist(sql("describe function extended upper"),
"Function: upper",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
index 4a070ace377..8277a8e2979 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2FunctionSuite.scala
@@ -168,15 +168,6 @@ class DataSourceV2FunctionSuite extends DatasourceV2SQLBase {
assert(e1.message.contains("requires a single-part namespace"))
}
- test("SHOW FUNCTIONS: only support session catalog") {
- addFunction(Identifier.of(Array.empty, "abc"), new JavaStrLen(new JavaStrLenNoImpl))
-
- val e = intercept[AnalysisException] {
- sql(s"SHOW FUNCTIONS LIKE testcat.abc")
- }
- assert(e.message.contains("Catalog testcat does not support functions"))
- }
-
test("DROP FUNCTION: only support session catalog") {
addFunction(Identifier.of(Array.empty, "abc"), new JavaStrLen(new JavaStrLenNoImpl))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
index 1921de9b8b4..f0925826b09 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala
@@ -28,7 +28,7 @@ import org.apache.spark.{SparkException, SparkFiles}
import org.apache.spark.internal.config
import org.apache.spark.sql.{AnalysisException, QueryTest, Row, SaveMode}
import org.apache.spark.sql.catalyst.{FunctionIdentifier, QualifiedTableName, TableIdentifier}
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TableFunctionRegistry, TempTableAlreadyExistsException}
+import org.apache.spark.sql.catalyst.analysis.TempTableAlreadyExistsException
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
import org.apache.spark.sql.connector.catalog.SupportsNamespaces.PROP_OWNER
@@ -1464,23 +1464,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils {
}
}
- test("show functions") {
- withUserDefinedFunction("add_one" -> true) {
- val numFunctions = FunctionRegistry.functionSet.size.toLong +
- TableFunctionRegistry.functionSet.size.toLong +
- FunctionRegistry.builtinOperators.size.toLong
- assert(sql("show functions").count() === numFunctions)
- assert(sql("show system functions").count() === numFunctions)
- assert(sql("show all functions").count() === numFunctions)
- assert(sql("show user functions").count() === 0L)
- spark.udf.register("add_one", (x: Long) => x + 1)
- assert(sql("show functions").count() === numFunctions + 1L)
- assert(sql("show system functions").count() === numFunctions)
- assert(sql("show all functions").count() === numFunctions + 1L)
- assert(sql("show user functions").count() === 1L)
- }
- }
-
test("show columns - negative test") {
// When case sensitivity is true, the user supplied database name in table identifier
// should match the supplied database name in case sensitive way.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsParserSuite.scala
new file mode 100644
index 00000000000..cd99d2f3cb0
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsParserSuite.scala
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace}
+import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan
+import org.apache.spark.sql.catalyst.plans.logical.ShowFunctions
+
+class ShowFunctionsParserSuite extends AnalysisTest {
+ val nsPlan = UnresolvedNamespace(Nil)
+
+ test("show functions in the scope") {
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS"),
+ ShowFunctions(nsPlan, true, true, None))
+ comparePlans(
+ parsePlan("SHOW USER FUNCTIONS"),
+ ShowFunctions(nsPlan, true, false, None))
+ comparePlans(
+ parsePlan("SHOW user FUNCTIONS"),
+ ShowFunctions(nsPlan, true, false, None))
+ comparePlans(
+ parsePlan("SHOW SYSTEM FUNCTIONS"),
+ ShowFunctions(nsPlan, false, true, None))
+ comparePlans(
+ parsePlan("SHOW ALL FUNCTIONS"),
+ ShowFunctions(nsPlan, true, true, None))
+ }
+
+ test("show functions matched to a pattern") {
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS 'funct*'"),
+ ShowFunctions(nsPlan, true, true, Some("funct*")))
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS LIKE 'funct*'"),
+ ShowFunctions(nsPlan, true, true, Some("funct*")))
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS IN db LIKE 'funct*'"),
+ ShowFunctions(UnresolvedNamespace(Seq("db")), true, true, Some("funct*")))
+ }
+
+ test("show functions using the legacy syntax") {
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS a"),
+ ShowFunctions(nsPlan, true, true, Some("a")))
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS LIKE a"),
+ ShowFunctions(nsPlan, true, true, Some("a")))
+ comparePlans(
+ parsePlan("SHOW FUNCTIONS LIKE a.b.c"),
+ ShowFunctions(UnresolvedNamespace(Seq("a", "b")), true, true, Some("c")))
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
new file mode 100644
index 00000000000..130914d35d9
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowFunctionsSuiteBase.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command
+
+import java.util.Locale
+
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.util.Utils
+
+/**
+ * This base suite contains unified tests for the `SHOW FUNCTIONS` command that check V1 and V2
+ * table catalogs. The tests that cannot run for all supported catalogs are located in more
+ * specific test suites:
+ *
+ * - V2 catalog tests: `org.apache.spark.sql.execution.command.v2.ShowFunctionsSuite`
+ * - V1 catalog tests:
+ * `org.apache.spark.sql.execution.command.v1.ShowFunctionsSuiteBase`
+ * - Temporary functions:
+ * `org.apache.spark.sql.execution.command.v1.ShowTempFunctionsSuite`
+ * - Permanent functions:
+ * `org.apache.spark.sql.hive.execution.command.ShowFunctionsSuite`
+ */
+trait ShowFunctionsSuiteBase extends QueryTest with DDLCommandTestUtils {
+ override val command = "SHOW FUNCTIONS"
+
+ protected def createFunction(name: String): Unit = {}
+ protected def dropFunction(name: String): Unit = {}
+ protected def showFun(ns: String, name: String): String = s"$ns.$name".toLowerCase(Locale.ROOT)
+
+ /**
+ * Drops function `funName` after calling `f`.
+ */
+ protected def withFunction(functionNames: String*)(f: => Unit): Unit = {
+ Utils.tryWithSafeFinally(f) {
+ functionNames.foreach(dropFunction)
+ }
+ }
+
+ protected def withNamespaceAndFuns(ns: String, funNames: Seq[String], cat: String = catalog)
+ (f: (String, Seq[String]) => Unit): Unit = {
+ val nsCat = s"$cat.$ns"
+ withNamespace(nsCat) {
+ sql(s"CREATE NAMESPACE $nsCat")
+ val nsCatFns = funNames.map(funName => s"$nsCat.$funName")
+ withFunction(nsCatFns: _*) {
+ f(nsCat, nsCatFns)
+ }
+ }
+ }
+
+ protected def withNamespaceAndFun(ns: String, funName: String, cat: String = catalog)
+ (f: (String, String) => Unit): Unit = {
+ withNamespaceAndFuns(ns, Seq(funName), cat) { case (ns, Seq(name)) =>
+ f(ns, name)
+ }
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowFunctionsSuite.scala
new file mode 100644
index 00000000000..f14b550688b
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowFunctionsSuite.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v1
+
+import java.util.Locale
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.command
+
+/**
+ * This base suite contains unified tests for the `SHOW FUNCTIONS` command that checks V1
+ * table catalogs. The tests that cannot run for all V1 catalogs are located in more
+ * specific test suites:
+ *
+ * - Temporary functions of V1 catalog:
+ * `org.apache.spark.sql.execution.command.v1.ShowTempFunctionsSuite`
+ * - Permanent functions of V1 catalog:
+ * `org.apache.spark.sql.hive.execution.command.ShowFunctionsSuite`
+ */
+trait ShowFunctionsSuiteBase extends command.ShowFunctionsSuiteBase
+ with command.TestsV1AndV2Commands {
+
+ test("show a function") {
+ withNamespaceAndFun("ns", "iiilog") { (ns, f) =>
+ val totalFuns = sql(s"SHOW FUNCTIONS IN $ns").count()
+ createFunction(f)
+ assert(sql(s"SHOW FUNCTIONS IN $ns").count() - totalFuns === 1)
+ assert(!sql(s"SHOW FUNCTIONS IN $ns").filter("contains(function, 'iiilog')").isEmpty)
+ }
+ }
+
+ test("show a function in the USER name space") {
+ withNamespaceAndFun("ns", "logiii") { (ns, f) =>
+ assert(sql(s"SHOW USER FUNCTIONS IN $ns").count() === 0)
+ createFunction(f)
+ checkAnswer(sql(s"SHOW USER FUNCTIONS IN $ns"), Row(showFun("ns", "logiii")))
+ }
+ }
+
+ test("show functions in the SYSTEM name space") {
+ withNamespaceAndFun("ns", "date_addi") { (ns, f) =>
+ val systemFuns = sql(s"SHOW SYSTEM FUNCTIONS IN $ns").count()
+ assert(systemFuns > 0)
+ createFunction(f)
+ assert(sql(s"SHOW SYSTEM FUNCTIONS IN $ns").count() === systemFuns)
+ }
+ }
+
+ test("show functions among both user and system defined functions") {
+ withNamespaceAndFun("ns", "current_datei") { (ns, f) =>
+ val allFuns = sql(s"SHOW ALL FUNCTIONS IN $ns").collect()
+ assert(allFuns.nonEmpty)
+ createFunction(f)
+ checkAnswer(
+ sql(s"SHOW ALL FUNCTIONS IN $ns"),
+ allFuns :+ Row(showFun("ns", "current_datei")))
+ }
+ }
+
+ test("show functions matched to the wildcard pattern") {
+ val testFuns = Seq("crc32i", "crc16j", "date1900", "Date1")
+ withNamespaceAndFuns("ns", testFuns) { (ns, funs) =>
+ assert(sql(s"SHOW USER FUNCTIONS IN $ns").isEmpty)
+ funs.foreach(createFunction)
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS IN $ns LIKE '*'"),
+ testFuns.map(testFun => Row(showFun("ns", testFun))))
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS IN $ns LIKE '*rc*'"),
+ Seq("crc32i", "crc16j").map(testFun => Row(showFun("ns", testFun))))
+ }
+ }
+}
+
+/**
+ * The class contains tests for the `SHOW FUNCTIONS` command to check temporary functions.
+ */
+class ShowTempFunctionsSuite extends ShowFunctionsSuiteBase with CommandSuiteBase {
+ override def commandVersion: String = super[ShowFunctionsSuiteBase].commandVersion
+
+ override protected def createFunction(name: String): Unit = {
+ spark.udf.register(name, (arg1: Int, arg2: String) => arg2 + arg1)
+ }
+
+ override protected def dropFunction(name: String): Unit = {
+ spark.sessionState.catalog.dropTempFunction(name, false)
+ }
+
+ override protected def showFun(ns: String, name: String): String = {
+ s"$catalog.$ns.$name".toLowerCase(Locale.ROOT)
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala
index 1ff9e74f180..ac38a589ffd 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/CommandSuiteBase.scala
@@ -20,8 +20,10 @@ package org.apache.spark.sql.execution.command.v2
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.analysis.ResolvePartitionSpec
import org.apache.spark.sql.catalyst.catalog.CatalogTypes.TablePartitionSpec
-import org.apache.spark.sql.connector.catalog.{CatalogV2Implicits, Identifier, InMemoryPartitionTable, InMemoryPartitionTableCatalog, InMemoryTableCatalog}
+import org.apache.spark.sql.connector.catalog.{CatalogV2Implicits, Identifier, InMemoryCatalog, InMemoryPartitionTable, InMemoryPartitionTableCatalog, InMemoryTableCatalog}
+import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
/**
* The trait contains settings and utility functions. It can be mixed to the test suites for
@@ -33,12 +35,14 @@ trait CommandSuiteBase extends SharedSparkSession {
def catalogVersion: String = "V2" // The catalog version is added to test names
def commandVersion: String = "V2" // The command version is added to test names
def catalog: String = "test_catalog" // The default V2 catalog for testing
+ def funCatalog: String = s"fun_$catalog"
def defaultUsing: String = "USING _" // The clause is used in creating v2 tables under testing
// V2 catalogs created and used especially for testing
override def sparkConf: SparkConf = super.sparkConf
.set(s"spark.sql.catalog.$catalog", classOf[InMemoryPartitionTableCatalog].getName)
.set(s"spark.sql.catalog.non_part_$catalog", classOf[InMemoryTableCatalog].getName)
+ .set(s"spark.sql.catalog.$funCatalog", classOf[InMemoryCatalog].getName)
def checkLocation(
t: String,
@@ -61,4 +65,17 @@ trait CommandSuiteBase extends SharedSparkSession {
assert(partMetadata.containsKey("location"))
assert(partMetadata.get("location") === expected)
}
+
+
+ def withFun(ident: Identifier, fn: UnboundFunction)(f: => Unit): Unit = {
+ val cat = spark.sessionState
+ .catalogManager
+ .catalog(funCatalog)
+ .asInstanceOf[InMemoryCatalog]
+
+ cat.createFunction(ident, fn)
+ Utils.tryWithSafeFinally(f) {
+ cat.dropFunction(ident)
+ }
+ }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowFunctionsSuite.scala
new file mode 100644
index 00000000000..88c16a5552a
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowFunctionsSuite.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.command.v2
+
+import test.org.apache.spark.sql.connector.catalog.functions.JavaStrLen
+import test.org.apache.spark.sql.connector.catalog.functions.JavaStrLen.JavaStrLenNoImpl
+
+import org.apache.spark.sql.AnalysisException
+import org.apache.spark.sql.connector.catalog.Identifier
+import org.apache.spark.sql.execution.command
+
+/**
+ * The class contains tests for the `SHOW FUNCTIONS` command to check V2 table catalogs.
+ */
+class ShowFunctionsSuite extends command.ShowFunctionsSuiteBase with CommandSuiteBase {
+
+ test("only support session catalog") {
+ withFun(Identifier.of(Array.empty, "abc"), new JavaStrLen(new JavaStrLenNoImpl)) {
+ val e = intercept[AnalysisException] {
+ sql(s"SHOW FUNCTIONS LIKE $funCatalog.abc")
+ }
+ assert(e.getMessage === s"Catalog $funCatalog does not support functions")
+ }
+ }
+}
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
index fc2501c117d..084850e6f0c 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala
@@ -34,7 +34,6 @@ import org.apache.hadoop.io.{LongWritable, Writable}
import org.apache.spark.{SparkFiles, TestUtils}
import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.catalyst.plans.logical.Project
import org.apache.spark.sql.functions.max
import org.apache.spark.sql.hive.test.TestHiveSingleton
@@ -551,32 +550,6 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils {
}
}
- test("Show persistent functions") {
- val testData = spark.sparkContext.parallelize(StringCaseClass("") :: Nil).toDF()
- withTempView("inputTable") {
- testData.createOrReplaceTempView("inputTable")
- withUserDefinedFunction("testUDFToListInt" -> false) {
- val numFunc = spark.catalog.listFunctions().count()
- sql(s"CREATE FUNCTION testUDFToListInt AS '${classOf[UDFToListInt].getName}'")
- assert(spark.catalog.listFunctions().count() == numFunc + 1)
- checkAnswer(
- sql("SELECT testUDFToListInt(s) FROM inputTable"),
- Seq(Row(Seq(1, 2, 3))))
- assert(sql("show functions").count() ==
- numFunc + FunctionRegistry.builtinOperators.size + 1)
- assert(spark.catalog.listFunctions().count() == numFunc + 1)
-
- withDatabase("db2") {
- sql("CREATE DATABASE db2")
- sql(s"CREATE FUNCTION db2.testUDFToListInt AS '${classOf[UDFToListInt].getName}'")
- checkAnswer(
- sql("SHOW FUNCTIONS IN db2 LIKE 'testUDF*'"),
- Seq(Row("db2.testudftolistint")))
- }
- }
- }
- }
-
test("Temp function has dots in the names") {
withUserDefinedFunction("test_avg" -> false, "`default.test_avg`" -> true) {
sql(s"CREATE FUNCTION test_avg AS '${classOf[GenericUDAFAverage].getName}'")
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index f2711db8399..9ce48a9ab75 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -29,7 +29,7 @@ import org.apache.hadoop.fs.{FileSystem, Path}
import org.apache.spark.{SparkException, TestUtils}
import org.apache.spark.sql._
import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.analysis.{EliminateSubqueryAliases, FunctionRegistry}
+import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases
import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, CatalogUtils, HiveTableRelation}
import org.apache.spark.sql.catalyst.parser.ParseException
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias}
@@ -195,56 +195,6 @@ abstract class SQLQuerySuiteBase extends QueryTest with SQLTestUtils with TestHi
}
}
- test("show functions") {
- val allBuiltinFunctions = FunctionRegistry.builtin.listFunction().map(_.unquotedString)
- val allFunctions = sql("SHOW functions").collect().map(r => r(0))
- allBuiltinFunctions.foreach { f =>
- assert(allFunctions.contains(f))
- }
-
- FunctionRegistry.builtinOperators.keys.foreach { f =>
- assert(allFunctions.contains(f))
- }
-
- withTempDatabase { db =>
- def createFunction(names: Seq[String]): Unit = {
- names.foreach { name =>
- sql(
- s"""
- |CREATE TEMPORARY FUNCTION $name
- |AS '${classOf[PairUDF].getName}'
- """.stripMargin)
- }
- }
- def dropFunction(names: Seq[String]): Unit = {
- names.foreach { name =>
- sql(s"DROP TEMPORARY FUNCTION $name")
- }
- }
- createFunction(Seq("temp_abs", "temp_weekofyear", "temp_sha", "temp_sha1", "temp_sha2"))
-
- checkAnswer(sql("SHOW functions temp_abs"), Row("temp_abs"))
- checkAnswer(sql("SHOW functions 'temp_abs'"), Row("temp_abs"))
- checkAnswer(sql(s"SHOW functions $db.temp_abs"), Row("temp_abs"))
- checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs"))
- checkAnswer(sql(s"SHOW functions `$db`.`temp_abs`"), Row("temp_abs"))
- checkAnswer(sql("SHOW functions `a function doesn't exist`"), Nil)
- checkAnswer(sql("SHOW functions `temp_weekofyea*`"), Row("temp_weekofyear"))
-
- // this probably will failed if we add more function with `sha` prefixing.
- checkAnswer(
- sql("SHOW functions `temp_sha*`"),
- List(Row("temp_sha"), Row("temp_sha1"), Row("temp_sha2")))
-
- // Test '|' for alternation.
- checkAnswer(
- sql("SHOW functions 'temp_sha*|temp_weekofyea*'"),
- List(Row("temp_sha"), Row("temp_sha1"), Row("temp_sha2"), Row("temp_weekofyear")))
-
- dropFunction(Seq("temp_abs", "temp_weekofyear", "temp_sha", "temp_sha1", "temp_sha2"))
- }
- }
-
test("describe functions - built-in functions") {
checkKeywordsExist(sql("describe function extended upper"),
"Function: upper",
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowFunctionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowFunctionsSuite.scala
new file mode 100644
index 00000000000..9d40f33f404
--- /dev/null
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowFunctionsSuite.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.hive.execution.command
+
+import org.apache.spark.sql.Row
+import org.apache.spark.sql.execution.command.v1
+import org.apache.spark.sql.hive.execution.UDFToListInt
+
+/**
+ * The class contains tests for the `SHOW FUNCTIONS` command to check permanent functions.
+ */
+class ShowFunctionsSuite extends v1.ShowFunctionsSuiteBase with CommandSuiteBase {
+ override def commandVersion: String = super[ShowFunctionsSuiteBase].commandVersion
+
+ override protected def createFunction(name: String): Unit = {
+ sql(s"CREATE FUNCTION $name AS '${classOf[UDFToListInt].getName}'")
+ }
+ override protected def dropFunction(name: String): Unit = {
+ sql(s"DROP FUNCTION IF EXISTS $name")
+ }
+
+ test("show a function by its string name") {
+ val testFuns = Seq("crc32i", "crc16j")
+ withNamespaceAndFuns("ns", testFuns) { (ns, funs) =>
+ assert(sql(s"SHOW USER FUNCTIONS IN $ns").isEmpty)
+ funs.foreach(createFunction)
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS IN $ns 'crc32i'"),
+ Row(showFun("ns", "crc32i")))
+ }
+ }
+
+ test("show functions matched to the '|' pattern") {
+ val testFuns = Seq("crc32i", "crc16j", "date1900", "Date1")
+ withNamespaceAndFuns("ns", testFuns) { (ns, funs) =>
+ assert(sql(s"SHOW USER FUNCTIONS IN $ns").isEmpty)
+ funs.foreach(createFunction)
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS IN $ns LIKE 'crc32i|date1900'"),
+ Seq("crc32i", "date1900").map(testFun => Row(showFun("ns", testFun))))
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS IN $ns LIKE 'crc32i|date*'"),
+ Seq("crc32i", "date1900", "Date1").map(testFun => Row(showFun("ns", testFun))))
+ }
+ }
+
+ test("show a function by its id") {
+ withNamespaceAndFun("ns", "crc32i") { (ns, fun) =>
+ assert(sql(s"SHOW USER FUNCTIONS IN $ns").isEmpty)
+ createFunction(fun)
+ checkAnswer(
+ sql(s"SHOW USER FUNCTIONS $fun"),
+ Row(showFun("ns", "crc32i")))
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org