You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/05/25 07:44:02 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 49352b5ac [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
49352b5ac is described below
commit 49352b5ac3376b82e91557ef40ad5ea6cb41ec76
Author: jiaoqingbo <11...@qq.com>
AuthorDate: Wed May 25 15:43:52 2022 +0800
[KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
### _Why are the changes needed?_
fix #2665
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #2737 from jiaoqingbo/kyuubi-2665.
Closes #2665
d0bbe20b [jiaoqingbo] [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
Authored-by: jiaoqingbo <11...@qq.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../kyuubi/spark/connector/tpch/TPCHCatalog.scala | 32 ++++++++++++++++++++--
.../spark/connector/tpch/TPCHCatalogSuite.scala | 6 ++++
2 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala b/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
index 630c90d09..a53a3485f 100644
--- a/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
@@ -23,12 +23,12 @@ import scala.collection.JavaConverters._
import io.trino.tpch.TpchTable
import org.apache.spark.sql.catalyst.analysis.{NoSuchNamespaceException, NoSuchTableException}
-import org.apache.spark.sql.connector.catalog.{Identifier, Table => SparkTable, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, SupportsNamespaces, Table => SparkTable, TableCatalog, TableChange}
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
-class TPCHCatalog extends TableCatalog {
+class TPCHCatalog extends TableCatalog with SupportsNamespaces {
val tables: Array[String] = TpchTable.getTables.asScala
.map(_.getTableName).toArray
@@ -75,4 +75,32 @@ class TPCHCatalog extends TableCatalog {
override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit =
throw new UnsupportedOperationException
+ override def listNamespaces(): Array[Array[String]] = databases.map(Array(_))
+
+ override def listNamespaces(namespace: Array[String]): Array[Array[String]] = namespace match {
+ case Array() => listNamespaces()
+ case Array(db) if databases contains db => Array.empty
+ case _ => throw new NoSuchNamespaceException(namespace)
+ }
+
+ override def loadNamespaceMetadata(namespace: Array[String]): util.Map[String, String] =
+ namespace match {
+ case Array(_) => Map.empty[String, String].asJava
+ case _ => throw new NoSuchNamespaceException(namespace)
+ }
+
+ override def createNamespace(namespace: Array[String], metadata: util.Map[String, String]): Unit =
+ throw new UnsupportedOperationException
+
+ override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit =
+ throw new UnsupportedOperationException
+
+ // Removed in SPARK-37929
+ def dropNamespace(namespace: Array[String]): Boolean =
+ throw new UnsupportedOperationException
+
+ // Introduced in SPARK-37929
+ def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean =
+ throw new UnsupportedOperationException
+
}
diff --git a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
index 9be0babb5..2496a0cd8 100644
--- a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
@@ -34,6 +34,12 @@ class TPCHCatalogSuite extends KyuubiFunSuite {
.getOrCreate()
}
+ test("supports namespaces") {
+ spark.sql("use tpch")
+ assert(spark.sql(s"SHOW DATABASES").collect().length == 11)
+ assert(spark.sql(s"SHOW NAMESPACES IN tpch.sf1").collect().length == 0)
+ }
+
test("tpch.sf1 count") {
assert(spark.table("tpch.sf1.customer").count === 150000)
assert(spark.table("tpch.sf1.orders").count === 1500000)