You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/05/25 07:44:02 UTC

[incubator-kyuubi] branch master updated: [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new 49352b5ac [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
49352b5ac is described below

commit 49352b5ac3376b82e91557ef40ad5ea6cb41ec76
Author: jiaoqingbo <11...@qq.com>
AuthorDate: Wed May 25 15:43:52 2022 +0800

    [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
    
    ### _Why are the changes needed?_
    
    fix #2665
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #2737 from jiaoqingbo/kyuubi-2665.
    
    Closes #2665
    
    d0bbe20b [jiaoqingbo] [KYUUBI #2665] Kyuubi Spark TPC-H Connector - SupportsNamespaces
    
    Authored-by: jiaoqingbo <11...@qq.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../kyuubi/spark/connector/tpch/TPCHCatalog.scala  | 32 ++++++++++++++++++++--
 .../spark/connector/tpch/TPCHCatalogSuite.scala    |  6 ++++
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala b/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
index 630c90d09..a53a3485f 100644
--- a/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpch/src/main/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalog.scala
@@ -23,12 +23,12 @@ import scala.collection.JavaConverters._
 
 import io.trino.tpch.TpchTable
 import org.apache.spark.sql.catalyst.analysis.{NoSuchNamespaceException, NoSuchTableException}
-import org.apache.spark.sql.connector.catalog.{Identifier, Table => SparkTable, TableCatalog, TableChange}
+import org.apache.spark.sql.connector.catalog.{Identifier, NamespaceChange, SupportsNamespaces, Table => SparkTable, TableCatalog, TableChange}
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
 
-class TPCHCatalog extends TableCatalog {
+class TPCHCatalog extends TableCatalog with SupportsNamespaces {
 
   val tables: Array[String] = TpchTable.getTables.asScala
     .map(_.getTableName).toArray
@@ -75,4 +75,32 @@ class TPCHCatalog extends TableCatalog {
   override def renameTable(oldIdent: Identifier, newIdent: Identifier): Unit =
     throw new UnsupportedOperationException
 
+  override def listNamespaces(): Array[Array[String]] = databases.map(Array(_))
+
+  override def listNamespaces(namespace: Array[String]): Array[Array[String]] = namespace match {
+    case Array() => listNamespaces()
+    case Array(db) if databases contains db => Array.empty
+    case _ => throw new NoSuchNamespaceException(namespace)
+  }
+
+  override def loadNamespaceMetadata(namespace: Array[String]): util.Map[String, String] =
+    namespace match {
+      case Array(_) => Map.empty[String, String].asJava
+      case _ => throw new NoSuchNamespaceException(namespace)
+    }
+
+  override def createNamespace(namespace: Array[String], metadata: util.Map[String, String]): Unit =
+    throw new UnsupportedOperationException
+
+  override def alterNamespace(namespace: Array[String], changes: NamespaceChange*): Unit =
+    throw new UnsupportedOperationException
+
+  // Removed in SPARK-37929
+  def dropNamespace(namespace: Array[String]): Boolean =
+    throw new UnsupportedOperationException
+
+  // Introduced in SPARK-37929
+  def dropNamespace(namespace: Array[String], cascade: Boolean): Boolean =
+    throw new UnsupportedOperationException
+
 }
diff --git a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
index 9be0babb5..2496a0cd8 100644
--- a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHCatalogSuite.scala
@@ -34,6 +34,12 @@ class TPCHCatalogSuite extends KyuubiFunSuite {
       .getOrCreate()
   }
 
+  test("supports namespaces") {
+    spark.sql("use tpch")
+    assert(spark.sql(s"SHOW DATABASES").collect().length == 11)
+    assert(spark.sql(s"SHOW NAMESPACES IN tpch.sf1").collect().length == 0)
+  }
+
   test("tpch.sf1 count") {
     assert(spark.table("tpch.sf1.customer").count === 150000)
     assert(spark.table("tpch.sf1.orders").count === 1500000)