You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/10/08 08:05:28 UTC

[incubator-kyuubi] branch branch-1.6 updated: [KYUUBI #3504] Extend JDBC URL to support catalog

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/branch-1.6 by this push:
     new a8a232e4f [KYUUBI #3504] Extend JDBC URL to support catalog
a8a232e4f is described below

commit a8a232e4ff6fded2ed99f82b2d07b628418e629f
Author: jiaoqingbo <11...@qq.com>
AuthorDate: Sat Oct 8 16:05:03 2022 +0800

    [KYUUBI #3504] Extend JDBC URL to support catalog
    
    ### _Why are the changes needed?_
    
    fix #3504
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #3516 from jiaoqingbo/kyuubi3504.
    
    Closes #3504
    
    93fba696 [jiaoqingbo] remove outdated comment
    af3e6a16 [jiaoqingbo] code review
    894877b7 [jiaoqingbo] code review
    856b28fc [jiaoqingbo] code review
    60876a95 [jiaoqingbo] [KYUUBI #3504] Extend JDBC URL to support catalog
    
    Authored-by: jiaoqingbo <11...@qq.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
    (cherry picked from commit f76834415a30eacbf2bfa702c7ae34ed6c814e7b)
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../engine/flink/session/FlinkSessionImpl.scala    |  8 ++++++
 .../engine/spark/session/SparkSessionImpl.scala    | 10 ++++---
 .../engine/trino/session/TrinoSessionImpl.scala    | 31 ++++++++++++----------
 .../kyuubi/jdbc/hive/JdbcConnectionParams.java     | 13 ++++++++-
 .../apache/kyuubi/jdbc/hive/KyuubiConnection.java  |  4 +++
 .../java/org/apache/kyuubi/jdbc/hive/Utils.java    | 22 ++++++++++++---
 .../org/apache/kyuubi/jdbc/hive/UtilsTest.java     | 30 ++++++++++++++++-----
 .../apache/kyuubi/session/KyuubiSessionImpl.scala  |  1 +
 8 files changed, 92 insertions(+), 27 deletions(-)

diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala
index 7c6107ee3..03d9ce42e 100644
--- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala
+++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala
@@ -54,6 +54,14 @@ class FlinkSessionImpl(
   override def open(): Unit = {
     executor.openSession(handle.identifier.toString)
     normalizedConf.foreach {
+      case ("use:catalog", catalog) =>
+        val tableEnv = sessionContext.getExecutionContext.getTableEnvironment
+        try {
+          tableEnv.useCatalog(catalog)
+        } catch {
+          case NonFatal(e) =>
+            throw e
+        }
       case ("use:database", database) =>
         val tableEnv = sessionContext.getExecutionContext.getTableEnvironment
         try {
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
index 4985b95ac..eb4c84e24 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala
@@ -51,6 +51,13 @@ class SparkSessionImpl(
 
   override def open(): Unit = {
     normalizedConf.foreach {
+      case ("use:catalog", catalog) =>
+        try {
+          SparkCatalogShim().setCurrentCatalog(spark, catalog)
+        } catch {
+          case e if e.getMessage.contains("Cannot find catalog plugin class for catalog") =>
+            warn(e.getMessage())
+        }
       case ("use:database", database) =>
         try {
           SparkCatalogShim().setCurrentDatabase(spark, database)
@@ -58,9 +65,6 @@ class SparkSessionImpl(
           case e
               if database == "default" && e.getMessage != null &&
                 e.getMessage.contains("not found") =>
-          // use:database is from hive so the catalog is always session catalog which must have
-          // default namespace `default`. But as spark support v2 catalog, catalog may not have
-          // default namespace. Here we do nothing for compatible both session and v2 catalog.
         }
       case (key, value) => setModifiableConfig(key, value)
     }
diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala
index ac7c246e8..a19d74d58 100644
--- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala
+++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala
@@ -19,9 +19,7 @@ package org.apache.kyuubi.engine.trino.session
 
 import java.net.URI
 import java.time.ZoneId
-import java.util.Collections
-import java.util.Locale
-import java.util.Optional
+import java.util.{Collections, Locale, Optional}
 import java.util.concurrent.TimeUnit
 
 import io.airlift.units.Duration
@@ -36,8 +34,7 @@ import org.apache.kyuubi.engine.trino.{TrinoConf, TrinoContext, TrinoStatement}
 import org.apache.kyuubi.engine.trino.event.TrinoSessionEvent
 import org.apache.kyuubi.events.EventBus
 import org.apache.kyuubi.operation.{Operation, OperationHandle}
-import org.apache.kyuubi.session.AbstractSession
-import org.apache.kyuubi.session.SessionManager
+import org.apache.kyuubi.session.{AbstractSession, SessionManager}
 
 class TrinoSessionImpl(
     protocol: TProtocolVersion,
@@ -50,32 +47,38 @@ class TrinoSessionImpl(
 
   var trinoContext: TrinoContext = _
   private var clientSession: ClientSession = _
+  private var catalogName: String = null
+  private var databaseName: String = null
 
   private val sessionEvent = TrinoSessionEvent(this)
 
   override def open(): Unit = {
     normalizedConf.foreach {
-      case ("use:database", database) => clientSession = createClientSession(database)
+      case ("use:catalog", catalog) => catalogName = catalog
+      case ("use:database", database) => databaseName = database
       case _ => // do nothing
     }
 
     val httpClient = new OkHttpClient.Builder().build()
 
-    if (clientSession == null) {
-      clientSession = createClientSession()
-    }
+    clientSession = createClientSession()
     trinoContext = TrinoContext(httpClient, clientSession)
 
     super.open()
     EventBus.post(sessionEvent)
   }
 
-  private def createClientSession(schema: String = null): ClientSession = {
+  private def createClientSession(): ClientSession = {
     val sessionConf = sessionManager.getConf
     val connectionUrl = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_URL).getOrElse(
       throw KyuubiSQLException("Trino server url can not be null!"))
-    val catalog = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_CATALOG).getOrElse(
-      throw KyuubiSQLException("Trino default catalog can not be null!"))
+
+    if (catalogName == null) {
+      catalogName = sessionConf.get(
+        KyuubiConf.ENGINE_TRINO_CONNECTION_CATALOG).getOrElse(
+        throw KyuubiSQLException("Trino default catalog can not be null!"))
+    }
+
     val user = sessionConf
       .getOption(KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY).getOrElse(currentUser)
     val clientRequestTimeout = sessionConf.get(TrinoConf.CLIENT_REQUEST_TIMEOUT)
@@ -88,8 +91,8 @@ class TrinoSessionImpl(
       Optional.empty(),
       Collections.emptySet(),
       null,
-      catalog,
-      schema,
+      catalogName,
+      databaseName,
       null,
       ZoneId.systemDefault(),
       Locale.getDefault,
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java
index 670b7a243..385a9d6d6 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java
@@ -27,7 +27,8 @@ public class JdbcConnectionParams {
   // Prefer using a shorter camelCase param name instead of using the same name as the
   // corresponding
   // HiveServer2 config.
-  // For a jdbc url: jdbc:hive2://<host>:<port>/dbName;sess_var_list?hive_conf_list#hive_var_list,
+  // For a jdbc url:
+  // jdbc:hive2://<host>:<port>/catalogName/dbName;sess_var_list?hive_conf_list#hive_var_list,
   // client side params are specified in sess_var_list
 
   // Client param names:
@@ -115,6 +116,7 @@ public class JdbcConnectionParams {
   private String host = null;
   private int port = 0;
   private String jdbcUriString;
+  private String catalogName;
   private String dbName = Utils.DEFAULT_DATABASE;
   private Map<String, String> hiveConfs = new LinkedHashMap<>();
   private Map<String, String> hiveVars = new LinkedHashMap<>();
@@ -130,6 +132,7 @@ public class JdbcConnectionParams {
     this.host = params.host;
     this.port = params.port;
     this.jdbcUriString = params.jdbcUriString;
+    this.catalogName = params.catalogName;
     this.dbName = params.dbName;
     this.hiveConfs.putAll(params.hiveConfs);
     this.hiveVars.putAll(params.hiveVars);
@@ -152,6 +155,10 @@ public class JdbcConnectionParams {
     return jdbcUriString;
   }
 
+  public String getCatalogName() {
+    return catalogName;
+  }
+
   public String getDbName() {
     return dbName;
   }
@@ -196,6 +203,10 @@ public class JdbcConnectionParams {
     this.jdbcUriString = jdbcUriString;
   }
 
+  public void setCatalogName(String catalogName) {
+    this.catalogName = catalogName;
+  }
+
   public void setDbName(String dbName) {
     this.dbName = dbName;
   }
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
index 8076f9588..ec54e6fa1 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java
@@ -702,6 +702,10 @@ public class KyuubiConnection implements SQLConnection, KyuubiLoggable {
     for (Entry<String, String> hiveVar : connParams.getHiveVars().entrySet()) {
       openConf.put("set:hivevar:" + hiveVar.getKey(), hiveVar.getValue());
     }
+    // switch the catalog
+    if (connParams.getCatalogName() != null) {
+      openConf.put("use:catalog", connParams.getCatalogName());
+    }
     // switch the database
     openConf.put("use:database", connParams.getDbName());
     // set the fetchSize
diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
index 5309cdfc6..886daeb71 100644
--- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
+++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java
@@ -152,14 +152,27 @@ public class Utils {
     String sessVars = jdbcURI.getPath();
     if ((sessVars != null) && !sessVars.isEmpty()) {
       String dbName = "";
+      String catalogName = "";
       // removing leading '/' returned by getPath()
       sessVars = sessVars.substring(1);
       if (!sessVars.contains(";")) {
-        // only dbname is provided
-        dbName = sessVars;
+        if (sessVars.contains("/")) {
+          catalogName = sessVars.substring(0, sessVars.indexOf('/'));
+          dbName = sessVars.substring(sessVars.indexOf('/') + 1);
+        } else {
+          // only dbname is provided
+          dbName = sessVars;
+        }
       } else {
         // we have dbname followed by session parameters
-        dbName = sessVars.substring(0, sessVars.indexOf(';'));
+        String catalogAndDb = sessVars.substring(0, sessVars.indexOf(';'));
+        if (catalogAndDb.contains("/")) {
+          catalogName = catalogAndDb.substring(0, catalogAndDb.indexOf('/'));
+          dbName = catalogAndDb.substring(catalogAndDb.indexOf('/') + 1);
+        } else {
+          // only dbname is provided
+          dbName = catalogAndDb;
+        }
         sessVars = sessVars.substring(sessVars.indexOf(';') + 1);
         Matcher sessMatcher = pattern.matcher(sessVars);
         while (sessMatcher.find()) {
@@ -169,6 +182,9 @@ public class Utils {
           }
         }
       }
+      if (!catalogName.isEmpty()) {
+        connParams.setCatalogName(catalogName);
+      }
       if (!dbName.isEmpty()) {
         connParams.setDbName(dbName);
       }
diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
index c87b463c3..c890c8731 100644
--- a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
+++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java
@@ -33,23 +33,39 @@ public class UtilsTest {
 
   private String expectedHost;
   private String expectedPort;
+  private String expectedCatalog;
+  private String expectedDb;
   private String uri;
 
   @Parameterized.Parameters
   public static Collection<String[]> data() {
     return Arrays.asList(
         new String[][] {
-          {"localhost", "10009", "jdbc:hive2:///db;k1=v1?k2=v2#k3=v3"},
-          {"localhost", "10009", "jdbc:hive2:///"},
-          {"localhost", "10009", "jdbc:kyuubi://"},
-          {"localhost", "10009", "jdbc:hive2://"},
-          {"hostname", "10018", "jdbc:hive2://hostname:10018/db;k1=v1?k2=v2#k3=v3"}
+          {"localhost", "10009", null, "db", "jdbc:hive2:///db;k1=v1?k2=v2#k3=v3"},
+          {"localhost", "10009", null, "default", "jdbc:hive2:///"},
+          {"localhost", "10009", null, "default", "jdbc:kyuubi://"},
+          {"localhost", "10009", null, "default", "jdbc:hive2://"},
+          {"hostname", "10018", null, "db", "jdbc:hive2://hostname:10018/db;k1=v1?k2=v2#k3=v3"},
+          {
+            "hostname",
+            "10018",
+            "catalog",
+            "db",
+            "jdbc:hive2://hostname:10018/catalog/db;k1=v1?k2=v2#k3=v3"
+          }
         });
   }
 
-  public UtilsTest(String expectedHost, String expectedPort, String uri) {
+  public UtilsTest(
+      String expectedHost,
+      String expectedPort,
+      String expectedCatalog,
+      String expectedDb,
+      String uri) {
     this.expectedHost = expectedHost;
     this.expectedPort = expectedPort;
+    this.expectedCatalog = expectedCatalog;
+    this.expectedDb = expectedDb;
     this.uri = uri;
   }
 
@@ -58,5 +74,7 @@ public class UtilsTest {
     JdbcConnectionParams jdbcConnectionParams1 = extractURLComponents(uri, new Properties());
     assertEquals(expectedHost, jdbcConnectionParams1.getHost());
     assertEquals(Integer.parseInt(expectedPort), jdbcConnectionParams1.getPort());
+    assertEquals(expectedCatalog, jdbcConnectionParams1.getCatalogName());
+    assertEquals(expectedDb, jdbcConnectionParams1.getDbName());
   }
 }
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala
index edbb5c45e..2f96c20f5 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionImpl.scala
@@ -63,6 +63,7 @@ class KyuubiSessionImpl(
 
   // TODO: needs improve the hardcode
   optimizedConf.foreach {
+    case ("use:catalog", _) =>
     case ("use:database", _) =>
     case ("kyuubi.engine.pool.size.threshold", _) =>
     case (key, value) => sessionConf.set(key, value)