You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/07/14 10:48:49 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 419d725c0 [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
419d725c0 is described below
commit 419d725c0062b2399f8e6ff120cee35d1ab06b89
Author: Min <zh...@163.com>
AuthorDate: Thu Jul 14 18:48:39 2022 +0800
[KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
### _Why are the changes needed?_
close https://github.com/apache/incubator-kyuubi/issues/3018#issue-1294239449
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3024 from zhaomin1423/doris_columns.
Closes #3018
9dff7e85 [Min] fix style
87f45ba9 [Min] [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
9b13eef3 [Min] [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
d8074af9 [Min] [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
ec1e9583 [Min] [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
d7a1783d [zhaomin] [KYUUBI #3018] [Subtask] DorisSQLEngine - GetColumns Operation
Lead-authored-by: Min <zh...@163.com>
Co-authored-by: zhaomin <zh...@163.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../kyuubi/engine/jdbc/dialect/DorisDialect.scala | 41 +++++++-
.../kyuubi/engine/jdbc/dialect/JdbcDialect.scala | 7 +-
.../jdbc/operation/JdbcOperationManager.scala | 8 +-
.../engine/jdbc/doris/DorisOperationSuite.scala | 106 ++++++++++++++++++++-
4 files changed, 153 insertions(+), 9 deletions(-)
diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
index 8987e5cdc..7ce736a50 100644
--- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
+++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala
@@ -27,7 +27,7 @@ import org.apache.kyuubi.KyuubiSQLException
import org.apache.kyuubi.engine.jdbc.doris.{DorisRowSetHelper, DorisSchemaHelper}
import org.apache.kyuubi.engine.jdbc.schema.{RowSetHelper, SchemaHelper}
import org.apache.kyuubi.operation.Operation
-import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.{TABLE_CATALOG, TABLE_NAME, TABLE_SCHEMA, TABLE_TYPE}
+import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
import org.apache.kyuubi.session.Session
class DorisDialect extends JdbcDialect {
@@ -100,8 +100,43 @@ class DorisDialect extends JdbcDialect {
throw KyuubiSQLException.featureNotSupported()
}
- override def getColumnsOperation(session: Session): Operation = {
- throw KyuubiSQLException.featureNotSupported()
+ override def getColumnsQuery(
+ session: Session,
+ catalogName: String,
+ schemaName: String,
+ tableName: String,
+ columnName: String): String = {
+ val query = new StringBuilder(
+ """
+ |SELECT
+ |`TABLE_CATALOG`,`TABLE_SCHEMA`,`TABLE_NAME`, `COLUMN_NAME`,`ORDINAL_POSITION`,
+ |`COLUMN_DEFAULT`,`IS_NULLABLE`,`DATA_TYPE`,`CHARACTER_MAXIMUM_LENGTH`,
+ |`CHARACTER_OCTET_LENGTH`,`NUMERIC_PRECISION`,`NUMERIC_SCALE`,`DATETIME_PRECISION`,
+ |`CHARACTER_SET_NAME`,`COLLATION_NAME`,`COLUMN_TYPE`,`COLUMN_KEY`,`EXTRA`,`PRIVILEGES`,
+ |`COLUMN_COMMENT`,`COLUMN_SIZE`,`DECIMAL_DIGITS`,`GENERATION_EXPRESSION`,`SRS_ID`
+ |FROM information_schema.columns
+ |""".stripMargin)
+
+ val filters = ArrayBuffer[String]()
+ if (StringUtils.isNotEmpty(catalogName)) {
+ filters += s"$TABLE_CATALOG = '$catalogName'"
+ }
+ if (StringUtils.isNotEmpty(schemaName)) {
+ filters += s"$TABLE_SCHEMA LIKE '$schemaName'"
+ }
+ if (StringUtils.isNotEmpty(tableName)) {
+ filters += s"$TABLE_NAME LIKE '$tableName'"
+ }
+ if (StringUtils.isNotEmpty(columnName)) {
+ filters += s"$COLUMN_NAME LIKE '$columnName'"
+ }
+
+ if (filters.nonEmpty) {
+ query.append(" WHERE ")
+ query.append(filters.mkString(" AND "))
+ }
+
+ query.toString()
}
override def getFunctionsOperation(session: Session): Operation = {
diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala
index 3f79102ca..b7ac7f43b 100644
--- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala
+++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala
@@ -48,7 +48,12 @@ abstract class JdbcDialect extends SupportServiceLoader with Logging {
def getTableTypesOperation(session: Session): Operation
- def getColumnsOperation(session: Session): Operation
+ def getColumnsQuery(
+ session: Session,
+ catalogName: String,
+ schemaName: String,
+ tableName: String,
+ columnName: String): String
def getFunctionsOperation(session: Session): Operation
diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
index 1f31735b7..d10bb34cf 100644
--- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
+++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperationManager.scala
@@ -95,8 +95,10 @@ class JdbcOperationManager(conf: KyuubiConf) extends OperationManager("JdbcOpera
schemaName: String,
tableName: String,
columnName: String): Operation = {
- val operation = dialect.getColumnsOperation(session)
- addOperation(operation)
+ val query = dialect.getColumnsQuery(session, catalogName, schemaName, tableName, columnName)
+ val executeStatement =
+ new ExecuteStatement(session, query, false, 0L, true)
+ addOperation(executeStatement)
}
override def newGetFunctionsOperation(
@@ -104,7 +106,7 @@ class JdbcOperationManager(conf: KyuubiConf) extends OperationManager("JdbcOpera
catalogName: String,
schemaName: String,
functionName: String): Operation = {
- val operation = dialect.getColumnsOperation(session)
+ val operation = dialect.getFunctionsOperation(session)
addOperation(operation)
}
diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala
index 1364cf83b..4af943d0e 100644
--- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala
+++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/DorisOperationSuite.scala
@@ -16,13 +16,14 @@
*/
package org.apache.kyuubi.engine.jdbc.doris
+import java.sql.ResultSet
+
import scala.collection.mutable.ArrayBuffer
import org.apache.kyuubi.operation.HiveJDBCTestHelper
-import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.{TABLE_CATALOG, TABLE_NAME, TABLE_SCHEMA, TABLE_TYPE}
+import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
class DorisOperationSuite extends WithDorisEngine with HiveJDBCTestHelper {
-
test("doris - get tables") {
case class Table(catalog: String, schema: String, tableName: String, tableType: String)
@@ -157,5 +158,106 @@ class DorisOperationSuite extends WithDorisEngine with HiveJDBCTestHelper {
}
}
+ test("doris - get columns") {
+ case class Column(tableSchema: String, tableName: String, columnName: String)
+
+ def buildColumn(resultSet: ResultSet): Column = {
+ val schema = resultSet.getString(TABLE_SCHEMA)
+ val tableName = resultSet.getString(TABLE_NAME)
+ val columnName = resultSet.getString(COLUMN_NAME)
+ val column = Column(schema, tableName, columnName)
+ column
+ }
+
+ withJdbcStatement() { statement =>
+ val metadata = statement.getConnection.getMetaData
+ statement.execute("create database if not exists db1")
+ statement.execute("create table if not exists db1.test1" +
+ "(id bigint, str1 string, str2 string, age int)" +
+ "ENGINE=OLAP DISTRIBUTED BY HASH(`id`) BUCKETS 32 " +
+ "PROPERTIES ('replication_num' = '1')")
+ statement.execute("create table if not exists db1.test2" +
+ "(id bigint, str1 string, str2 string, age int)" +
+ "ENGINE=OLAP DISTRIBUTED BY HASH(`id`) BUCKETS 32 " +
+ "PROPERTIES ('replication_num' = '1')")
+
+ statement.execute("create database if not exists db2")
+
+ statement.execute("create table if not exists db2.test1" +
+ "(id bigint, str1 string, str2 string, age int)" +
+ "ENGINE=OLAP DISTRIBUTED BY HASH(`id`) BUCKETS 32 " +
+ "PROPERTIES ('replication_num' = '1')")
+
+ val resultBuffer = ArrayBuffer[Column]()
+ val resultSet1 = metadata.getColumns(null, "db1", null, null)
+ while (resultSet1.next()) {
+ val column = buildColumn(resultSet1)
+ resultBuffer += column
+ }
+
+ assert(resultBuffer.contains(Column("db1", "test1", "id")))
+ assert(resultBuffer.contains(Column("db1", "test1", "str1")))
+ assert(resultBuffer.contains(Column("db1", "test1", "str2")))
+ assert(resultBuffer.contains(Column("db1", "test1", "age")))
+
+ assert(resultBuffer.contains(Column("db1", "test2", "id")))
+ assert(resultBuffer.contains(Column("db1", "test2", "str1")))
+ assert(resultBuffer.contains(Column("db1", "test2", "str2")))
+ assert(resultBuffer.contains(Column("db1", "test2", "age")))
+
+ resultBuffer.clear()
+
+ val resultSet2 = metadata.getColumns(null, null, "test1", null)
+ while (resultSet2.next()) {
+ val column = buildColumn(resultSet2)
+ resultBuffer += column
+ }
+
+ assert(resultBuffer.contains(Column("db1", "test1", "id")))
+ assert(resultBuffer.contains(Column("db1", "test1", "str1")))
+ assert(resultBuffer.contains(Column("db1", "test1", "str2")))
+ assert(resultBuffer.contains(Column("db1", "test1", "age")))
+
+ assert(resultBuffer.contains(Column("db2", "test1", "id")))
+ assert(resultBuffer.contains(Column("db2", "test1", "str1")))
+ assert(resultBuffer.contains(Column("db2", "test1", "str2")))
+ assert(resultBuffer.contains(Column("db2", "test1", "age")))
+
+ resultBuffer.clear()
+
+ val resultSet3 = metadata.getColumns(null, null, null, "age")
+ while (resultSet3.next()) {
+ val column = buildColumn(resultSet3)
+ resultBuffer += column
+ }
+
+ assert(resultBuffer.contains(Column("db1", "test1", "age")))
+ assert(resultBuffer.contains(Column("db1", "test2", "age")))
+ assert(resultBuffer.contains(Column("db2", "test1", "age")))
+
+ resultBuffer.clear()
+
+ val resultSet4 = metadata.getColumns(null, "d%1", "t%1", "str%")
+ while (resultSet4.next()) {
+ val column = buildColumn(resultSet4)
+ resultBuffer += column
+ }
+
+ assert(resultBuffer.contains(Column("db1", "test1", "str1")))
+ assert(resultBuffer.contains(Column("db1", "test1", "str2")))
+
+ resultBuffer.clear()
+
+ val resultSet5 = metadata.getColumns(null, "d%1", "t%1", "fake")
+ assert(!resultSet5.next())
+
+ statement.execute("drop table db1.test1")
+ statement.execute("drop table db1.test2")
+ statement.execute("drop database db1")
+ statement.execute("drop table db2.test1")
+ statement.execute("drop database db2")
+ }
+ }
+
override protected def jdbcUrl: String = getJdbcUrl
}