You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2023/01/11 08:20:58 UTC
[kyuubi] branch master updated: [KYUUBI #4140] Support GetColumns for Trino Fe
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 984eac131 [KYUUBI #4140] Support GetColumns for Trino Fe
984eac131 is described below
commit 984eac131cbb15a74d82c982136e3f31f2001b5a
Author: Yikf <yi...@gmail.com>
AuthorDate: Wed Jan 11 16:20:37 2023 +0800
[KYUUBI #4140] Support GetColumns for Trino Fe
### _Why are the changes needed?_
Close https://github.com/apache/kyuubi/issues/4140
This pr as a subtask of the https://github.com/apache/kyuubi/issues/3901 to support GetColumns for Trino Fe.
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4141 from Yikf/get-columns.
Closes #4140
6efb99cb [Yikf] Support GetColumns for Trino Fe
Authored-by: Yikf <yi...@gmail.com>
Signed-off-by: ulysses-you <ul...@apache.org>
---
.../apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 | 15 +++
.../apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 | 12 +++
.../trino/api/KyuubiTrinoOperationTranslator.scala | 9 +-
.../sql/parser/trino/KyuubiTrinoFeAstBuilder.scala | 15 ++-
.../kyuubi/sql/plan/trino/TrinoFeOperations.scala | 8 ++
.../parser/trino/KyuubiTrinoFeParserSuite.scala | 111 ++++++++++++++++++++-
6 files changed, 167 insertions(+), 3 deletions(-)
diff --git a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4
index adda23f6e..0b9543a43 100644
--- a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4
+++ b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4
@@ -70,6 +70,7 @@ SQL_DATETIME_SUB: 'SQL_DATETIME_SUB';
SYSTEM_JDBC_CATALOGS: 'SYSTEM.JDBC.CATALOGS';
SYSTEM_JDBC_SCHEMAS: 'SYSTEM.JDBC.SCHEMAS';
SYSTEM_JDBC_TABLES: 'SYSTEM.JDBC.TABLES';
+SYSTEM_JDBC_COLUMNS: 'SYSTEM.JDBC.COLUMNS';
SYSTEM_JDBC_TABLE_TYPES: 'SYSTEM.JDBC.TABLE_TYPES';
SYSTEM_JDBC_TYPES: 'SYSTEM.JDBC.TYPES';
SELF_REFERENCING_COL_NAME: 'SELF_REFERENCING_COL_NAME';
@@ -82,6 +83,20 @@ TABLE_TYPE: 'TABLE_TYPE';
TYPE_CAT: 'TYPE_CAT';
TYPE_NAME: 'TYPE_NAME';
TYPE_SCHEM: 'TYPE_SCHEM';
+COLUMN_NAME: 'COLUMN_NAME';
+COLUMN_SIZE: 'COLUMN_SIZE';
+BUFFER_LENGTH: 'BUFFER_LENGTH';
+DECIMAL_DIGITS: 'DECIMAL_DIGITS';
+COLUMN_DEF: 'COLUMN_DEF';
+CHAR_OCTET_LENGTH: 'CHAR_OCTET_LENGTH';
+ORDINAL_POSITION: 'ORDINAL_POSITION';
+IS_NULLABLE: 'IS_NULLABLE';
+SCOPE_CATALOG: 'SCOPE_CATALOG';
+SCOPE_SCHEMA: 'SCOPE_SCHEMA';
+SCOPE_TABLE: 'SCOPE_TABLE';
+SOURCE_DATA_TYPE: 'SOURCE_DATA_TYPE';
+IS_AUTOINCREMENT: 'IS_AUTOINCREMENT';
+IS_GENERATEDCOLUMN: 'IS_GENERATEDCOLUMN';
fragment SEARCH_STRING_ESCAPE: '\'' '\\' '\'';
diff --git a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4
index 9c3ee3282..590c4378d 100644
--- a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4
+++ b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4
@@ -39,6 +39,14 @@ statement
FROM SYSTEM_JDBC_TABLES
(WHERE tableCatalogFilter? AND? tableSchemaFilter? AND? tableNameFilter? AND? tableTypeFilter?)?
ORDER BY TABLE_TYPE COMMA TABLE_CAT COMMA TABLE_SCHEM COMMA TABLE_NAME #getTables
+ | SELECT TABLE_CAT COMMA TABLE_SCHEM COMMA TABLE_NAME COMMA COLUMN_NAME COMMA DATA_TYPE COMMA
+ TYPE_NAME COMMA COLUMN_SIZE COMMA BUFFER_LENGTH COMMA DECIMAL_DIGITS COMMA NUM_PREC_RADIX COMMA
+ NULLABLE COMMA REMARKS COMMA COLUMN_DEF COMMA SQL_DATA_TYPE COMMA SQL_DATETIME_SUB COMMA
+ CHAR_OCTET_LENGTH COMMA ORDINAL_POSITION COMMA IS_NULLABLE COMMA
+ SCOPE_CATALOG COMMA SCOPE_SCHEMA COMMA SCOPE_TABLE COMMA
+ SOURCE_DATA_TYPE COMMA IS_AUTOINCREMENT COMMA IS_GENERATEDCOLUMN FROM SYSTEM_JDBC_COLUMNS
+ (WHERE tableCatalogFilter? AND? tableSchemaFilter? AND? tableNameFilter? AND? colNameFilter?)?
+ ORDER BY TABLE_CAT COMMA TABLE_SCHEM COMMA TABLE_NAME COMMA ORDINAL_POSITION #getColumns
| .*? #passThrough
;
@@ -56,6 +64,10 @@ tableNameFilter
: TABLE_NAME LIKE tableNamePattern=stringLit ESCAPE STRING_ESCAPE
;
+colNameFilter
+ : COLUMN_NAME LIKE colNamePattern=stringLit ESCAPE STRING_ESCAPE
+ ;
+
tableTypeFilter
: FALSE #tableTypesAlwaysFalse
| TABLE_TYPE IN LEFT_PAREN stringLit (COMMA stringLit)* RIGHT_PAREN #typesFilter
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiTrinoOperationTranslator.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiTrinoOperationTranslator.scala
index 5e8460883..6ec9fc1c8 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiTrinoOperationTranslator.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiTrinoOperationTranslator.scala
@@ -25,7 +25,7 @@ import org.apache.kyuubi.operation.OperationHandle
import org.apache.kyuubi.service.BackendService
import org.apache.kyuubi.sql.parser.trino.KyuubiTrinoFeParser
import org.apache.kyuubi.sql.plan.PassThroughNode
-import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
+import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetColumns, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
class KyuubiTrinoOperationTranslator(backendService: BackendService) {
lazy val parser = new KyuubiTrinoFeParser()
@@ -61,6 +61,13 @@ class KyuubiTrinoOperationTranslator(backendService: BackendService) {
tableTypes.asJava)
operationHandle.setHasResultSet(!emptyResult)
operationHandle
+ case GetColumns(catalogName, schemaPattern, tableNamePattern, colNamePattern) =>
+ backendService.getColumns(
+ sessionHandle,
+ catalogName,
+ schemaPattern,
+ tableNamePattern,
+ colNamePattern)
case PassThroughNode() =>
backendService.executeStatement(sessionHandle, statement, configs, runAsync, queryTimeout)
}
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/parser/trino/KyuubiTrinoFeAstBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/parser/trino/KyuubiTrinoFeAstBuilder.scala
index 83062fe4d..c5ae97199 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/parser/trino/KyuubiTrinoFeAstBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/parser/trino/KyuubiTrinoFeAstBuilder.scala
@@ -25,7 +25,7 @@ import org.apache.kyuubi.sql.KyuubiTrinoFeBaseParser._
import org.apache.kyuubi.sql.KyuubiTrinoFeBaseParserBaseVisitor
import org.apache.kyuubi.sql.parser.KyuubiParser.unescapeSQLString
import org.apache.kyuubi.sql.plan.{KyuubiTreeNode, PassThroughNode}
-import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
+import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetColumns, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
class KyuubiTrinoFeAstBuilder extends KyuubiTrinoFeBaseParserBaseVisitor[AnyRef] {
@@ -83,6 +83,15 @@ class KyuubiTrinoFeAstBuilder extends KyuubiTrinoFeBaseParserBaseVisitor[AnyRef]
GetTables(catalog, schemaPattern, tableNamePattern, tableTypes, emptyResult)
}
+ override def visitGetColumns(ctx: GetColumnsContext): KyuubiTreeNode = {
+ val catalog = visit(ctx.tableCatalogFilter()).asInstanceOf[String]
+ val schemaPattern = visit(ctx.tableSchemaFilter()).asInstanceOf[String]
+ val tableNamePattern = visit(ctx.tableNameFilter()).asInstanceOf[String]
+ val colNamePattern = visit(ctx.colNameFilter()).asInstanceOf[String]
+
+ GetColumns(catalog, schemaPattern, tableNamePattern, colNamePattern)
+ }
+
override def visitNullCatalog(ctx: NullCatalogContext): AnyRef = {
null
}
@@ -103,6 +112,10 @@ class KyuubiTrinoFeAstBuilder extends KyuubiTrinoFeBaseParserBaseVisitor[AnyRef]
unescapeSQLString(ctx.tableNamePattern.getText)
}
+ override def visitColNameFilter(ctx: ColNameFilterContext): String = {
+ unescapeSQLString(ctx.colNamePattern.getText)
+ }
+
override def visitTypesFilter(ctx: TypesFilterContext): List[String] = {
ctx.stringLit().asScala.map(v => unescapeSQLString(v.getText)).toList
}
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/plan/trino/TrinoFeOperations.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/plan/trino/TrinoFeOperations.scala
index 60a4abd7d..85e6f168b 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/plan/trino/TrinoFeOperations.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/sql/plan/trino/TrinoFeOperations.scala
@@ -47,3 +47,11 @@ case class GetTables(
emptyResult: Boolean = false) extends KyuubiTreeNode {
override def name(): String = "Get Tables"
}
+
+case class GetColumns(
+ catalogName: String,
+ schemaPattern: String,
+ tableNamePattern: String,
+ colNamePattern: String) extends KyuubiTreeNode {
+ override def name(): String = "Get Columns"
+}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/parser/trino/KyuubiTrinoFeParserSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/parser/trino/KyuubiTrinoFeParserSuite.scala
index 7848ad7b2..3f5cf70b5 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/parser/trino/KyuubiTrinoFeParserSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/parser/trino/KyuubiTrinoFeParserSuite.scala
@@ -20,7 +20,7 @@ package org.apache.kyuubi.parser.trino
import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.sql.parser.trino.KyuubiTrinoFeParser
import org.apache.kyuubi.sql.plan.{KyuubiTreeNode, PassThroughNode}
-import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
+import org.apache.kyuubi.sql.plan.trino.{GetCatalogs, GetColumns, GetSchemas, GetTables, GetTableTypes, GetTypeInfo}
class KyuubiTrinoFeParserSuite extends KyuubiFunSuite {
val parser = new KyuubiTrinoFeParser()
@@ -245,4 +245,113 @@ class KyuubiTrinoFeParserSuite extends KyuubiFunSuite {
tableName = "bb%",
types = List("ykf_type", "ykf2_type"))
}
+
+ test("Support GetColumns for Trino Fe") {
+ def check(
+ query: String,
+ catalog: String = null,
+ schema: String = null,
+ tableName: String = null,
+ colName: String = null): Unit = {
+ parse(query) match {
+ case GetColumns(catalogName, schemaPattern, tableNamePattern, colNamePattern) =>
+ assert(catalog == catalogName)
+ assert(schema == schemaPattern)
+ assert(tableName == tableNamePattern)
+ assert(colName == colNamePattern)
+ case _ => fail(s"Query $query parse failed. ")
+ }
+ }
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin)
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | WHERE TABLE_CAT IS NULL
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin)
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | WHERE TABLE_CAT = 'ykf_catalog'
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin,
+ catalog = "ykf_catalog")
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | WHERE TABLE_CAT = 'ykf_catalog' AND TABLE_NAME LIKE '%aa' ESCAPE '\'
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin,
+ catalog = "ykf_catalog",
+ tableName = "%aa")
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | WHERE TABLE_CAT = 'ykf_catalog' AND TABLE_NAME LIKE '%aa' ESCAPE '\'
+ | AND COLUMN_NAME LIKE '%bb' ESCAPE '\'
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin,
+ catalog = "ykf_catalog",
+ tableName = "%aa",
+ colName = "%bb")
+
+ check(
+ """
+ | SELECT TABLE_CAT, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, DATA_TYPE,
+ | TYPE_NAME, COLUMN_SIZE, BUFFER_LENGTH, DECIMAL_DIGITS, NUM_PREC_RADIX,
+ | NULLABLE, REMARKS, COLUMN_DEF, SQL_DATA_TYPE, SQL_DATETIME_SUB,
+ | CHAR_OCTET_LENGTH, ORDINAL_POSITION, IS_NULLABLE,
+ | SCOPE_CATALOG, SCOPE_SCHEMA, SCOPE_TABLE,
+ | SOURCE_DATA_TYPE, IS_AUTOINCREMENT, IS_GENERATEDCOLUMN
+ | FROM system.jdbc.columns
+ | WHERE TABLE_CAT = 'ykf_catalog'
+ | AND TABLE_SCHEM LIKE '%cc' ESCAPE '\'
+ | AND TABLE_NAME LIKE '%aa' ESCAPE '\'
+ | AND COLUMN_NAME LIKE '%bb' ESCAPE '\'
+ | ORDER BY TABLE_CAT, TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION
+ |""".stripMargin,
+ catalog = "ykf_catalog",
+ schema = "%cc",
+ tableName = "%aa",
+ colName = "%bb")
+ }
}