You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/10/17 09:33:16 UTC
[kyuubi] branch master updated: [KYUUBI #5359] [AUTHZ] Support Create Table Commands for Hudi
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new f6ccc4d80 [KYUUBI #5359] [AUTHZ] Support Create Table Commands for Hudi
f6ccc4d80 is described below
commit f6ccc4d80fc925d51540595c3b18556c220dd70e
Author: Angerszhuuuu <an...@gmail.com>
AuthorDate: Tue Oct 17 17:33:07 2023 +0800
[KYUUBI #5359] [AUTHZ] Support Create Table Commands for Hudi
### _Why are the changes needed?_
To close #5359. Kyuubi authz support hudi create table commands
- [CreateHoodieTableCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala): https://hudi.apache.org/docs/sql_ddl#create-table
- [CreateHoodieTableAsSelectCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala): https://hudi.apache.org/docs/sql_ddl#create-table-as-select-ctas
- [CreateHoodieTableLikeCommand](https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableLikeCommand.scala): https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableLikeCommand.scala
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request
### _Was this patch authored or co-authored using generative AI tooling?_
No
Closes #5439 from AngersZhuuuu/KYUUBI-5359.
Closes #5359
d07010913 [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
f0bc79a77 [Angerszhuuuu] [KYUUBI #5284] Support Hudi Creeate Table Command in Authz
Authored-by: Angerszhuuuu <an...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../src/main/resources/table_command_spec.json | 54 +++++++++++++++
.../plugin/spark/authz/gen/HudiCommands.scala | 33 +++++++++-
.../HudiCatalogRangerSparkExtensionSuite.scala | 77 ++++++++++++++++++++++
3 files changed, 163 insertions(+), 1 deletion(-)
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
index 513259e13..913fef016 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
+++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json
@@ -1513,6 +1513,60 @@
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ]
+}, {
+ "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand",
+ "tableDescs" : [ {
+ "fieldName" : "table",
+ "fieldExtractor" : "CatalogTableTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "CREATETABLE_AS_SELECT",
+ "queryDescs" : [ {
+ "fieldName" : "query",
+ "fieldExtractor" : "LogicalPlanQueryExtractor"
+ } ]
+}, {
+ "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand",
+ "tableDescs" : [ {
+ "fieldName" : "table",
+ "fieldExtractor" : "CatalogTableTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : false
+ } ],
+ "opType" : "CREATETABLE",
+ "queryDescs" : [ ]
+}, {
+ "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand",
+ "tableDescs" : [ {
+ "fieldName" : "targetTable",
+ "fieldExtractor" : "TableIdentifierTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : false,
+ "setCurrentDatabaseIfMissing" : true
+ }, {
+ "fieldName" : "sourceTable",
+ "fieldExtractor" : "TableIdentifierTableExtractor",
+ "columnDesc" : null,
+ "actionTypeDesc" : null,
+ "tableTypeDesc" : null,
+ "catalogDesc" : null,
+ "isInput" : true,
+ "setCurrentDatabaseIfMissing" : true
+ } ],
+ "opType" : "CREATETABLE",
+ "queryDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand",
"tableDescs" : [ {
diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
index 6e3237d2a..c4488edbf 100644
--- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala
@@ -72,11 +72,42 @@ object HudiCommands {
TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES)
}
+ val CreateHoodieTableCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand"
+ val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
+ TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE)
+ }
+
+ val CreateHoodieTableAsSelectCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand"
+ CreateHoodieTableCommand.copy(
+ classname = cmd,
+ opType = CREATETABLE_AS_SELECT,
+ queryDescs = Seq(QueryDesc("query")))
+ }
+
+ val CreateHoodieTableLikeCommand = {
+ val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand"
+ val tableDesc1 = TableDesc(
+ "targetTable",
+ classOf[TableIdentifierTableExtractor],
+ setCurrentDatabaseIfMissing = true)
+ val tableDesc2 = TableDesc(
+ "sourceTable",
+ classOf[TableIdentifierTableExtractor],
+ isInput = true,
+ setCurrentDatabaseIfMissing = true)
+ TableCommandSpec(cmd, Seq(tableDesc1, tableDesc2), CREATETABLE)
+ }
+
val data: Array[TableCommandSpec] = Array(
AlterHoodieTableAddColumnsCommand,
AlterHoodieTableChangeColumnCommand,
AlterHoodieTableDropPartitionCommand,
AlterHoodieTableRenameCommand,
AlterTableCommand,
- Spark31AlterTableCommand)
+ Spark31AlterTableCommand,
+ CreateHoodieTableCommand,
+ CreateHoodieTableAsSelectCommand,
+ CreateHoodieTableLikeCommand)
}
diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
index 8fcae6cf9..ac1e357a2 100644
--- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala
@@ -129,4 +129,81 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
s"does not have [alter] privilege on [$namespace1/$table1]")
}
}
+
+ test("CreateHoodieTableCommand") {
+ withCleanTmpResources(Seq((namespace1, "database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ interceptContains[AccessControlException](
+ doAs(
+ someone,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin)))(s"does not have [create] privilege on [$namespace1/$table1]")
+ }
+ }
+
+ test("CreateHoodieTableAsSelectCommand") {
+ withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+ interceptContains[AccessControlException](
+ doAs(
+ someone,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table2
+ |USING HUDI
+ |AS
+ |SELECT id FROM $namespace1.$table1
+ |""".stripMargin)))(s"does not have [select] privilege on [$namespace1/$table1/id]")
+ }
+ }
+
+ test("CreateHoodieTableLikeCommand") {
+ withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) {
+ doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
+ doAs(
+ admin,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string)
+ |USING HUDI
+ |OPTIONS (
+ | type = 'cow',
+ | primaryKey = 'id',
+ | 'hoodie.datasource.hive_sync.enable' = 'false'
+ |)
+ |PARTITIONED BY(city)
+ |""".stripMargin))
+ interceptContains[AccessControlException](
+ doAs(
+ someone,
+ sql(
+ s"""
+ |CREATE TABLE IF NOT EXISTS $namespace1.$table2
+ |LIKE $namespace1.$table1
+ |USING HUDI
+ |""".stripMargin)))(s"does not have [select] privilege on [$namespace1/$table1]")
+ }
+ }
}