You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/05/23 07:26:05 UTC
[kyuubi] branch master updated: [KYUUBI #4873] [AUTHZ] Refactor Authz reflection with kyuubi-util's DynMethods
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 320178bb6 [KYUUBI #4873] [AUTHZ] Refactor Authz reflection with kyuubi-util's DynMethods
320178bb6 is described below
commit 320178bb685081d99ed1474504a318f34ee1f19e
Author: liangbowen <li...@gf.com.cn>
AuthorDate: Tue May 23 15:25:54 2023 +0800
[KYUUBI #4873] [AUTHZ] Refactor Authz reflection with kyuubi-util's DynMethods
### _Why are the changes needed?_
- add reflection utils in kyuubi-util-scala, using kyuubi-util's DynMethods
- continue to provided simplified reflection calling in scala
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4873 from bowenliang123/authz-reflect.
Closes #4873
d0a508400 [liangbowen] import
95d4760ad [Cheng Pan] Update kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/reflect/ReflectUtils.scala
83e70f09b [liangbowen] authz reflect
Lead-authored-by: liangbowen <li...@gf.com.cn>
Co-authored-by: Cheng Pan <pa...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../plugin/spark/authz/PrivilegesBuilder.scala | 1 +
.../plugin/spark/authz/ranger/AccessRequest.scala | 2 +-
.../ranger/RuleReplaceShowObjectCommands.scala | 3 +-
.../plugin/spark/authz/serde/Descriptor.scala | 2 +-
.../spark/authz/serde/catalogExtractors.scala | 2 +-
.../spark/authz/serde/databaseExtractors.scala | 1 +
.../plugin/spark/authz/serde/tableExtractors.scala | 1 +
.../plugin/spark/authz/util/AuthZUtils.scala | 61 +---------------------
.../spark/authz/util/RangerConfigProvider.scala | 13 +++--
.../authz/ranger/RangerSparkExtensionSuite.scala | 1 +
.../apache/kyuubi/util/reflect/ReflectUtils.scala | 50 ++++++++++++++++++
11 files changed, 69 insertions(+), 68 deletions(-)
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
index b8220ea27..eef89deea 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala
@@ -28,6 +28,7 @@ import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType
import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._
import org.apache.kyuubi.plugin.spark.authz.serde._
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
object PrivilegesBuilder {
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala
index 39e172daf..7d4999fde 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala
@@ -27,7 +27,7 @@ import org.apache.ranger.plugin.policyengine.{RangerAccessRequestImpl, RangerPol
import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType
import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType._
-import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
case class AccessRequest private (accessType: AccessType) extends RangerAccessRequestImpl
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala
index 08d2b4fd0..4f72ecfee 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.execution.command.{RunnableCommand, ShowColumnsComma
import org.apache.kyuubi.plugin.spark.authz.{ObjectType, OperationType}
import org.apache.kyuubi.plugin.spark.authz.util.{AuthZUtils, ObjectFilterPlaceHolder, WithInternalChildren}
+import org.apache.kyuubi.util.reflect.ReflectUtils._
class RuleReplaceShowObjectCommands extends Rule[LogicalPlan] {
override def apply(plan: LogicalPlan): LogicalPlan = plan match {
@@ -48,7 +49,7 @@ class RuleReplaceShowObjectCommands extends Rule[LogicalPlan] {
case class FilteredShowTablesCommand(delegated: RunnableCommand)
extends FilteredShowObjectCommand(delegated) {
- var isExtended: Boolean = AuthZUtils.getFieldVal(delegated, "isExtended").asInstanceOf[Boolean]
+ private val isExtended = getFieldVal[Boolean](delegated, "isExtended")
override protected def isAllowed(r: Row, ugi: UserGroupInformation): Boolean = {
val database = r.getString(0)
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
index d8c866b88..b72f3296b 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala
@@ -34,7 +34,7 @@ import org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors
import org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors
import org.apache.kyuubi.plugin.spark.authz.serde.TableType.TableType
import org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors
-import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
/**
* A database object(such as database, table, function) descriptor describes its name and getter
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
index 0b7d71223..f87943943 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala
@@ -17,7 +17,7 @@
package org.apache.kyuubi.plugin.spark.authz.serde
-import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
trait CatalogExtractor extends (AnyRef => Option[String]) with Extractor
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
index 4e9270e78..d14ba7805 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala
@@ -18,6 +18,7 @@
package org.apache.kyuubi.plugin.spark.authz.serde
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
trait DatabaseExtractor extends (AnyRef => Database) with Extractor
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
index c848381d4..5af619bcf 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala
@@ -27,6 +27,7 @@ import org.apache.spark.sql.catalyst.catalog.CatalogTable
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
/**
* A trait for extracting database and table as string tuple
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala
index d1571a9bb..e13968e2e 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala
@@ -23,8 +23,6 @@ import java.security.interfaces.ECPublicKey
import java.security.spec.X509EncodedKeySpec
import java.util.Base64
-import scala.util.{Failure, Success, Try}
-
import org.apache.commons.lang3.StringUtils
import org.apache.hadoop.security.UserGroupInformation
import org.apache.ranger.plugin.service.RangerBasePlugin
@@ -34,67 +32,10 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View}
import org.apache.kyuubi.plugin.spark.authz.AccessControlException
import org.apache.kyuubi.plugin.spark.authz.util.ReservedKeys._
import org.apache.kyuubi.util.SemanticVersion
+import org.apache.kyuubi.util.reflect.ReflectUtils._
private[authz] object AuthZUtils {
- /**
- * fixme error handling need improve here
- */
- def getFieldVal[T](o: Any, name: String): T = {
- Try {
- val field = o.getClass.getDeclaredField(name)
- field.setAccessible(true)
- field.get(o)
- } match {
- case Success(value) => value.asInstanceOf[T]
- case Failure(e) =>
- val candidates = o.getClass.getDeclaredFields.map(_.getName).mkString("[", ",", "]")
- throw new RuntimeException(s"$name not in ${o.getClass} $candidates", e)
- }
- }
-
- def getFieldValOpt[T](o: Any, name: String): Option[T] = Try(getFieldVal[T](o, name)).toOption
-
- def invoke(
- obj: AnyRef,
- methodName: String,
- args: (Class[_], AnyRef)*): AnyRef = {
- try {
- val (types, values) = args.unzip
- val method = obj.getClass.getMethod(methodName, types: _*)
- method.setAccessible(true)
- method.invoke(obj, values: _*)
- } catch {
- case e: NoSuchMethodException =>
- val candidates = obj.getClass.getMethods.map(_.getName).mkString("[", ",", "]")
- throw new RuntimeException(s"$methodName not in ${obj.getClass} $candidates", e)
- }
- }
-
- def invokeAs[T](
- obj: AnyRef,
- methodName: String,
- args: (Class[_], AnyRef)*): T = {
- invoke(obj, methodName, args: _*).asInstanceOf[T]
- }
-
- def invokeStatic(
- obj: Class[_],
- methodName: String,
- args: (Class[_], AnyRef)*): AnyRef = {
- val (types, values) = args.unzip
- val method = obj.getMethod(methodName, types: _*)
- method.setAccessible(true)
- method.invoke(obj, values: _*)
- }
-
- def invokeStaticAs[T](
- obj: Class[_],
- methodName: String,
- args: (Class[_], AnyRef)*): T = {
- invokeStatic(obj, methodName, args: _*).asInstanceOf[T]
- }
-
/**
* Get the active session user
* @param spark spark context instance
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala
index 83fe048e6..cb3a2371b 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala
@@ -20,6 +20,7 @@ package org.apache.kyuubi.plugin.spark.authz.util
import org.apache.hadoop.conf.Configuration
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.DynMethods
trait RangerConfigProvider {
@@ -36,12 +37,16 @@ trait RangerConfigProvider {
def getRangerConf: Configuration = {
if (isRanger21orGreater) {
// for Ranger 2.1+
- invokeAs[Configuration](this, "getConfig")
+ DynMethods.builder("getConfig")
+ .impl("org.apache.ranger.plugin.service.RangerBasePlugin")
+ .build()
+ .invoke[Configuration](this)
} else {
// for Ranger 2.0 and below
- invokeStaticAs[Configuration](
- Class.forName("org.apache.ranger.authorization.hadoop.config.RangerConfiguration"),
- "getInstance")
+ DynMethods.builder("getInstance")
+ .impl("org.apache.ranger.authorization.hadoop.config.RangerConfiguration")
+ .buildStatic()
+ .invoke[Configuration]()
}
}
}
diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
index d044ad46c..89b81ccee 100644
--- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
@@ -35,6 +35,7 @@ import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._
import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._
import org.apache.kyuubi.plugin.spark.authz.ranger.RuleAuthorization.KYUUBI_AUTHZ_TAG
import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._
+import org.apache.kyuubi.util.reflect.ReflectUtils._
abstract class RangerSparkExtensionSuite extends AnyFunSuite
with SparkSessionProvider with BeforeAndAfterAll {
diff --git a/kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/reflect/ReflectUtils.scala b/kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/reflect/ReflectUtils.scala
new file mode 100644
index 000000000..843b8489a
--- /dev/null
+++ b/kyuubi-util-scala/src/main/scala/org/apache/kyuubi/util/reflect/ReflectUtils.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.util.reflect
+
+import scala.util.{Failure, Success, Try}
+
+object ReflectUtils {
+
+ def getFieldVal[T](target: Any, fieldName: String): T =
+ Try {
+ DynFields.builder().hiddenImpl(target.getClass, fieldName).build[T]().get(target)
+ } match {
+ case Success(value) => value
+ case Failure(e) =>
+ val candidates = target.getClass.getDeclaredFields.map(_.getName).mkString("[", ",", "]")
+ throw new RuntimeException(s"$fieldName not in ${target.getClass} $candidates", e)
+ }
+
+ def getFieldValOpt[T](target: Any, name: String): Option[T] =
+ Try(getFieldVal[T](target, name)).toOption
+
+ def invoke(target: AnyRef, methodName: String, args: (Class[_], AnyRef)*): AnyRef =
+ try {
+ val (types, values) = args.unzip
+ DynMethods.builder(methodName).hiddenImpl(target.getClass, types: _*).build()
+ .invoke(target, values: _*)
+ } catch {
+ case e: NoSuchMethodException =>
+ val candidates = target.getClass.getMethods.map(_.getName).mkString("[", ",", "]")
+ throw new RuntimeException(s"$methodName not in ${target.getClass} $candidates", e)
+ }
+
+ def invokeAs[T](target: AnyRef, methodName: String, args: (Class[_], AnyRef)*): T =
+ invoke(target, methodName, args: _*).asInstanceOf[T]
+}