(kyuubi) branch master updated: [KYUUBI #7585] Queries with 'TRANSFORM ... USING ' must be rejected

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

pan3793 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new 25d43f16d4 [KYUUBI #7585] Queries with 'TRANSFORM ... USING <shell-script>' must be rejected
25d43f16d4 is described below

commit 25d43f16d4a9187478e43590623de359b7d87a1d
Author: attilapiros <pi...@gmail.com>
AuthorDate: Tue Jul 28 20:58:58 2026 +0800

    [KYUUBI #7585] Queries with 'TRANSFORM ... USING <shell-script>' must be rejected
    
    ### Why are the changes needed?
    
    This PR adds a configurable plan-node deny list to kyuubi-spark-authz. The first (and currently only) entry blocks `ScriptTransformation` — the Spark SQL `TRANSFORM ... USING <script>` feature — which would otherwise let any SQL user execute arbitrary shell commands on Spark executors, bypassing all Ranger data-access controls.
    
    ### How was this patch tested?
    
    Added a new unit test suite `NodeDenyListCheckerSuite`:
    
    ```
    Discovery starting.
    Discovery completed in 134 milliseconds.
    Run starting. Expected test count is: 3
    NodeDenyListCheckerSuite:
    - plain SELECT is allowed
    - TRANSFORM ... USING is rejected
    - TRANSFORM ... USING inside a subquery is rejected
    Run completed in 4 seconds, 858 milliseconds.
    Total number of tests run: 3
    Suites: completed 2, aborted 0
    Tests: succeeded 3, failed 0, canceled 0, ignored 0, pending 0
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes.
    
    Assisted-by: Claude:claude-sonnet
    
    Closes #7586 from attilapiros/master.
    
    Closes #7585
    
    1929982a2 [attilapiros] [KYUUBI #7585] Queries with 'TRANSFORM ... USING <shell-script>' must be rejected
    
    Authored-by: attilapiros <pi...@gmail.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../src/main/resources/denied_plan_node_spec.json  |  4 ++
 .../spark/authz/ranger/RangerSparkExtension.scala  |  3 +-
 .../authz/rule/config/NodeDenyListChecker.scala    | 39 ++++++++++++
 .../plugin/spark/authz/serde/CommandSpec.scala     |  8 +++
 .../kyuubi/plugin/spark/authz/serde/package.scala  |  6 ++
 .../authz/rule/NodeDenyListCheckerSuite.scala      | 71 ++++++++++++++++++++++
 6 files changed, 130 insertions(+), 1 deletion(-)

diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/denied_plan_node_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/denied_plan_node_spec.json
new file mode 100644
index 0000000000..54b83aacdc
--- /dev/null
+++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/denied_plan_node_spec.json
@@ -0,0 +1,4 @@
+[ {
+  "classname" : "org.apache.spark.sql.catalyst.plans.logical.ScriptTransformation",
+  "message"   : "TRANSFORM ... USING scripts are not allowed"
+} ]
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
index 93f9a37dfc..bbe1b9c2c6 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.SparkSessionExtensions
 
 import org.apache.kyuubi.plugin.spark.authz.rule.{RuleEliminateMarker, RuleEliminatePermanentViewMarker, RuleEliminateTypeOf}
 import org.apache.kyuubi.plugin.spark.authz.rule.Authorization.RuleClearAuthzTag
-import org.apache.kyuubi.plugin.spark.authz.rule.config.AuthzConfigurationChecker
+import org.apache.kyuubi.plugin.spark.authz.rule.config.{AuthzConfigurationChecker, NodeDenyListChecker}
 import org.apache.kyuubi.plugin.spark.authz.rule.datamasking.{RuleApplyDataMaskingStage0, RuleApplyDataMaskingStage1}
 import org.apache.kyuubi.plugin.spark.authz.rule.expression.RuleApplyTypeOfMarker
 import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.RuleApplyPermanentViewMarker
@@ -54,6 +54,7 @@ class RangerSparkExtension extends (SparkSessionExtensions => Unit) {
     // before the optimizer runs. The tag is only set by optimizer rules, so a single
     // post-hoc pass is sufficient - no need for fixed-point iteration.
     v1.injectPostHocResolutionRule(_ => RuleClearAuthzTag)
+    v1.injectCheckRule(NodeDenyListChecker)
     v1.injectResolutionRule(_ => RuleReplaceShowObjectCommands)
     v1.injectResolutionRule(_ => RuleApplyPermanentViewMarker)
     v1.injectResolutionRule(_ => RuleApplyTypeOfMarker)
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/NodeDenyListChecker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/NodeDenyListChecker.scala
new file mode 100644
index 0000000000..9c7d90bdff
--- /dev/null
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/NodeDenyListChecker.scala
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.plugin.spark.authz.rule.config
+
+import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+
+import org.apache.kyuubi.plugin.spark.authz.AccessControlException
+import org.apache.kyuubi.plugin.spark.authz.serde.DENIED_PLAN_NODE_SPECS
+
+/**
+ * Rejects queries whose plan contains any node listed in denied_plan_node_spec.json.
+ * Matching is by exact classname; add entries to the spec file to deny additional node types
+ * without changing code.
+ */
+case class NodeDenyListChecker(spark: SparkSession) extends (LogicalPlan => Unit) {
+  override def apply(plan: LogicalPlan): Unit = {
+    plan.foreach { node =>
+      DENIED_PLAN_NODE_SPECS.get(node.getClass.getName).foreach { spec =>
+        throw new AccessControlException(spec.message)
+      }
+    }
+  }
+}
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
index c4fd721ca9..2e6e1b6c1f 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala
@@ -99,6 +99,14 @@ case class TableCommandSpec(
   }
 }
 
+/**
+ * A specification for a plan node that should be unconditionally denied.
+ *
+ * @param classname the fully-qualified plan node classname
+ * @param message   the error message surfaced in the AccessControlException
+ */
+case class DeniedPlanNodeSpec(classname: String, message: String)
+
 case class ScanSpec(
     classname: String,
     scanDescs: Seq[ScanDesc],
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
index 5cc1c8c19a..f9a8c38ff5 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala
@@ -44,6 +44,12 @@ package object serde {
   def loadExtractorsToMap[T <: Extractor](implicit ct: ClassTag[T]): Map[String, T] =
     loadFromServiceLoader[T]()(ct).map { e: T => (e.key, e) }.toMap
 
+  final lazy val DENIED_PLAN_NODE_SPECS: Map[String, DeniedPlanNodeSpec] = {
+    val is = getClass.getClassLoader.getResourceAsStream("denied_plan_node_spec.json")
+    mapper.readValue(is, new TypeReference[Array[DeniedPlanNodeSpec]] {})
+      .map(e => (e.classname, e)).toMap
+  }
+
   final lazy val DB_COMMAND_SPECS: Map[String, DatabaseCommandSpec] = {
     val is = getClass.getClassLoader.getResourceAsStream("database_command_spec.json")
     mapper.readValue(is, new TypeReference[Array[DatabaseCommandSpec]] {})
diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/NodeDenyListCheckerSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/NodeDenyListCheckerSuite.scala
new file mode 100644
index 0000000000..bc075e3a46
--- /dev/null
+++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/NodeDenyListCheckerSuite.scala
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kyuubi.plugin.spark.authz.rule
+
+import org.scalatest.BeforeAndAfterAll
+
+import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.plugin.spark.authz.{AccessControlException, SparkSessionProvider}
+import org.apache.kyuubi.plugin.spark.authz.rule.config.NodeDenyListChecker
+
+class NodeDenyListCheckerSuite extends KyuubiFunSuite with SparkSessionProvider
+  with BeforeAndAfterAll {
+
+  // TRANSFORM ... USING requires Hive catalog support
+  override protected val catalogImpl: String = "hive"
+
+  override def afterAll(): Unit = {
+    spark.stop()
+    super.afterAll()
+  }
+
+  test("plain SELECT is allowed") {
+    sql("CREATE TABLE IF NOT EXISTS ndl_plain (k INT, v STRING) USING hive")
+    try {
+      val plan = sql("SELECT k, v FROM ndl_plain").queryExecution.analyzed
+      NodeDenyListChecker(spark).apply(plan)
+    } finally {
+      sql("DROP TABLE IF EXISTS ndl_plain PURGE")
+    }
+  }
+
+  test("TRANSFORM ... USING is rejected") {
+    sql("CREATE TABLE IF NOT EXISTS ndl_transform (k INT, v STRING) USING hive")
+    try {
+      val plan =
+        sql("SELECT TRANSFORM(k, v) USING 'cat' AS (k STRING, v STRING) FROM ndl_transform")
+          .queryExecution.analyzed
+      intercept[AccessControlException](NodeDenyListChecker(spark).apply(plan))
+    } finally {
+      sql("DROP TABLE IF EXISTS ndl_transform PURGE")
+    }
+  }
+
+  test("TRANSFORM ... USING inside a subquery is rejected") {
+    sql("CREATE TABLE IF NOT EXISTS ndl_sub (k INT, v STRING) USING hive")
+    try {
+      val plan = sql(
+        """SELECT * FROM (
+          |  SELECT TRANSFORM(k, v) USING 'cat' AS (k STRING, v STRING) FROM ndl_sub
+          |) t""".stripMargin).queryExecution.analyzed
+      intercept[AccessControlException](NodeDenyListChecker(spark).apply(plan))
+    } finally {
+      sql("DROP TABLE IF EXISTS ndl_sub PURGE")
+    }
+  }
+}