You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/01/04 10:05:03 UTC
[kyuubi] branch branch-1.6 updated: [KYUUBI #4076] [AUTHZ] Modified query plan should correctly report stats
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.6 by this push:
new 8c93a4871 [KYUUBI #4076] [AUTHZ] Modified query plan should correctly report stats
8c93a4871 is described below
commit 8c93a4871ea955e3e22f27ac0ed4eede73de1d7f
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Wed Jan 4 18:04:33 2023 +0800
[KYUUBI #4076] [AUTHZ] Modified query plan should correctly report stats
### _Why are the changes needed?_
It was reported in https://github.com/apache/kyuubi/discussions/4063
When run `SHOW DATABASES;` on Databricks Runtime, the error occurs
```
com.databricks.backend.common.rpc.SparkDriverExceptions$SQLExecutionException: java.lang.UnsupportedOperationException
at org.apache.spark.sql.catalyst.plans.logical.LeafNode.computeStats(LogicalPlan.scala:183)
at org.apache.spark.sql.catalyst.plans.logical.LeafNode.computeStats$(LogicalPlan.scala:183)
at org.apache.kyuubi.plugin.spark.authz.util.ObjectFilterPlaceHolder.computeStats(ObjectFilterPlaceHolder.scala:23)
at org.apache.spark.sql.catalyst.plans.QueryPlanStats.$anonfun$stats$1(QueryPlanStats.scala:39)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.catalyst.plans.QueryPlanStats.stats(QueryPlanStats.scala:38)
at org.apache.spark.sql.catalyst.plans.QueryPlanStats.stats$(QueryPlanStats.scala:38)
at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.stats(LogicalPlan.scala:31)
```
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4076 from pan3793/stats.
Closes #4076
8238f317 [Cheng Pan] ut
a3c5f7d3 [Cheng Pan] [AUTHZ] Modified query plan should correctly report stats
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
(cherry picked from commit 71e46bd3168c4ce530d96d38ebc7e9209f03c498)
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../plugin/spark/authz/util/ObjectFilterPlaceHolder.scala | 3 ++-
.../spark/authz/ranger/RangerSparkExtensionSuite.scala | 14 ++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala
index d0f7f3055..a5d1c0d3b 100644
--- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala
@@ -18,8 +18,9 @@
package org.apache.kyuubi.plugin.spark.authz.util
import org.apache.spark.sql.catalyst.expressions.Attribute
-import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan}
+import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics}
case class ObjectFilterPlaceHolder(child: LogicalPlan) extends LeafNode {
override def output: Seq[Attribute] = child.output
+ override def computeStats(): Statistics = child.stats
}
diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
index 54c7491ba..1ea2575bb 100644
--- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
+++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala
@@ -501,4 +501,18 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
sql(s"DROP DATABASE IF EXISTS $db")
})
}
+
+ test("modified query plan should correctly report stats") {
+ val db = "stats_test"
+ val table = "stats"
+ withCleanTmpResources(
+ Seq(
+ (s"$db.$table", "table"),
+ (s"$db", "database"))) {
+ doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db"))
+ doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.$table (key int) USING $format"))
+ sql("SHOW DATABASES").queryExecution.optimizedPlan.stats
+ sql(s"SHOW TABLES IN $db").queryExecution.optimizedPlan.stats
+ }
+ }
}