You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2023/03/01 02:05:31 UTC
[kyuubi] branch branch-1.7 updated: [KYUUBI #4429] Add support for `OneRelation`
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.7 by this push:
new b95a24402 [KYUUBI #4429] Add support for `OneRelation`
b95a24402 is described below
commit b95a244024f61c02e859f5754cb80283fad8babe
Author: odone <od...@gmail.com>
AuthorDate: Wed Mar 1 10:05:14 2023 +0800
[KYUUBI #4429] Add support for `OneRelation`
### _Why are the changes needed?_
The `LogicalPlan` of SQL with `OneRowRelation`:
```
select 1,2,(select count(distinct" +
" ifnull(get_json_object(a, '$.b.imei'), get_json_object(a, '$.b.android_id'))) from t2)
```
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4429 from iodone/kyuubi-count.
Closes #4429
c3759239 [odone] add OneRowRelation
Authored-by: odone <od...@gmail.com>
Signed-off-by: ulyssesyou <ul...@apache.org>
(cherry picked from commit abc0a1d9f0973a2d9b5bdcbf92a814d9b2d6b3b3)
Signed-off-by: ulyssesyou <ul...@apache.org>
---
.../lineage/helper/SparkSQLLineageParseHelper.scala | 12 ++++++++++++
.../helper/SparkSQLLineageParserHelperSuite.scala | 17 +++++++++++++++++
2 files changed, 29 insertions(+)
diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala
index a58653113..cfd155527 100644
--- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala
+++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala
@@ -402,6 +402,18 @@ trait LineageParser {
case p: LocalRelation =>
joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(LOCAL_TABLE_IDENTIFIER))
+ case _: OneRowRelation =>
+ parentColumnsLineage.map {
+ case (k, attrs) =>
+ k -> AttributeSet(attrs.map {
+ case attr
+ if attr.qualifier.nonEmpty && attr.qualifier.last.equalsIgnoreCase(
+ SUBQUERY_COLUMN_IDENTIFIER) =>
+ attr.withQualifier(attr.qualifier.init)
+ case attr => attr
+ })
+ }
+
case p: InMemoryRelation =>
// get logical plan from cachedPlan
val cachedTableLogical = findSparkPlanLogicalLink(Seq(p.cacheBuilder.cachedPlan))
diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
index 050f3ddc9..4e1edc5c1 100644
--- a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
+++ b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala
@@ -1213,6 +1213,23 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite
}
}
+ test("test count()") {
+ withTable("t1", "t2") { _ =>
+ spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive")
+ spark.sql("CREATE TABLE t2 (a string, b string, c string) USING hive")
+ val ret0 = exectractLineage("insert into t1 select 1,2,(select count(distinct" +
+ " ifnull(get_json_object(a, '$.b.imei'), get_json_object(a, '$.b.android_id'))) from t2)")
+
+ assert(ret0 == Lineage(
+ List("default.t2"),
+ List("default.t1"),
+ List(
+ ("default.t1.a", Set()),
+ ("default.t1.b", Set()),
+ ("default.t1.c", Set("default.t2.a")))))
+ }
+ }
+
private def exectractLineageWithoutExecuting(sql: String): Lineage = {
val parsed = spark.sessionState.sqlParser.parsePlan(sql)
val analyzed = spark.sessionState.analyzer.execute(parsed)