You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2023/06/04 00:30:31 UTC
[spark] branch branch-3.3 updated: [SPARK-43956][SQL][3.3] Fix the bug doesn't display column's sql for Percentile[Cont|Disc]
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 86d22f47d81 [SPARK-43956][SQL][3.3] Fix the bug doesn't display column's sql for Percentile[Cont|Disc]
86d22f47d81 is described below
commit 86d22f47d81031b10af7044b271cec6cfe75172e
Author: Jiaan Geng <be...@163.com>
AuthorDate: Sat Jun 3 17:30:16 2023 -0700
[SPARK-43956][SQL][3.3] Fix the bug doesn't display column's sql for Percentile[Cont|Disc]
### What changes were proposed in this pull request?
This PR used to backport https://github.com/apache/spark/pull/41436 to 3.3
### Why are the changes needed?
Fix the bug doesn't display column's sql for Percentile[Cont|Disc].
### Does this PR introduce _any_ user-facing change?
'Yes'.
Users could see the correct sql information.
### How was this patch tested?
Test cases updated.
Closes #41446 from beliefer/SPARK-43956_followup2.
Authored-by: Jiaan Geng <be...@163.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/sql/catalyst/expressions/aggregate/percentiles.scala | 4 ++--
.../resources/sql-tests/results/ansi/string-functions.sql.out | 2 +-
sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out | 2 +-
sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out | 2 +-
sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out | 2 +-
.../src/test/resources/sql-tests/results/cte-nonlegacy.sql.out | 2 +-
.../src/test/resources/sql-tests/results/grouping_set.sql.out | 2 +-
.../src/test/resources/sql-tests/results/inline-table.sql.out | 2 +-
.../sql-tests/results/postgreSQL/aggregates_part4.sql.out | 8 ++++----
.../src/test/resources/sql-tests/results/string-functions.sql.out | 2 +-
.../sql-tests/results/udf/postgreSQL/udf-aggregates_part4.sql.out | 8 ++++----
sql/core/src/test/resources/sql-tests/results/window.sql.out | 6 +++---
12 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
index bd62c0aef75..4dea92f6802 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/percentiles.scala
@@ -369,7 +369,7 @@ case class PercentileCont(left: Expression, right: Expression, reverse: Boolean
override def sql(isDistinct: Boolean): String = {
val distinct = if (isDistinct) "DISTINCT " else ""
val direction = if (reverse) " DESC" else ""
- s"$prettyName($distinct${right.sql}) WITHIN GROUP (ORDER BY v$direction)"
+ s"$prettyName($distinct${right.sql}) WITHIN GROUP (ORDER BY ${left.sql}$direction)"
}
override protected def withNewChildrenInternal(
newLeft: Expression, newRight: Expression): PercentileCont =
@@ -409,7 +409,7 @@ case class PercentileDisc(
override def sql(isDistinct: Boolean): String = {
val distinct = if (isDistinct) "DISTINCT " else ""
val direction = if (reverse) " DESC" else ""
- s"$prettyName($distinct${right.sql}) WITHIN GROUP (ORDER BY v$direction)"
+ s"$prettyName($distinct${right.sql}) WITHIN GROUP (ORDER BY ${left.sql}$direction)"
}
override protected def withNewChildrenInternal(
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
index 5b35b20f564..e947b96af86 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 150
+-- Number of queries: 152
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
index a1eb20da9f6..ca9283833e6 100644
--- a/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 83
+-- Number of queries: 85
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
index 264b64ffe96..f563e2d9492 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-legacy.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
+-- Number of queries: 21
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
index 2c622de3f36..74c5107d903 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nested.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
+-- Number of queries: 21
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
index 283f5a54a42..96abf5d2011 100644
--- a/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/cte-nonlegacy.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
+-- Number of queries: 21
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
index 0429efddafa..1803bf93019 100644
--- a/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/grouping_set.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 18
+-- Number of queries: 20
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
index d9aa34da6f1..47a6681e47a 100644
--- a/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/inline-table.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 17
+-- Number of queries: 19
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part4.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part4.sql.out
index 8c21a5067bf..8a19dabe430 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/aggregates_part4.sql.out
@@ -5,7 +5,7 @@
-- !query
select percentile_cont(0.5) within group (order by b) from aggtest
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY b):double>
-- !query output
53.44850015640259
@@ -13,7 +13,7 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
-- !query
select percentile_cont(0.5) within group (order by b), sum(b) from aggtest
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double,sum(b):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY b):double,sum(b):double>
-- !query output
53.44850015640259 431.77260909229517
@@ -21,7 +21,7 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double,sum(b):double>
-- !query
select percentile_cont(0.5) within group (order by thousand) from tenk1
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY thousand):double>
-- !query output
499.5
@@ -29,6 +29,6 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
-- !query
select percentile_disc(0.5) within group (order by thousand) from tenk1
-- !query schema
-struct<percentile_disc(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_disc(0.5) WITHIN GROUP (ORDER BY thousand):double>
-- !query output
499.0
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index 335a2b0c9b4..54d177a0fda 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
--- Number of queries: 150
+-- Number of queries: 152
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part4.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part4.sql.out
index 8c21a5067bf..8a19dabe430 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/postgreSQL/udf-aggregates_part4.sql.out
@@ -5,7 +5,7 @@
-- !query
select percentile_cont(0.5) within group (order by b) from aggtest
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY b):double>
-- !query output
53.44850015640259
@@ -13,7 +13,7 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
-- !query
select percentile_cont(0.5) within group (order by b), sum(b) from aggtest
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double,sum(b):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY b):double,sum(b):double>
-- !query output
53.44850015640259 431.77260909229517
@@ -21,7 +21,7 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double,sum(b):double>
-- !query
select percentile_cont(0.5) within group (order by thousand) from tenk1
-- !query schema
-struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY thousand):double>
-- !query output
499.5
@@ -29,6 +29,6 @@ struct<percentile_cont(0.5) WITHIN GROUP (ORDER BY v):double>
-- !query
select percentile_disc(0.5) within group (order by thousand) from tenk1
-- !query schema
-struct<percentile_disc(0.5) WITHIN GROUP (ORDER BY v):double>
+struct<percentile_disc(0.5) WITHIN GROUP (ORDER BY thousand):double>
-- !query output
499.0
diff --git a/sql/core/src/test/resources/sql-tests/results/window.sql.out b/sql/core/src/test/resources/sql-tests/results/window.sql.out
index 7e0fd2772d8..54e9ff20c29 100644
--- a/sql/core/src/test/resources/sql-tests/results/window.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/window.sql.out
@@ -1211,7 +1211,7 @@ SELECT
FROM basic_pays
ORDER BY salary
-- !query schema
-struct<employee_name:string,department:string,salary:int,percentile_cont(0.25) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.25) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,p [...]
+struct<employee_name:string,department:string,salary:int,percentile_cont(0.25) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.25) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.25) WITHIN GROUP (ORDER BY salary DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLL [...]
-- !query output
Leslie Thompson IT 5186 5917.75 5186.0 7381.25 8113.0
Anthony Bow Accounting 6627 8543.75 8435.0 9746.5 9998.0
@@ -1309,7 +1309,7 @@ FROM basic_pays
WINDOW w AS (PARTITION BY department)
ORDER BY salary
-- !query schema
-struct<employee_name:string,department:string,salary:int,percentile_cont(0.25) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.25) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.25) WITHIN GROUP (ORDER BY v DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,p [...]
+struct<employee_name:string,department:string,salary:int,percentile_cont(0.25) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.25) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.25) WITHIN GROUP (ORDER BY salary DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLL [...]
-- !query output
Leslie Thompson IT 5186 5917.75 5186.0 7381.25 8113.0
Anthony Bow Accounting 6627 8543.75 8435.0 9746.5 9998.0
@@ -1344,7 +1344,7 @@ WHERE salary > 8900
WINDOW w AS (PARTITION BY department)
ORDER BY salary
-- !query schema
-struct<employee_name:string,department:string,salary:int,percentile_cont(0.5) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.5) WITHIN GROUP (ORDER BY v) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.5) WITHIN GROUP (ORDER BY v DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,perc [...]
+struct<employee_name:string,department:string,salary:int,percentile_cont(0.5) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_disc(0.5) WITHIN GROUP (ORDER BY salary) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING):double,percentile_cont(0.5) WITHIN GROUP (ORDER BY salary DESC) OVER (PARTITION BY department ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWI [...]
-- !query output
Jeff Firrelli Accounting 8992 9998.0 9998.0 9998.0 9998.0
Julie Firrelli Sales 9181 9441.0 9441.0 9441.0 9441.0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org