You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2020/05/27 06:09:18 UTC
[spark] branch branch-3.0 updated: [SPARK-31393][SQL][FOLLOW-UP]
Show the correct alias in schema for expression
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 59f9a88 [SPARK-31393][SQL][FOLLOW-UP] Show the correct alias in schema for expression
59f9a88 is described below
commit 59f9a88e61b680b34f7b4155acb4548a2bd8adbe
Author: beliefer <be...@163.com>
AuthorDate: Wed May 27 15:05:06 2020 +0900
[SPARK-31393][SQL][FOLLOW-UP] Show the correct alias in schema for expression
Some alias of expression can not display correctly in schema. This PR will fix them.
- `ln`
- `rint`
- `lcase`
- `position`
Improve the implement of some expression.
'Yes'. This PR will let user see the correct alias in schema.
Jenkins test.
Closes #28551 from beliefer/show-correct-alias-in-schema.
Lead-authored-by: beliefer <be...@163.com>
Co-authored-by: gengjiaan <ge...@360.cn>
Signed-off-by: HyukjinKwon <gu...@apache.org>
(cherry picked from commit 8f2b6f3a0b9d4297cb1f62e682958239fd6f9dbd)
Signed-off-by: HyukjinKwon <gu...@apache.org>
---
.../spark/sql/catalyst/analysis/FunctionRegistry.scala | 4 ++--
.../spark/sql/catalyst/expressions/mathExpressions.scala | 5 ++++-
.../spark/sql/catalyst/expressions/stringExpressions.scala | 6 +++++-
.../test/resources/sql-functions/sql-expression-schema.md | 10 +++++-----
.../sql-tests/results/ansi/string-functions.sql.out | 2 +-
.../resources/sql-tests/results/postgreSQL/numeric.sql.out | 14 +++++++-------
.../resources/sql-tests/results/string-functions.sql.out | 2 +-
7 files changed, 25 insertions(+), 18 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index c2c0df5..8608558 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -339,7 +339,7 @@ object FunctionRegistry {
expression[GetJsonObject]("get_json_object"),
expression[InitCap]("initcap"),
expression[StringInstr]("instr"),
- expression[Lower]("lcase"),
+ expression[Lower]("lcase", true),
expression[Length]("length"),
expression[Levenshtein]("levenshtein"),
expression[Like]("like"),
@@ -350,7 +350,7 @@ object FunctionRegistry {
expression[StringTrimLeft]("ltrim"),
expression[JsonTuple]("json_tuple"),
expression[ParseUrl]("parse_url"),
- expression[StringLocate]("position"),
+ expression[StringLocate]("position", true),
expression[FormatString]("printf", true),
expression[RegExpExtract]("regexp_extract"),
expression[RegExpReplace]("regexp_replace"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
index 66e6334..8c6fbc0 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathExpressions.scala
@@ -491,7 +491,9 @@ case class Factorial(child: Expression) extends UnaryExpression with ImplicitCas
> SELECT _FUNC_(1);
0.0
""")
-case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG")
+case class Log(child: Expression) extends UnaryLogExpression(StrictMath.log, "LOG") {
+ override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("ln")
+}
@ExpressionDescription(
usage = "_FUNC_(expr) - Returns the logarithm of `expr` with base 2.",
@@ -546,6 +548,7 @@ case class Log1p(child: Expression) extends UnaryLogExpression(StrictMath.log1p,
// scalastyle:on line.size.limit
case class Rint(child: Expression) extends UnaryMathExpression(math.rint, "ROUND") {
override def funcName: String = "rint"
+ override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("rint")
}
@ExpressionDescription(
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
index 82b1e5f..2cd92d7 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala
@@ -365,6 +365,9 @@ case class Lower(child: Expression) extends UnaryExpression with String2StringEx
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
defineCodeGen(ctx, ev, c => s"($c).toLowerCase()")
}
+
+ override def prettyName: String =
+ getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("lower")
}
/** A base trait for functions that compare two strings, returning a boolean. */
@@ -1182,7 +1185,8 @@ case class StringLocate(substr: Expression, str: Expression, start: Expression)
""")
}
- override def prettyName: String = "locate"
+ override def prettyName: String =
+ getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("locate")
}
/**
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index ed972ed..8040d0d 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -155,12 +155,12 @@
| org.apache.spark.sql.catalyst.expressions.LessThanOrEqual | <= | SELECT 2 <= 2 | struct<(2 <= 2):boolean> |
| org.apache.spark.sql.catalyst.expressions.Levenshtein | levenshtein | SELECT levenshtein('kitten', 'sitting') | struct<levenshtein(kitten, sitting):int> |
| org.apache.spark.sql.catalyst.expressions.Like | like | SELECT like('Spark', '_park') | struct<Spark LIKE _park:boolean> |
-| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<LOG(CAST(1 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Log | ln | SELECT ln(1) | struct<ln(CAST(1 AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.Log10 | log10 | SELECT log10(10) | struct<LOG10(CAST(10 AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.Log1p | log1p | SELECT log1p(0) | struct<LOG1P(CAST(0 AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.Log2 | log2 | SELECT log2(2) | struct<LOG2(CAST(2 AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.Logarithm | log | SELECT log(10, 100) | struct<LOG(CAST(10 AS DOUBLE), CAST(100 AS DOUBLE)):double> |
-| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lower(SparkSql):string> |
+| org.apache.spark.sql.catalyst.expressions.Lower | lcase | SELECT lcase('SparkSql') | struct<lcase(SparkSql):string> |
| org.apache.spark.sql.catalyst.expressions.Lower | lower | SELECT lower('SparkSql') | struct<lower(SparkSql):string> |
| org.apache.spark.sql.catalyst.expressions.MakeDate | make_date | SELECT make_date(2013, 7, 15) | struct<make_date(2013, 7, 15):date> |
| org.apache.spark.sql.catalyst.expressions.MakeInterval | make_interval | SELECT make_interval(100, 11, 1, 1, 12, 30, 01.001001) | struct<make_interval(100, 11, 1, 1, 12, 30, CAST(1.001001 AS DECIMAL(8,6))):interval> |
@@ -211,7 +211,7 @@
| org.apache.spark.sql.catalyst.expressions.Remainder | mod | SELECT 2 % 1.8 | struct<(CAST(CAST(2 AS DECIMAL(1,0)) AS DECIMAL(2,1)) % CAST(1.8 AS DECIMAL(2,1))):decimal(2,1)> |
| org.apache.spark.sql.catalyst.expressions.Reverse | reverse | SELECT reverse('Spark SQL') | struct<reverse(Spark SQL):string> |
| org.apache.spark.sql.catalyst.expressions.Right | right | SELECT right('Spark SQL', 3) | struct<right('Spark SQL', 3):string> |
-| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<ROUND(CAST(12.3456 AS DOUBLE)):double> |
+| org.apache.spark.sql.catalyst.expressions.Rint | rint | SELECT rint(12.3456) | struct<rint(CAST(12.3456 AS DOUBLE)):double> |
| org.apache.spark.sql.catalyst.expressions.Rollup | rollup | SELECT name, age, count(*) FROM VALUES (2, 'Alice'), (5, 'Bob') people(age, name) GROUP BY rollup(name, age) | struct<name:string,age:int,count(1):bigint> |
| org.apache.spark.sql.catalyst.expressions.Round | round | SELECT round(2.5, 0) | struct<round(2.5, 0):decimal(2,0)> |
| org.apache.spark.sql.catalyst.expressions.RowNumber | row_number | N/A | N/A |
@@ -242,7 +242,7 @@
| org.apache.spark.sql.catalyst.expressions.Stack | stack | SELECT stack(2, 1, 2, 3) | struct<col0:int,col1:int> |
| org.apache.spark.sql.catalyst.expressions.StringInstr | instr | SELECT instr('SparkSQL', 'SQL') | struct<instr(SparkSQL, SQL):int> |
| org.apache.spark.sql.catalyst.expressions.StringLPad | lpad | SELECT lpad('hi', 5, '??') | struct<lpad(hi, 5, ??):string> |
-| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
+| org.apache.spark.sql.catalyst.expressions.StringLocate | position | SELECT position('bar', 'foobarbar') | struct<position(bar, foobarbar, 1):int> |
| org.apache.spark.sql.catalyst.expressions.StringLocate | locate | SELECT locate('bar', 'foobarbar') | struct<locate(bar, foobarbar, 1):int> |
| org.apache.spark.sql.catalyst.expressions.StringRPad | rpad | SELECT rpad('hi', 5, '??') | struct<rpad(hi, 5, ??):string> |
| org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
@@ -333,4 +333,4 @@
| org.apache.spark.sql.catalyst.expressions.xml.XPathList | xpath | SELECT xpath('<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>','a/b/text()') | struct<xpath(<a><b>b1</b><b>b2</b><b>b3</b><c>c1</c><c>c2</c></a>, a/b/text()):array<string>> |
| org.apache.spark.sql.catalyst.expressions.xml.XPathLong | xpath_long | SELECT xpath_long('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_long(<a><b>1</b><b>2</b></a>, sum(a/b)):bigint> |
| org.apache.spark.sql.catalyst.expressions.xml.XPathShort | xpath_short | SELECT xpath_short('<a><b>1</b><b>2</b></a>', 'sum(a/b)') | struct<xpath_short(<a><b>1</b><b>2</b></a>, sum(a/b)):smallint> |
-| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
\ No newline at end of file
+| org.apache.spark.sql.catalyst.expressions.xml.XPathString | xpath_string | SELECT xpath_string('<a><b>b</b><c>cc</c></a>','a/c') | struct<xpath_string(<a><b>b</b><c>cc</c></a>, a/c):string> |
diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
index 3329150..dfdf183 100644
--- a/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
-- !query
select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
-- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
-- !query output
4 NULL NULL
diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
index e59b9d5..7b7aeb4 100644
--- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/numeric.sql.out
@@ -4654,7 +4654,7 @@ struct<id:bigint>
-- !query
select ln(1.2345678e-28)
-- !query schema
-struct<LOG(1.2345678E-28):double>
+struct<ln(1.2345678E-28):double>
-- !query output
-64.26166165451762
@@ -4662,7 +4662,7 @@ struct<LOG(1.2345678E-28):double>
-- !query
select ln(0.0456789)
-- !query schema
-struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
+struct<ln(CAST(0.0456789 AS DOUBLE)):double>
-- !query output
-3.0861187944847437
@@ -4670,7 +4670,7 @@ struct<LOG(CAST(0.0456789 AS DOUBLE)):double>
-- !query
select ln(0.99949452)
-- !query schema
-struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
+struct<ln(CAST(0.99949452 AS DOUBLE)):double>
-- !query output
-5.056077980832118E-4
@@ -4678,7 +4678,7 @@ struct<LOG(CAST(0.99949452 AS DOUBLE)):double>
-- !query
select ln(1.00049687395)
-- !query schema
-struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
+struct<ln(CAST(1.00049687395 AS DOUBLE)):double>
-- !query output
4.967505490136803E-4
@@ -4686,7 +4686,7 @@ struct<LOG(CAST(1.00049687395 AS DOUBLE)):double>
-- !query
select ln(1234.567890123456789)
-- !query schema
-struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
+struct<ln(CAST(1234.567890123456789 AS DOUBLE)):double>
-- !query output
7.11847630129779
@@ -4694,7 +4694,7 @@ struct<LOG(CAST(1234.567890123456789 AS DOUBLE)):double>
-- !query
select ln(5.80397490724e5)
-- !query schema
-struct<LOG(580397.490724):double>
+struct<ln(580397.490724):double>
-- !query output
13.271468476626518
@@ -4702,7 +4702,7 @@ struct<LOG(580397.490724):double>
-- !query
select ln(9.342536355e34)
-- !query schema
-struct<LOG(9.342536355E34):double>
+struct<ln(9.342536355E34):double>
-- !query output
80.52247093552418
diff --git a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
index cb5501d..c9aad35 100644
--- a/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/string-functions.sql.out
@@ -55,7 +55,7 @@ struct<length(uuid()):int,(NOT (uuid() = uuid())):boolean>
-- !query
select position('bar' in 'foobarbar'), position(null, 'foobarbar'), position('aaads', null)
-- !query schema
-struct<locate(bar, foobarbar, 1):int,locate(CAST(NULL AS STRING), foobarbar, 1):int,locate(aaads, CAST(NULL AS STRING), 1):int>
+struct<locate(bar, foobarbar, 1):int,position(CAST(NULL AS STRING), foobarbar, 1):int,position(aaads, CAST(NULL AS STRING), 1):int>
-- !query output
4 NULL NULL
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org