You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/07/04 10:52:40 UTC
[spark] branch master updated: [SPARK-28516][SQL][FOLLOWUP] Remove try_to_char function
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 196095e7b96 [SPARK-28516][SQL][FOLLOWUP] Remove try_to_char function
196095e7b96 is described below
commit 196095e7b968b9081d0de3ac66528e85eba1c001
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Mon Jul 4 18:52:10 2022 +0800
[SPARK-28516][SQL][FOLLOWUP] Remove try_to_char function
### What changes were proposed in this pull request?
`try_to_char` is exactly the same as `to_char` now, because `to_char` only fails if the format string is invalid, which `try_to_char` will fai as well. This PR removes `try_to_char`.
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
No because the function is not released yet.
### How was this patch tested?
N/A
Closes #37051 from cloud-fan/tochar.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 -
.../expressions/numberFormatExpressions.scala | 69 ----------------------
.../expressions/StringExpressionsSuite.scala | 56 ------------------
.../sql-functions/sql-expression-schema.md | 1 -
4 files changed, 127 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index b236cf33af0..52d84cfa175 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -530,7 +530,6 @@ object FunctionRegistry {
expression[ToNumber]("to_number"),
expression[TryToNumber]("try_to_number"),
expression[ToCharacter]("to_char"),
- expression[TryToCharacter]("try_to_char"),
expression[GetJsonObject]("get_json_object"),
expression[InitCap]("initcap"),
expression[StringInstr]("instr"),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
index 0bf573334f2..ee8ef4633d9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/numberFormatExpressions.scala
@@ -255,72 +255,3 @@ case class ToCharacter(left: Expression, right: Expression)
newLeft: Expression, newRight: Expression): ToCharacter =
copy(left = newLeft, right = newRight)
}
-
-/**
- * A function that converts decimal values to strings, returning NULL if the decimal value fails to
- * match the format string.
- */
-@ExpressionDescription(
- usage = """
-
- _FUNC_(numberExpr, formatExpr) - Convert `numberExpr` to a string based on the `formatExpr`.
- Returns NULL if the conversion fails. The format follows the same semantics as the
- to_char function.
- """,
- examples = """
- Examples:
- > SELECT _FUNC_(454, '999');
- 454
- > SELECT _FUNC_(454.00, '000D00');
- 454.00
- > SELECT _FUNC_(12454, '99G999');
- 12,454
- > SELECT _FUNC_(78.12, '$99.99');
- $78.12
- > SELECT _FUNC_(-12454.8, '99G999D9S');
- 12,454.8-
- """,
- since = "3.4.0",
- group = "string_funcs")
-case class TryToCharacter(left: Expression, right: Expression)
- extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
- private lazy val numberFormat = right.eval().toString.toUpperCase(Locale.ROOT)
- private lazy val numberFormatter = new ToNumberParser(numberFormat, false)
-
- override def dataType: DataType = StringType
- override def inputTypes: Seq[AbstractDataType] = Seq(DecimalType, StringType)
- override def nullable: Boolean = true
- override def checkInputDataTypes(): TypeCheckResult =
- ToCharacter(left, right).checkInputDataTypes()
- override def prettyName: String = "try_to_char"
- override def nullSafeEval(decimal: Any, format: Any): Any = {
- val input = decimal.asInstanceOf[Decimal]
- numberFormatter.format(input)
- }
- override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val builder =
- ctx.addReferenceObj("builder", numberFormatter, classOf[ToNumberParser].getName)
- val eval = left.genCode(ctx)
- ev.copy(code =
- code"""
- |${eval.code}
- |boolean ${ev.isNull} = ${eval.isNull};
- |${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
- |if (!${ev.isNull}) {
- | UTF8String result = $builder.format(${eval.value});
- | if (result == null) {
- | ${ev.isNull} = true;
- | ${ev.value} = null;
- | } else {
- | ${ev.isNull} = false;
- | ${ev.value} = result;
- | }
- |}
- """.stripMargin)
- }
-
- override protected def withNewChildrenInternal(
- newLeft: Expression,
- newRight: Expression): TryToCharacter =
- copy(left = newLeft, right = newRight)
-}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 537ffb815b8..cae8cb9957d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -1076,14 +1076,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
case TypeCheckResult.TypeCheckFailure(message) =>
assert(message.contains(expectedErrMsg))
}
-
- val tryToCharResult = TryToCharacter(Decimal(456), Literal(format)).checkInputDataTypes()
- assert(tryToCharResult != TypeCheckResult.TypeCheckSuccess,
- s"The format string should have been invalid: $format")
- tryToCharResult match {
- case TypeCheckResult.TypeCheckFailure(message) =>
- assert(message.contains(expectedErrMsg))
- }
}
}
@@ -1156,10 +1148,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test '.' and 'D'
@@ -1194,14 +1182,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
expr = ToCharacter(Literal(decimal), Literal(format2))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format2))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
Seq(
@@ -1228,10 +1208,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test ',' and 'G'
@@ -1263,14 +1239,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
expr = ToCharacter(Literal(decimal), Literal(format2))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format2))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
Seq(
@@ -1323,10 +1291,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test '$'
@@ -1341,10 +1305,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test 'S'
@@ -1380,10 +1340,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test 'MI'
@@ -1416,10 +1372,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test 'PR'
@@ -1455,10 +1407,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
// Test overflows
@@ -1481,10 +1429,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
var expr: Expression = ToCharacter(Literal(decimal), Literal(format))
assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
checkEvaluation(expr, expected)
-
- expr = TryToCharacter(Literal(decimal), Literal(format))
- assert(expr.checkInputDataTypes() == TypeCheckResult.TypeCheckSuccess)
- checkEvaluation(expr, expected)
}
}
diff --git a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
index 369594c902a..e24ae5d0f7b 100644
--- a/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
+++ b/sql/core/src/test/resources/sql-functions/sql-expression-schema.md
@@ -317,7 +317,6 @@
| org.apache.spark.sql.catalyst.expressions.TryMultiply | try_multiply | SELECT try_multiply(2, 3) | struct<try_multiply(2, 3):int> |
| org.apache.spark.sql.catalyst.expressions.TrySubtract | try_subtract | SELECT try_subtract(2, 1) | struct<try_subtract(2, 1):int> |
| org.apache.spark.sql.catalyst.expressions.TryToBinary | try_to_binary | SELECT try_to_binary('abc', 'utf-8') | struct<try_to_binary(abc, utf-8):binary> |
-| org.apache.spark.sql.catalyst.expressions.TryToCharacter | try_to_char | SELECT try_to_char(454, '999') | struct<try_to_char(454, 999):string> |
| org.apache.spark.sql.catalyst.expressions.TryToNumber | try_to_number | SELECT try_to_number('454', '999') | struct<try_to_number(454, 999):decimal(3,0)> |
| org.apache.spark.sql.catalyst.expressions.TypeOf | typeof | SELECT typeof(1) | struct<typeof(1):string> |
| org.apache.spark.sql.catalyst.expressions.UnBase64 | unbase64 | SELECT unbase64('U3BhcmsgU1FM') | struct<unbase64(U3BhcmsgU1FM):binary> |
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org