You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2023/08/15 02:13:25 UTC
[spark] branch branch-3.5 updated: [SPARK-43063][SQL][FOLLOWUP] Add a space between -> and value when first value is null
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 247948194a0 [SPARK-43063][SQL][FOLLOWUP] Add a space between -> and value when first value is null
247948194a0 is described below
commit 247948194a08c6d090bfd26daf6b63a68374e6f2
Author: Andy Grove <an...@gmail.com>
AuthorDate: Tue Aug 15 10:12:55 2023 +0800
[SPARK-43063][SQL][FOLLOWUP] Add a space between -> and value when first value is null
As noted by cloud-fan https://github.com/apache/spark/pull/41432#discussion_r1242593592, https://github.com/apache/spark/pull/41432 fixed a formatting issue when casting map to string but did not fix it in the codegen case.
This PR fixes the issue and updates a unit test to test the codegen path. Without the fix, the test fails with:
```
- SPARK-22973 Cast map to string *** FAILED ***
Incorrect evaluation (fallback mode = CODEGEN_ONLY): cast(map(keys: [1,2,3], values: [null,[B5c3fd9f3,[B70f84210]) as string),
actual: {1 ->null, 2 -> a, 3 -> c},
expected: {1 -> null, 2 -> a, 3 -> c} (ExpressionEvalHelper.scala:270)
```
Closes #42434 from andygrove/space-before-null-cast-codegen.
Authored-by: Andy Grove <an...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 7e52169433575a7df164368106fa3f6c73e4233e)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/sql/catalyst/expressions/ToStringBase.scala | 2 +-
.../org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala
index f903863bec6..1eac386d873 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ToStringBase.scala
@@ -352,7 +352,7 @@ trait ToStringBase { self: UnaryExpression with TimeZoneAwareExpression =>
| $buffer.append($keyToStringFunc($getMapFirstKey));
| $buffer.append(" ->");
| if ($map.valueArray().isNullAt(0)) {
- | ${appendNull(buffer, isFirstElement = true)}
+ | ${appendNull(buffer, isFirstElement = false)}
| } else {
| $buffer.append(" ");
| $buffer.append($valueToStringFunc($getMapFirstValue));
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index 34f87f940a7..0172fd9b3e4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -826,9 +826,10 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
val ret1 = cast(Literal.create(Map(1 -> "a", 2 -> "b", 3 -> "c")), StringType)
checkEvaluation(ret1, s"${lb}1 -> a, 2 -> b, 3 -> c$rb")
val ret2 = cast(
- Literal.create(Map("1" -> "a".getBytes, "2" -> null, "3" -> "c".getBytes)),
+ Literal.create(Map("1" -> null, "2" -> "a".getBytes, "3" -> null, "4" -> "c".getBytes)),
StringType)
- checkEvaluation(ret2, s"${lb}1 -> a, 2 ->${if (legacyCast) "" else " null"}, 3 -> c$rb")
+ val nullStr = if (legacyCast) "" else " null"
+ checkEvaluation(ret2, s"${lb}1 ->$nullStr, 2 -> a, 3 ->$nullStr, 4 -> c$rb")
val ret3 = cast(
Literal.create(Map(
1 -> Date.valueOf("2014-12-03"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org