You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/01/04 04:40:53 UTC
[spark] branch branch-3.2 updated: [SPARK-37800][SQL] TreeNode.argString incorrectly formats arguments of type Set[_]
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c280f08 [SPARK-37800][SQL] TreeNode.argString incorrectly formats arguments of type Set[_]
c280f08 is described below
commit c280f08e18c7372fc94295bb8e9e24d6f77f2e53
Author: Simeon Simeonov <si...@fastignite.com>
AuthorDate: Tue Jan 4 12:38:43 2022 +0800
[SPARK-37800][SQL] TreeNode.argString incorrectly formats arguments of type Set[_]
### What changes were proposed in this pull request?
Fixing [SPARK-37800: TreeNode.argString incorrectly formats arguments of type Set\[_\]](https://issues.apache.org/jira/browse/SPARK-37800).
### Why are the changes needed?
On May 21, 2021, 746d80d87a480344d4cc0d30b481047cf85c0aa9 by maropu introduced this bug.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New test added to `TreeNodeSuite`.
Closes #35084 from ssimeonov/ss_spark37800.
Authored-by: Simeon Simeonov <si...@fastignite.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit a8addd487bcd4118fd87dabe686d07f658216eb7)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala | 12 +++---------
.../org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala | 8 ++++++++
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
index f3f6744..52cd1bb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreeNode.scala
@@ -787,7 +787,7 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product with Tre
truncatedString(seq.map(formatArg(_, maxFields)), "[", ", ", "]", maxFields)
case set: Set[_] =>
// Sort elements for deterministic behaviours
- truncatedString(set.toSeq.map(formatArg(_, maxFields).sorted), "{", ", ", "}", maxFields)
+ truncatedString(set.toSeq.map(formatArg(_, maxFields)).sorted, "{", ", ", "}", maxFields)
case array: Array[_] =>
truncatedString(array.map(formatArg(_, maxFields)), "[", ", ", "]", maxFields)
case other =>
@@ -802,15 +802,9 @@ abstract class TreeNode[BaseType <: TreeNode[BaseType]] extends Product with Tre
case tn: TreeNode[_] => tn.simpleString(maxFields) :: Nil
case seq: Seq[Any] if seq.toSet.subsetOf(allChildren.asInstanceOf[Set[Any]]) => Nil
case iter: Iterable[_] if iter.isEmpty => Nil
- case seq: Seq[_] =>
- truncatedString(seq.map(formatArg(_, maxFields)), "[", ", ", "]", maxFields) :: Nil
- case set: Set[_] =>
- // Sort elements for deterministic behaviours
- val sortedSeq = set.toSeq.map(formatArg(_, maxFields).sorted)
- truncatedString(sortedSeq, "{", ", ", "}", maxFields) :: Nil
case array: Array[_] if array.isEmpty => Nil
- case array: Array[_] =>
- truncatedString(array.map(formatArg(_, maxFields)), "[", ", ", "]", maxFields) :: Nil
+ case xs @ (_: Seq[_] | _: Set[_] | _: Array[_]) =>
+ formatArg(xs, maxFields) :: Nil
case null => Nil
case None => Nil
case Some(null) => Nil
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
index 91cc1f6..481153e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/trees/TreeNodeSuite.scala
@@ -810,4 +810,12 @@ class TreeNodeSuite extends SparkFunSuite with SQLHelper {
fail("TreeNode.nodeName should not throw malformed class name error")
}
}
+
+ test("SPARK-37800: TreeNode.argString incorrectly formats arguments of type Set[_]") {
+ case class Node(set: Set[String], nested: Seq[Set[Int]]) extends LogicalPlan with LeafNode {
+ val output: Seq[Attribute] = Nil
+ }
+ val node = Node(Set("second", "first"), Seq(Set(3, 1), Set(2, 1)))
+ assert(node.argString(10) == "{first, second}, [{1, 3}, {1, 2}]")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org