You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/09/20 13:12:15 UTC
(spark) branch branch-4.0 updated: [SPARK-59602][SQL] Handle floating-point normalization in array set operations
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 7fff7b8889ac [SPARK-59602][SQL] Handle floating-point normalization in array set operations
7fff7b8889ac is described below
commit 7fff7b8889ac0a420320f461e7422cef1967e7e4
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Sun Sep 20 12:37:07 2026 +0000
[SPARK-59602][SQL] Handle floating-point normalization in array set operations
This replaces the logical-plan implementation introduced by
https://github.com/apache/spark/pull/53695 with normalization inside the array set expressions:
- Canonicalize primitive `Float` and `Double` values inside the interpreted and generated
hash-set loops.
- Canonicalize complex element values only when an element is emitted into a result array.
- Handle primitive floating-point values locally in `arrays_overlap`.
- Remove the array-expression rewrites and their tree patterns from
`NormalizeFloatingNumbers`, as well as its extra invocation from `FinishAnalysis`.
`NormalizeFloatingNumbers` continues to handle joins and window partitioning in its existing
optimizer batch.
The target branches are `master`, `branch-4.x`, `branch-4.3`, `branch-4.2`, `branch-4.1`, and
`branch-4.0`. Each contains the SPARK-54918 implementation that this PR replaces.
JIRA: https://issues.apache.org/jira/browse/SPARK-59602
The array set operations own their element equality and hashing semantics. Wrapping their inputs
in `ArrayTransform` expressions scans and materializes normalized arrays before the operations
scan them again, and exposes an execution detail as a logical-plan rewrite.
Applying this rewrite in `FinishAnalysis` also expands array set expressions into
`ArrayTransform` and normalization expressions too early. This complicates the logical plan and
can invalidate assumptions made by downstream or custom optimizer rules that match the original
expression shape.
Handling special floating-point values at the point where each operation hashes, compares, or
emits an element preserves the SQL semantics with less work. It also covers interpreted
evaluation, code generation, and constant folding without depending on a `FinishAnalysis`
rewrite.
No. Across all target branches, `array_distinct`, `array_union`, `array_intersect`,
`array_except`, and `arrays_overlap` continue to treat `-0.0` and `+0.0`, as well as different
NaN representations, as equivalent. The implementation no longer injects normalization
transforms into their input plans.
The following focused tests passed:
```
build/sbt 'catalyst/testOnly org.apache.spark.sql.catalyst.expressions.CollectionExpressionsSuite -- -z SPARK-59602'
build/sbt 'catalyst/testOnly org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingPointNumbersSuite -- -z SPARK-59602'
build/sbt 'sql/testOnly org.apache.spark.sql.DataFrameFunctionsSuite -- -z SPARK-59602'
```
All four targeted tests passed. `git diff --check` also passed.
Generated-by: OpenAI Codex (GPT-5)
Closes #58818 from cloud-fan/wenchen/SPARK-54918-execution-normalization.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 16c3272d893f10fae8795e23018b60e61d9d654d)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../expressions/collectionOperations.scala | 110 +++++++++++++-----
.../optimizer/NormalizeFloatingNumbers.scala | 97 +++++++---------
.../spark/sql/catalyst/optimizer/Optimizer.scala | 1 -
.../spark/sql/catalyst/trees/TreePatterns.scala | 5 -
.../org/apache/spark/sql/util/SQLOpenHashSet.scala | 32 ++++--
.../expressions/CollectionExpressionsSuite.scala | 42 +++++++
.../NormalizeFloatingPointNumbersSuite.scala | 126 ++-------------------
.../apache/spark/sql/DataFrameFunctionsSuite.scala | 8 ++
8 files changed, 209 insertions(+), 212 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index dd7343f902ba..bac8f3ca7730 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -30,8 +30,13 @@ import org.apache.spark.sql.catalyst.expressions.KnownNotContainsNull
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke
+import org.apache.spark.sql.catalyst.optimizer.NormalizeFloatingNumbers
import org.apache.spark.sql.catalyst.trees.{BinaryLike, UnaryLike}
-import org.apache.spark.sql.catalyst.trees.TreePattern.{ARRAY_DISTINCT, ARRAY_EXCEPT, ARRAY_INTERSECT, ARRAY_UNION, ARRAYS_OVERLAP, ARRAYS_ZIP, CONCAT, TreePattern}
+import org.apache.spark.sql.catalyst.trees.TreePattern.{
+ ARRAYS_ZIP,
+ CONCAT,
+ TreePattern
+}
import org.apache.spark.sql.catalyst.types.{DataTypeUtils, PhysicalDataType, PhysicalIntegralType}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
@@ -1770,9 +1775,6 @@ case class ArrayAppend(left: Expression, right: Expression) extends ArrayPendBas
// scalastyle:off line.size.limit
case class ArraysOverlap(left: Expression, right: Expression)
extends BinaryArrayExpressionWithImplicitCast with Predicate {
-
- final override val nodePatterns: Seq[TreePattern] = Seq(ARRAYS_OVERLAP)
-
override def nullIntolerant: Boolean = true
override def checkInputDataTypes(): TypeCheckResult = super.checkInputDataTypes() match {
@@ -1784,6 +1786,12 @@ case class ArraysOverlap(left: Expression, right: Expression)
@transient private lazy val ordering: Ordering[Any] =
TypeUtils.getInterpretedOrdering(elementType)
+ @transient private lazy val normalizeElement: Any => Any = elementType match {
+ case FloatType => NormalizeFloatingNumbers.FLOAT_NORMALIZER
+ case DoubleType => NormalizeFloatingNumbers.DOUBLE_NORMALIZER
+ case _ => identity
+ }
+
@transient private lazy val doEvaluation = if (TypeUtils.typeWithProperEquals(elementType)) {
fastEval _
} else {
@@ -1816,12 +1824,12 @@ case class ArraysOverlap(left: Expression, right: Expression)
if (v == null) {
hasNull = true
} else {
- smallestSet.add(v)
+ smallestSet.add(normalizeElement(v))
})
bigger.foreach(elementType, (_, v1) =>
if (v1 == null) {
hasNull = true
- } else if (smallestSet.contains(v1)) {
+ } else if (smallestSet.contains(normalizeElement(v1))) {
return true
}
)
@@ -1894,22 +1902,70 @@ case class ArraysOverlap(left: Expression, right: Expression)
val i = ctx.freshName("i")
val getFromSmaller = CodeGenerator.getValue(smaller, elementType, i)
val getFromBigger = CodeGenerator.getValue(bigger, elementType, i)
+ val javaElementType = CodeGenerator.javaType(elementType)
val javaElementClass = CodeGenerator.boxedType(elementType)
val javaSet = classOf[java.util.HashSet[_]].getName
val set = ctx.freshName("set")
+
+ def normalize(value: String): String = elementType match {
+ case DoubleType =>
+ s"""
+ |if (java.lang.Double.isNaN($value)) {
+ | $value = java.lang.Double.NaN;
+ |} else if ($value == 0.0d) {
+ | $value = 0.0d;
+ |}
+ """.stripMargin
+ case FloatType =>
+ s"""
+ |if (java.lang.Float.isNaN($value)) {
+ | $value = java.lang.Float.NaN;
+ |} else if ($value == 0.0f) {
+ | $value = 0.0f;
+ |}
+ """.stripMargin
+ case _ => ""
+ }
+
+ val smallerValue = ctx.freshName("smallerValue")
+ val addToSet = elementType match {
+ case FloatType | DoubleType =>
+ s"""
+ |$javaElementType $smallerValue = $getFromSmaller;
+ |${normalize(smallerValue)}
+ |$set.add($smallerValue);
+ """.stripMargin
+ case _ => s"$set.add($getFromSmaller);"
+ }
val addToSetFromSmallerCode = nullSafeElementCodegen(
- smaller, i, s"$set.add($getFromSmaller);", s"${ev.isNull} = true;")
+ smaller, i, addToSet, s"${ev.isNull} = true;")
val setIsNullCode = if (nullable) s"${ev.isNull} = false;" else ""
+
+ val biggerValue = ctx.freshName("biggerValue")
+ val findInSet = elementType match {
+ case FloatType | DoubleType =>
+ s"""
+ |$javaElementType $biggerValue = $getFromBigger;
+ |${normalize(biggerValue)}
+ |if ($set.contains($biggerValue)) {
+ | $setIsNullCode
+ | ${ev.value} = true;
+ | break;
+ |}
+ """.stripMargin
+ case _ =>
+ s"""
+ |if ($set.contains($getFromBigger)) {
+ | $setIsNullCode
+ | ${ev.value} = true;
+ | break;
+ |}
+ """.stripMargin
+ }
val elementIsInSetCode = nullSafeElementCodegen(
bigger,
i,
- s"""
- |if ($set.contains($getFromBigger)) {
- | $setIsNullCode
- | ${ev.value} = true;
- | break;
- |}
- """.stripMargin,
+ findInSet,
s"${ev.isNull} = true;")
s"""
|$javaSet<$javaElementClass> $set = new $javaSet<$javaElementClass>();
@@ -4084,6 +4140,15 @@ trait ArraySetLike {
@transient protected lazy val ordering: Ordering[Any] =
TypeUtils.getInterpretedOrdering(et)
+ @transient protected lazy val normalizedElement: Any => Any =
+ if (NormalizeFloatingNumbers.needNormalize(et)) {
+ val ref = BoundReference(0, et, nullable = true)
+ val normalizer = NormalizeFloatingNumbers.normalize(ref)
+ (value: Any) => InternalRow.copyValue(normalizer.eval(InternalRow(value)))
+ } else {
+ identity
+ }
+
protected def resultArrayElementNullable = dt.asInstanceOf[ArrayType].containsNull
protected def genGetValue(array: String, i: String): String =
@@ -4170,9 +4235,6 @@ trait ArraySetLike {
since = "2.4.0")
case class ArrayDistinct(child: Expression)
extends UnaryExpression with ArraySetLike with ExpectsInputTypes {
-
- final override val nodePatterns: Seq[TreePattern] = Seq(ARRAY_DISTINCT)
-
override def nullIntolerant: Boolean = true
override def inputTypes: Seq[AbstractDataType] = Seq(ArrayType)
@@ -4235,7 +4297,7 @@ case class ArrayDistinct(child: Expression)
j += 1
}
if (!found) {
- arrayBuffer += array(i)
+ arrayBuffer += normalizedElement(array(i)).asInstanceOf[AnyRef]
}
} else {
// De-duplicate the null values.
@@ -4369,8 +4431,6 @@ trait ArrayBinaryLike
case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike
with ComplexTypeMergingExpression {
- final override val nodePatterns: Seq[TreePattern] = Seq(ARRAY_UNION)
-
@transient lazy val evalUnion: (ArrayData, ArrayData) => ArrayData = {
if (TypeUtils.typeWithProperEquals(elementType)) {
(array1, array2) =>
@@ -4427,7 +4487,7 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi
throw QueryExecutionErrors.arrayFunctionWithElementsExceedLimitError(
prettyName, arrayBuffer.length)
}
- arrayBuffer += elem
+ arrayBuffer += normalizedElement(elem)
}
}))
new GenericArrayData(arrayBuffer)
@@ -4548,8 +4608,6 @@ case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLi
case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike
with ComplexTypeMergingExpression {
- final override val nodePatterns: Seq[TreePattern] = Seq(ARRAY_INTERSECT)
-
private lazy val internalDataType: DataType = {
dataTypeCheck
ArrayType(elementType, leftArrayElementNullable && rightArrayElementNullable)
@@ -4643,7 +4701,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
}
}
if (found) {
- arrayBuffer += elem1
+ arrayBuffer += normalizedElement(elem1)
}
i += 1
}
@@ -4782,8 +4840,6 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina
case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike
with ComplexTypeMergingExpression {
- final override val nodePatterns: Seq[TreePattern] = Seq(ARRAY_EXCEPT)
-
private lazy val internalDataType: DataType = {
dataTypeCheck
left.dataType
@@ -4865,7 +4921,7 @@ case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryL
}
}
if (!found) {
- arrayBuffer += elem1
+ arrayBuffer += normalizedElement(elem1)
}
i += 1
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
index 44add1796169..a57ac32a282a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingNumbers.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.SparkException
-import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayDistinct, ArrayExcept, ArrayIntersect, ArraysOverlap, ArrayTransform, ArrayUnion, CaseWhen, Coalesce, CreateArray, CreateMap, CreateNamedStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, Literal, NamedLambdaVariable, TransformValues, UnaryExpression}
+import org.apache.spark.sql.catalyst.expressions.{Alias, And, ArrayTransform, CaseWhen, Coalesce, CreateArray, CreateMap, CreateNamedStruct, EqualTo, ExpectsInputTypes, Expression, GetStructField, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, Literal, NamedLambdaVariable, TransformValues, UnaryExpression}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Window}
@@ -47,67 +47,48 @@ import org.apache.spark.util.ArrayImplicits._
* binary `UnsafeRow` and compare the binary data directly. Different NaNs have different binary
* representation, and the same thing happens for `-0.0` and `0.0`.
*
- * Case 5 is problematic for a similar reason: hash-based array set operations compare elements by
- * their binary representation via hash sets.
+ * Case 2 is handled during planning in the `Aggregation` and `StatefulAggregationStrategy` objects
+ * of [[org.apache.spark.sql.execution.SparkStrategies]].
+ * Cases 3 and 4 are handled by this optimizer rule. Array set operations handle case 5 in their
+ * expression evaluation.
*
- * This rule runs in two places:
- * 1. Early in `FinishAnalysis` (right after `ReplaceExpressions` and before `EvalInlineTables`)
- * so that array set-like operations are wrapped before optimizer rules that pre-evaluate
- * expressions (e.g. `ConstantFolding`, `ConvertToLocalRelation`, `EvalInlineTables`).
- *
- * 2. As a late batch at the end of the optimizer, because rules like subquery rewrite and
- * join reorder can create new joins or join conditions after `FinishAnalysis` that still
- * need their keys to be normalized.
- *
- * Ideally we should do the normalization in the physical operators that compare the
- * binary `UnsafeRow` directly. We don't need this normalization if the Spark SQL execution engine
- * is not optimized to run on binary data. This rule is created to simplify the implementation, so
- * that we have a single place to do normalization, which is more maintainable.
+ * Ideally we should do the normalization in the physical operators that compare the binary
+ * `UnsafeRow` directly. We don't need this normalization if the Spark SQL execution engine is not
+ * optimized to run on binary data. Array set operations normalize values during expression
+ * evaluation instead, where values enter hash-based comparisons or result arrays.
*
+ * Note that, this rule must be executed at the end of optimizer, because the optimizer may create
+ * new joins(the subquery rewrite) and new join conditions(the join reorder).
*/
object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
- def apply(plan: LogicalPlan): LogicalPlan = {
- plan
- .transformWithPruning( _.containsAnyPattern(WINDOW, JOIN)) {
- case w: Window if w.partitionSpec.exists(p => needNormalize(p)) =>
- // Although the `windowExpressions` may refer to `partitionSpec` expressions,
- // we don't need to normalize the `windowExpressions`, as they are executed
- // per input row and should take the input row as it is.
- w.copy(partitionSpec = w.partitionSpec.map(normalize))
-
- // Only hash join and sort merge join need the normalization. Here we catch all Joins with
- // join keys, assuming Joins with join keys are always planned as hash join or sort merge
- // join. It's very unlikely that we will break this assumption in the near future.
- case j @ ExtractEquiJoinKeys(_, leftKeys, rightKeys, condition, _, _, _, _)
- // The analyzer guarantees left and right joins keys are of the same data type. Here we
- // only need to check join keys of one side.
- if leftKeys.exists(k => needNormalize(k)) =>
- val newLeftJoinKeys = leftKeys.map(normalize)
- val newRightJoinKeys = rightKeys.map(normalize)
- val newConditions = newLeftJoinKeys.zip(newRightJoinKeys).map {
- case (l, r) => EqualTo(l, r)
- } ++ condition
- j.copy(condition = Some(newConditions.reduce(And)))
-
- // TODO: ideally Aggregate should also be handled here, but its grouping expressions are
- // mixed in its aggregate expressions. It's unreliable to change the grouping expressions
- // here. For now we normalize grouping expressions in `AggUtils` during planning.
- }
- .transformAllExpressionsWithPruning(_.containsAnyPattern(
- ARRAY_DISTINCT, ARRAY_UNION, ARRAY_INTERSECT, ARRAY_EXCEPT, ARRAYS_OVERLAP)) {
- case e: ArrayDistinct if needNormalize(e.child) =>
- e.copy(child = normalize(e.child))
- case e: ArrayUnion if needNormalize(e.left) =>
- e.copy(left = normalize(e.left), right = normalize(e.right))
- case e: ArrayIntersect if needNormalize(e.left) =>
- e.copy(left = normalize(e.left), right = normalize(e.right))
- case e: ArrayExcept if needNormalize(e.left) =>
- e.copy(left = normalize(e.left), right = normalize(e.right))
- case e: ArraysOverlap if needNormalize(e.left) =>
- e.copy(left = normalize(e.left), right = normalize(e.right))
- }
- }
+ def apply(plan: LogicalPlan): LogicalPlan =
+ plan.transformWithPruning(_.containsAnyPattern(WINDOW, JOIN)) {
+ case w: Window if w.partitionSpec.exists(p => needNormalize(p)) =>
+ // Although the `windowExpressions` may refer to `partitionSpec` expressions, we don't need
+ // to normalize the `windowExpressions`, as they are executed per input row and should take
+ // the input row as it is.
+ w.copy(partitionSpec = w.partitionSpec.map(normalize))
+
+ // Only hash join and sort merge join need the normalization. Here we catch all Joins with
+ // join keys, assuming Joins with join keys are always planned as hash join or sort merge
+ // join. It's very unlikely that we will break this assumption in the near future.
+ case j @ ExtractEquiJoinKeys(_, leftKeys, rightKeys, condition, _, _, _, _)
+ // The analyzer guarantees left and right joins keys are of the same data type. Here we
+ // only need to check join keys of one side.
+ if leftKeys.exists(k => needNormalize(k)) =>
+ val newLeftJoinKeys = leftKeys.map(normalize)
+ val newRightJoinKeys = rightKeys.map(normalize)
+ val newConditions = newLeftJoinKeys.zip(newRightJoinKeys).map {
+ case (l, r) => EqualTo(l, r)
+ } ++ condition
+ j.copy(condition = Some(newConditions.reduce(And)))
+
+ // TODO: ideally Aggregate should also be handled here, but its grouping expressions are
+ // mixed in its aggregate expressions. It's unreliable to change the grouping expressions
+ // here. For now we normalize grouping expressions during planning. See Case 2 in the
+ // Scaladoc just above.
+ }
/**
* Short circuit if the underlying expression is already normalized
@@ -117,7 +98,7 @@ object NormalizeFloatingNumbers extends Rule[LogicalPlan] {
case _ => needNormalize(expr.dataType)
}
- private def needNormalize(dt: DataType): Boolean = dt match {
+ private[sql] def needNormalize(dt: DataType): Boolean = dt match {
case FloatType | DoubleType => true
case StructType(fields) => fields.exists(f => needNormalize(f.dataType))
case ArrayType(et, _) => needNormalize(et)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index e18034bed19c..6e7e6e8e46c8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -320,7 +320,6 @@ abstract class Optimizer(catalogManager: CatalogManager)
EliminateView,
EliminateSQLFunctionNode,
ReplaceExpressions,
- NormalizeFloatingNumbers,
RewriteNonCorrelatedExists,
PullOutGroupingExpressions,
// Put `InsertMapSortInGroupingExpressions` after `PullOutGroupingExpressions`,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
index 322d65455d22..9856a26346f6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/trees/TreePatterns.scala
@@ -26,12 +26,7 @@ object TreePattern extends Enumeration {
val AGGREGATE_EXPRESSION = Value(0)
val ALIAS: Value = Value
val AND: Value = Value
- val ARRAYS_OVERLAP: Value = Value
val ARRAYS_ZIP: Value = Value
- val ARRAY_DISTINCT: Value = Value
- val ARRAY_EXCEPT: Value = Value
- val ARRAY_INTERSECT: Value = Value
- val ARRAY_UNION: Value = Value
val ATTRIBUTE_REFERENCE: Value = Value
val APPEND_COLUMNS: Value = Value
val AVERAGE: Value = Value
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SQLOpenHashSet.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SQLOpenHashSet.scala
index 10023c2ecca5..bdbe15fe4032 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SQLOpenHashSet.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/util/SQLOpenHashSet.scala
@@ -24,7 +24,8 @@ import org.apache.spark.sql.catalyst.util.ArrayData
import org.apache.spark.sql.types.{DataType, DoubleType, FloatType}
import org.apache.spark.util.collection.OpenHashSet
-// A wrap of OpenHashSet that can handle null, Double.NaN and Float.NaN w.r.t. the SQL semantic.
+// A wrap of OpenHashSet that can handle null and special floating-point values according to SQL
+// semantics.
@Private
class SQLOpenHashSet[@specialized(Long, Int, Double, Float) T: ClassTag](
initialCapacity: Int,
@@ -110,19 +111,24 @@ object SQLOpenHashSet {
}
}
+ /**
+ * Handles NaNs specially and normalizes negative zero before invoking `handleNotNaN`.
+ */
def withNaNCheckFunc(
dataType: DataType,
hashSet: SQLOpenHashSet[Any],
handleNotNaN: Any => Unit,
handleNaN: Any => Unit): Any => Unit = {
- val (isNaN, valueNaN) = dataType match {
+ val (isNaN, normalize, valueNaN) = dataType match {
case DoubleType =>
((value: Any) => java.lang.Double.isNaN(value.asInstanceOf[java.lang.Double]),
+ (value: Any) => if (value.asInstanceOf[Double] == 0.0d) 0.0d else value,
java.lang.Double.NaN)
case FloatType =>
((value: Any) => java.lang.Float.isNaN(value.asInstanceOf[java.lang.Float]),
+ (value: Any) => if (value.asInstanceOf[Float] == 0.0f) 0.0f else value,
java.lang.Float.NaN)
- case _ => ((_: Any) => false, null)
+ case _ => ((_: Any) => false, (value: Any) => value, null)
}
(value: Any) =>
if (isNaN(value)) {
@@ -131,10 +137,15 @@ object SQLOpenHashSet {
handleNaN(valueNaN)
}
} else {
- handleNotNaN(value)
+ handleNotNaN(normalize(value))
}
}
+ /**
+ * Handles NaNs specially and normalizes negative zero before invoking `handleNotNaN`.
+ * `valueName` must refer to a writable generated-code local because zero normalization assigns
+ * the canonical value back to it.
+ */
def withNaNCheckCode(
dataType: DataType,
valueName: String,
@@ -143,12 +154,18 @@ object SQLOpenHashSet {
handleNaN: String => String): String = {
val ret = dataType match {
case DoubleType =>
- Some((s"java.lang.Double.isNaN((double)$valueName)", "java.lang.Double.NaN"))
+ Some((
+ s"java.lang.Double.isNaN((double)$valueName)",
+ s"if ($valueName == 0.0d) $valueName = 0.0d;",
+ "java.lang.Double.NaN"))
case FloatType =>
- Some((s"java.lang.Float.isNaN((float)$valueName)", "java.lang.Float.NaN"))
+ Some((
+ s"java.lang.Float.isNaN((float)$valueName)",
+ s"if ($valueName == 0.0f) $valueName = 0.0f;",
+ "java.lang.Float.NaN"))
case _ => None
}
- ret.map { case (isNaN, valueNaN) =>
+ ret.map { case (isNaN, normalizeZero, valueNaN) =>
s"""
|if ($isNaN) {
| if (!$hashSet.containsNaN()) {
@@ -156,6 +173,7 @@ object SQLOpenHashSet {
| ${handleNaN(valueNaN)}
| }
|} else {
+ | $normalizeZero
| $handleNotNaN
|}
""".stripMargin
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 78804144931a..a17f210e4ae6 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -3052,6 +3052,48 @@ class CollectionExpressionsSuite
Literal.create(Seq(Float.NaN, null, 1f), ArrayType(FloatType))), true)
}
+ test("SPARK-59602: array set operations normalize special floating-point values") {
+ val nonCanonicalNaN = java.lang.Double.longBitsToDouble(0x7ff8000000000001L)
+ val doubles = Literal.create(
+ Seq(-0.0d, 0.0d, nonCanonicalNaN, Double.NaN), ArrayType(DoubleType, false))
+ val doubleSet = Literal.create(Seq(0.0d, Double.NaN), ArrayType(DoubleType, false))
+ checkEvaluation(ArrayDistinct(doubles), Seq(0.0d, Double.NaN))
+ checkEvaluation(ArrayUnion(doubles, doubleSet), Seq(0.0d, Double.NaN))
+ checkEvaluation(ArrayIntersect(doubles, doubleSet), Seq(0.0d, Double.NaN))
+ checkEvaluation(ArrayExcept(doubles, doubleSet), Seq.empty[Double])
+ checkEvaluation(ArrayExcept(doubleSet, doubles), Seq.empty[Double])
+ checkEvaluation(ArraysOverlap(doubles, doubleSet), true)
+
+ val nonCanonicalFloatNaN = java.lang.Float.intBitsToFloat(0x7f800001)
+ val floats = Literal.create(
+ Seq(-0.0f, 0.0f, nonCanonicalFloatNaN, Float.NaN), ArrayType(FloatType, false))
+ val floatSet = Literal.create(Seq(0.0f, Float.NaN), ArrayType(FloatType, false))
+ checkEvaluation(ArrayDistinct(floats), Seq(0.0f, Float.NaN))
+ checkEvaluation(ArrayUnion(floats, floatSet), Seq(0.0f, Float.NaN))
+ checkEvaluation(ArrayIntersect(floats, floatSet), Seq(0.0f, Float.NaN))
+ checkEvaluation(ArrayExcept(floats, floatSet), Seq.empty[Float])
+ checkEvaluation(ArrayExcept(floatSet, floats), Seq.empty[Float])
+ checkEvaluation(ArraysOverlap(floats, floatSet), true)
+ }
+
+ test("SPARK-59602: array set operations normalize nested floating-point values") {
+ val nestedType = ArrayType(DoubleType, containsNull = false)
+ val nested = Literal.create(
+ Seq(Seq(-0.0d), Seq(0.0d), Seq(Double.NaN)), ArrayType(nestedType, false))
+ val nestedSet = Literal.create(
+ Seq(Seq(0.0d), Seq(Double.NaN)), ArrayType(nestedType, false))
+ checkEvaluation(ArrayDistinct(nested), Seq(Seq(0.0d), Seq(Double.NaN)))
+ checkEvaluation(ArrayUnion(nested, nestedSet), Seq(Seq(0.0d), Seq(Double.NaN)))
+ checkEvaluation(ArrayIntersect(nested, nestedSet), Seq(Seq(0.0d), Seq(Double.NaN)))
+ checkEvaluation(ArrayExcept(nested, nestedSet), Seq.empty[Seq[Double]])
+ checkEvaluation(ArraysOverlap(nested, nestedSet), true)
+
+ val structType = new StructType().add("d", DoubleType, nullable = false)
+ val structs = Literal.create(
+ Seq(Row(-0.0d), Row(0.0d)), ArrayType(structType, containsNull = false))
+ checkEvaluation(ArrayDistinct(structs), Seq(Row(0.0d)))
+ }
+
test("SPARK-36740: ArrayMin/ArrayMax/SortArray should handle NaN greater than non-NaN value") {
// ArrayMin
checkEvaluation(ArrayMin(
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
index a0a9c8ec3224..6816d56e0b80 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/NormalizeFloatingPointNumbersSuite.scala
@@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.optimizer
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
-import org.apache.spark.sql.catalyst.expressions.{ArrayDistinct, ArrayExcept, ArrayIntersect, ArraysOverlap, ArrayTransform, ArrayUnion, CaseWhen, Expression, If, IsNull, KnownFloatingPointNormalized, LambdaFunction, NamedLambdaVariable}
+import org.apache.spark.sql.catalyst.expressions.{ArrayDistinct, ArrayExcept, ArrayIntersect, ArraysOverlap, ArrayUnion, CaseWhen, If, IsNull, KnownFloatingPointNormalized}
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.catalyst.rules.RuleExecutor
@@ -35,18 +35,6 @@ class NormalizeFloatingPointNumbersSuite extends PlanTest {
val a = testRelation1.output(0)
val testRelation2 = LocalRelation($"a".double)
val b = testRelation2.output(0)
- val arrayRelation = LocalRelation($"arr1".array(DoubleType), $"arr2".array(DoubleType))
- val arr1 = arrayRelation.output(0)
- val arr2 = arrayRelation.output(1)
-
- private def normalizedArray(e: Expression): KnownFloatingPointNormalized = {
- val lv = NamedLambdaVariable("arg", DoubleType, nullable = true)
- KnownFloatingPointNormalized(
- ArrayTransform(e,
- LambdaFunction(
- KnownFloatingPointNormalized(NormalizeNaNAndZero(lv)),
- Seq(lv))))
- }
test("normalize floating points in window function expressions") {
val query = testRelation1.window(Seq(sum(a).as("sum")), Seq(a), Seq(a.asc))
@@ -146,107 +134,17 @@ class NormalizeFloatingPointNumbersSuite extends PlanTest {
assert(nestedExpr.dataType == normalizedExpr.dataType)
}
- test("SPARK-54918: normalize floating points in array_distinct") {
- val query = arrayRelation.select(ArrayDistinct(arr1).as("result"))
-
- val optimized = Optimize.execute(query)
- val correctAnswer = arrayRelation.select(ArrayDistinct(normalizedArray(arr1)).as("result"))
-
- comparePlans(optimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_distinct - idempotence") {
- val query = arrayRelation.select(ArrayDistinct(arr1).as("result"))
-
- val optimized = Optimize.execute(query)
- val doubleOptimized = Optimize.execute(optimized)
- val correctAnswer = arrayRelation.select(ArrayDistinct(normalizedArray(arr1)).as("result"))
-
- comparePlans(doubleOptimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_union") {
- val query = arrayRelation.select(ArrayUnion(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val correctAnswer = arrayRelation.select(
- ArrayUnion(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(optimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_union - idempotence") {
- val query = arrayRelation.select(ArrayUnion(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val doubleOptimized = Optimize.execute(optimized)
- val correctAnswer = arrayRelation.select(
- ArrayUnion(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(doubleOptimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_intersect") {
- val query = arrayRelation.select(ArrayIntersect(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val correctAnswer = arrayRelation.select(
- ArrayIntersect(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(optimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_intersect - idempotence") {
- val query = arrayRelation.select(ArrayIntersect(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val doubleOptimized = Optimize.execute(optimized)
- val correctAnswer = arrayRelation.select(
- ArrayIntersect(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(doubleOptimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_except") {
- val query = arrayRelation.select(ArrayExcept(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val correctAnswer = arrayRelation.select(
- ArrayExcept(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(optimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in array_except - idempotence") {
- val query = arrayRelation.select(ArrayExcept(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val doubleOptimized = Optimize.execute(optimized)
- val correctAnswer = arrayRelation.select(
- ArrayExcept(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(doubleOptimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in arrays_overlap") {
- val query = arrayRelation.select(ArraysOverlap(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val correctAnswer = arrayRelation.select(
- ArraysOverlap(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
-
- comparePlans(optimized, correctAnswer)
- }
-
- test("SPARK-54918: normalize floating points in arrays_overlap - idempotence") {
- val query = arrayRelation.select(ArraysOverlap(arr1, arr2).as("result"))
-
- val optimized = Optimize.execute(query)
- val doubleOptimized = Optimize.execute(optimized)
- val correctAnswer = arrayRelation.select(
- ArraysOverlap(normalizedArray(arr1), normalizedArray(arr2)).as("result"))
+ test("SPARK-59602: array set operations are not normalized by a plan rewrite") {
+ val relation = LocalRelation($"a".array(DoubleType), $"b".array(DoubleType))
+ val Seq(array1, array2) = relation.output
+ val query = relation.select(
+ ArrayDistinct(array1).as("distinct"),
+ ArrayUnion(array1, array2).as("union"),
+ ArrayIntersect(array1, array2).as("intersect"),
+ ArrayExcept(array1, array2).as("except"),
+ ArraysOverlap(array1, array2).as("overlap"))
- comparePlans(doubleOptimized, correctAnswer)
+ comparePlans(SimpleTestOptimizer.FinishAnalysis(query), query)
+ comparePlans(Optimize.execute(query), query)
}
}
-
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
index c57b2e97a767..3dcdc73759b0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameFunctionsSuite.scala
@@ -6364,6 +6364,14 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
assert(r2.exists(isPositiveZero))
}
+ test("SPARK-59602: array_distinct normalizes nested floating-point values") {
+ val result = Seq(Seq(Seq(-0.0d), Seq(0.0d))).toDF("a")
+ .select(array_distinct($"a")).head().getSeq[Seq[Double]](0)
+
+ assert(result.length == 1)
+ assert(isPositiveZero(result.head.head))
+ }
+
test("SPARK-54918: array_distinct normalizes -0.0 to +0.0") {
val r1 = Seq(Array(-0.0d, 0.0d)).toDF("a")
.select(array_distinct($"a")).head().getSeq[Double](0)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org