You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ge...@apache.org on 2026/10/16 21:49:40 UTC
(spark) branch branch-4.x updated: [SPARK-59511][SQL] Reduce ANSI type coercion allocations
This is an automated email from the ASF dual-hosted git repository.
gengliangwang pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 93bd861ead4c [SPARK-59511][SQL] Reduce ANSI type coercion allocations
93bd861ead4c is described below
commit 93bd861ead4cb5211d6370cefbc729f9fbeeb3dc
Author: bhollis-dbx <be...@databricks.com>
AuthorDate: Fri Oct 16 14:49:21 2026 -0700
[SPARK-59511][SQL] Reduce ANSI type coercion allocations
### What changes were proposed in this pull request?
This pull request reduces temporary allocations in ANSI type coercion by matching `BinaryOperator` expressions without tuple-producing extractors, replacing an `Option.map` chain with direct matching, and reusing the two immutable decimal coercion rule chains selected by `spark.sql.legacy.literal.pickMinimumPrecision`.
It also adds a regression test confirming that combined ANSI coercion invokes every rule in order when earlier rules return the original expression.
### Why are the changes needed?
ANSI type coercion allocates temporary tuples, closures, and partial-function chains while traversing expressions that remain unchanged. Large expression trees amplify this allocation cost and increase analysis memory and latency.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added an ordering regression test and ran:
`build/sbt 'catalyst/testOnly *AnsiTypeCoercionSuite *DecimalPrecisionSuite'`
All 62 tests passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #58802 from bhollis-dbx/reduce-ansi-type-coercion-allocations.
Authored-by: bhollis-dbx <be...@databricks.com>
Signed-off-by: Gengliang Wang <ge...@apache.org>
(cherry picked from commit 6fa6a6ab78d50454d061fc3e91b7b78fa2947578)
Signed-off-by: Gengliang Wang <ge...@apache.org>
---
.../analysis/AnsiStringPromotionTypeCoercion.scala | 8 +++----
.../analysis/DecimalPrecisionTypeCoercion.scala | 22 ++++++++++++++----
.../sql/catalyst/analysis/TypeCoercionHelper.scala | 27 ++++++++++------------
.../catalyst/analysis/AnsiTypeCoercionSuite.scala | 20 ++++++++++++++++
4 files changed, 53 insertions(+), 24 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiStringPromotionTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiStringPromotionTypeCoercion.scala
index f6d7a9605831..5b27c327f977 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiStringPromotionTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiStringPromotionTypeCoercion.scala
@@ -54,10 +54,10 @@ import org.apache.spark.sql.types.{
*/
object AnsiStringPromotionTypeCoercion {
def apply(expression: Expression): Expression = expression match {
- case b @ BinaryOperator(left, right)
- if findWiderTypeForString(left.dataType, right.dataType).isDefined =>
- val promoteType = findWiderTypeForString(left.dataType, right.dataType).get
- b.withNewChildren(Seq(castExpr(left, promoteType), castExpr(right, promoteType)))
+ case b: BinaryOperator
+ if findWiderTypeForString(b.left.dataType, b.right.dataType).isDefined =>
+ val promoteType = findWiderTypeForString(b.left.dataType, b.right.dataType).get
+ b.withNewChildren(Seq(castExpr(b.left, promoteType), castExpr(b.right, promoteType)))
case Abs(e @ StringTypeExpression(), failOnError) => Abs(Cast(e, DoubleType), failOnError)
case m @ UnaryMinus(e @ StringTypeExpression(), _) =>
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionTypeCoercion.scala
index 64f4c2b03c92..eaf0bd4fa6e7 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/DecimalPrecisionTypeCoercion.scala
@@ -31,6 +31,8 @@ import org.apache.spark.sql.types.{DataType, DecimalExpression, DecimalType, Dou
object DecimalPrecisionTypeCoercion extends SQLConfHelper {
import scala.math.max
+ private val identity: Expression => Expression = expression => expression
+
/**
* Strength reduction for comparing integral expressions with decimal literals.
* 1. int_col > decimal_literal => int_col > floor(decimal_literal)
@@ -125,16 +127,18 @@ object DecimalPrecisionTypeCoercion extends SQLConfHelper {
}
def apply(expression: Expression): Expression = {
- decimalAndDecimal()
- .orElse(integralAndDecimalLiteral)
- .orElse(nondecimalAndDecimal(conf.literalPickMinimumPrecision))
- .lift(expression).getOrElse(expression)
+ val transform = if (conf.literalPickMinimumPrecision) {
+ withLiteralMinimumPrecision
+ } else {
+ withoutLiteralMinimumPrecision
+ }
+ transform.applyOrElse(expression, identity)
}
private def isFloat(t: DataType): Boolean = t == FloatType || t == DoubleType
/** Decimal precision promotion for binary comparison. */
- private def decimalAndDecimal(): PartialFunction[Expression, Expression] = {
+ private val decimalAndDecimal: PartialFunction[Expression, Expression] = {
case b @ BinaryComparison(e1 @ DecimalExpression(p1, s1), e2 @ DecimalExpression(p2, s2))
if p1 != p2 || s1 != s2 =>
val resultType = widerDecimalType(p1, s1, p2, s2)
@@ -187,6 +191,14 @@ object DecimalPrecisionTypeCoercion extends SQLConfHelper {
}
}
+ private val withLiteralMinimumPrecision = decimalAndDecimal
+ .orElse(integralAndDecimalLiteral)
+ .orElse(nondecimalAndDecimal(literalPickMinimumPrecision = true))
+
+ private val withoutLiteralMinimumPrecision = decimalAndDecimal
+ .orElse(integralAndDecimalLiteral)
+ .orElse(nondecimalAndDecimal(literalPickMinimumPrecision = false))
+
// Returns the wider decimal type that's wider than both of them
def widerDecimalType(d1: DecimalType, d2: DecimalType): DecimalType = {
widerDecimalType(d1.precision, d1.scale, d2.precision, d2.scale)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionHelper.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionHelper.scala
index f76433684c40..f67aa877c850 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionHelper.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionHelper.scala
@@ -696,21 +696,18 @@ abstract class TypeCoercionHelper {
*/
object ImplicitTypeCoercion {
def apply(expression: Expression): Expression = expression match {
- case b @ BinaryOperator(left, right)
- if canHandleTypeCoercion(left.dataType, right.dataType) =>
- findTightestCommonType(left.dataType, right.dataType)
- .map { commonType =>
- if (b.inputType.acceptsType(commonType)) {
- // If the expression accepts the tightest common type, cast to that.
- val newLeft = if (left.dataType == commonType) left else Cast(left, commonType)
- val newRight = if (right.dataType == commonType) right else Cast(right, commonType)
- b.withNewChildren(Seq(newLeft, newRight))
- } else {
- // Otherwise, don't do anything with the expression.
- b
- }
- }
- .getOrElse(b) // If there is no applicable conversion, leave expression unchanged.
+ case b: BinaryOperator if canHandleTypeCoercion(b.left.dataType, b.right.dataType) =>
+ findTightestCommonType(b.left.dataType, b.right.dataType) match {
+ case Some(commonType) if b.inputType.acceptsType(commonType) =>
+ // If the expression accepts the tightest common type, cast to that.
+ val newLeft = if (b.left.dataType == commonType) b.left else Cast(b.left, commonType)
+ val newRight =
+ if (b.right.dataType == commonType) b.right else Cast(b.right, commonType)
+ b.withNewChildren(Seq(newLeft, newRight))
+ case _ =>
+ // If there is no applicable conversion, leave the expression unchanged.
+ b
+ }
case t: TimeBucket =>
val children = t.children.zip(t.inputTypes).zipWithIndex.map {
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala
index 107ca4300e8c..d74db5d205ad 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercionSuite.scala
@@ -19,6 +19,8 @@ package org.apache.spark.sql.catalyst.analysis
import java.sql.Timestamp
+import scala.collection.mutable.ArrayBuffer
+
import org.apache.spark.sql.catalyst.analysis.AnsiTypeCoercion._
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.dsl.plans._
@@ -31,6 +33,24 @@ import org.apache.spark.sql.types._
class AnsiTypeCoercionSuite extends TypeCoercionSuiteBase {
import TypeCoercionSuite._
+ test("combined type coercion invokes every rule in order") {
+ val invocations = ArrayBuffer.empty[Int]
+ val rules = (1 to 3).map { index =>
+ new TypeCoercionRule {
+ override val transform: PartialFunction[Expression, Expression] = {
+ case expression =>
+ invocations += index
+ expression
+ }
+ }
+ }
+ val input = Literal(1)
+
+ new AnsiCombinedTypeCoercionRule(rules).transform.applyOrElse(input, identity[Expression])
+
+ assert(invocations.toSeq == rules.indices.map(_ + 1))
+ }
+
// scalastyle:off line.size.limit
// The following table shows all implicit data type conversions that are not visible to the user.
// +----------------------+----------+-----------+-------------+----------+------------+------------+------------+------------+-------------+------------+----------+---------------+------------+----------+-------------+----------+----------------------+---------------------+-------------+--------------+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org