You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gluten.apache.org by ch...@apache.org on 2026/04/15 11:17:47 UTC
(gluten) 24/29: [GLUTEN-10134][VL] Implement Cast three-state EvalMode and remove isAnsiSupported whitelist
This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch feature/ansi-ci-baseline
in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 1314fbf4a3499b41e3775cb0bb74d5544c12068b
Author: Chang chen <ch...@microsoft.com>
AuthorDate: Mon Apr 13 15:20:38 2026 +0000
[GLUTEN-10134][VL] Implement Cast three-state EvalMode and remove isAnsiSupported whitelist
Replace boolean isTryCast with int evalMode (0=LEGACY, 1=TRY, 2=ANSI) in
CastNode/ExpressionBuilder, and add Cast to withAnsiEvalMode() in Spark34/35
shims. This enables the full three-state Cast semantics end-to-end:
- LEGACY (evalMode=0): kCast path, isTryCast=true, allowOverflow=true (truncate)
- ANSI (evalMode=2): kCast path, isTryCast=false, allowOverflow=false (throw)
- TRY (evalMode=1): kTryCast path, isTryCast=true, allowOverflow=false (null)
The Velox C++ side (SparkCastExpr.cpp) removes the isAnsiSupported() whitelist
so all cast types now respect ANSI mode uniformly. C++ changes are in the Velox
subproject and managed separately.
Co-Authored-By: Claude Opus 4 <no...@anthropic.com>
---
.../backendsapi/velox/VeloxTransformerApi.scala | 2 +-
.../execution/HashAggregateExecTransformer.scala | 4 ++--
.../apache/gluten/substrait/expression/CastNode.java | 18 ++++++++----------
.../substrait/expression/ExpressionBuilder.java | 5 ++---
.../expression/UnaryExpressionTransformer.scala | 20 ++++++++++++--------
.../gluten/sql/shims/spark34/Spark34Shims.scala | 1 +
.../gluten/sql/shims/spark35/Spark35Shims.scala | 1 +
7 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
index 3a1d53154f..42f50b3064 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxTransformerApi.scala
@@ -105,7 +105,7 @@ class VeloxTransformerApi extends TransformerApi with Logging {
childNode
} else {
val typeNode = ConverterUtils.getTypeNode(dataType, nullable)
- ExpressionBuilder.makeCast(typeNode, childNode, nullOnOverflow)
+ ExpressionBuilder.makeCast(typeNode, childNode, if (nullOnOverflow) 1 else 0)
}
}
diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala
index a0c52c7909..d252ab8bb8 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/execution/HashAggregateExecTransformer.scala
@@ -136,7 +136,7 @@ abstract class HashAggregateExecTransformer(
.makeCast(
ConverterUtils.getTypeNode(sparkType, nullable = false),
ExpressionBuilder.makeSelection(colIdx, adjustedOrders(idx)),
- SQLConf.get.ansiEnabled))
+ if (SQLConf.get.ansiEnabled) 2 else 0))
} else {
// Velox and Spark have the same type
expressionNodes.add(ExpressionBuilder.makeSelection(colIdx, adjustedOrders(idx)))
@@ -337,7 +337,7 @@ abstract class HashAggregateExecTransformer(
ExpressionBuilder.makeCast(
ConverterUtils.getTypeNode(veloxType, attr.nullable),
aggFuncInputAttrNode,
- SQLConf.get.ansiEnabled)
+ if (SQLConf.get.ansiEnabled) 2 else 0)
} else {
newInputAttributes += attr
aggFuncInputAttrNode
diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
index 1984c44d74..037777ce9c 100644
--- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
+++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/CastNode.java
@@ -26,12 +26,16 @@ public class CastNode implements ExpressionNode, Serializable {
private final TypeNode typeNode;
private final ExpressionNode expressionNode;
- public final boolean isTryCast;
+ // Maps directly to Substrait FailureBehavior:
+ // 0 = UNSPECIFIED (LEGACY: truncate on overflow)
+ // 1 = RETURN_NULL (TRY: return null on failure)
+ // 2 = THROW_EXCEPTION (ANSI: throw on failure)
+ public final int evalMode;
- CastNode(TypeNode typeNode, ExpressionNode expressionNode, boolean isTryCast) {
+ CastNode(TypeNode typeNode, ExpressionNode expressionNode, int evalMode) {
this.typeNode = typeNode;
this.expressionNode = expressionNode;
- this.isTryCast = isTryCast;
+ this.evalMode = evalMode;
}
@Override
@@ -39,13 +43,7 @@ public class CastNode implements ExpressionNode, Serializable {
Expression.Cast.Builder castBuilder = Expression.Cast.newBuilder();
castBuilder.setType(typeNode.toProtobuf());
castBuilder.setInput(expressionNode.toProtobuf());
- if (!isTryCast) {
- // Throw exception on failure.
- castBuilder.setFailureBehaviorValue(2);
- } else {
- // Return null on failure.
- castBuilder.setFailureBehaviorValue(1);
- }
+ castBuilder.setFailureBehaviorValue(evalMode);
Expression.Builder builder = Expression.newBuilder();
builder.setCast(castBuilder.build());
return builder.build();
diff --git a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
index 4bdef37878..2ee83ac53f 100644
--- a/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
+++ b/gluten-substrait/src/main/java/org/apache/gluten/substrait/expression/ExpressionBuilder.java
@@ -237,9 +237,8 @@ public class ExpressionBuilder {
return new AggregateFunctionNode(functionId, expressionNodes, phase, outputTypeNode);
}
- public static CastNode makeCast(
- TypeNode typeNode, ExpressionNode expressionNode, boolean isTryCast) {
- return new CastNode(typeNode, expressionNode, isTryCast);
+ public static CastNode makeCast(TypeNode typeNode, ExpressionNode expressionNode, int evalMode) {
+ return new CastNode(typeNode, expressionNode, evalMode);
}
public static StringMapNode makeStringMap(Map<String, String> values) {
diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
index d8404cd84d..ecb96afc53 100644
--- a/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
+++ b/gluten-substrait/src/main/scala/org/apache/gluten/expression/UnaryExpressionTransformer.scala
@@ -51,23 +51,27 @@ case class CastTransformer(substraitExprName: String, child: ExpressionTransform
buildCastWithWriteSideCheck(context, ExpressionNames.VARCHAR_TYPE_WRITE_SIDE_CHECK, length)
case _ =>
val typeNode = ConverterUtils.getTypeNode(dataType, original.nullable)
- ExpressionBuilder.makeCast(
- typeNode,
- child.doTransform(context),
- !SparkShimLoader.getSparkShims.withAnsiEvalMode(original))
+ ExpressionBuilder.makeCast(typeNode, child.doTransform(context), castEvalMode)
}
}
+ // Maps Spark Cast EvalMode to Substrait FailureBehavior:
+ // LEGACY=0(UNSPECIFIED), TRY=1(RETURN_NULL), ANSI=2(THROW_EXCEPTION)
+ private val castEvalMode: Int = {
+ val shims = SparkShimLoader.getSparkShims
+ if (shims.withAnsiEvalMode(original)) 2
+ else if (shims.withTryEvalMode(original)) 1
+ else 0
+ }
+
/** Builds Cast(child, StringType) + write-side check for CharType/VarcharType. */
private def buildCastWithWriteSideCheck(
context: SubstraitContext,
checkFuncName: String,
length: Int): ExpressionNode = {
val stringTypeNode = ConverterUtils.getTypeNode(StringType, original.nullable)
- val castNode = ExpressionBuilder.makeCast(
- stringTypeNode,
- child.doTransform(context),
- !SparkShimLoader.getSparkShims.withAnsiEvalMode(original))
+ val castNode =
+ ExpressionBuilder.makeCast(stringTypeNode, child.doTransform(context), castEvalMode)
val funcName =
ConverterUtils.makeFuncName(checkFuncName, Seq(StringType, IntegerType))
val functionId = context.registerFunction(funcName)
diff --git a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
index 61bc3bc945..9033738764 100644
--- a/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
+++ b/shims/spark34/src/main/scala/org/apache/gluten/sql/shims/spark34/Spark34Shims.scala
@@ -445,6 +445,7 @@ class Spark34Shims extends SparkShims {
case s: Subtract => s.evalMode == EvalMode.ANSI
case d: Divide => d.evalMode == EvalMode.ANSI
case m: Multiply => m.evalMode == EvalMode.ANSI
+ case c: Cast => c.evalMode == EvalMode.ANSI
case i: IntegralDivide => i.evalMode == EvalMode.ANSI
case _ => false
}
diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
index 7e31af9b67..181aad9919 100644
--- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
+++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/Spark35Shims.scala
@@ -482,6 +482,7 @@ class Spark35Shims extends SparkShims {
case s: Subtract => s.evalMode == EvalMode.ANSI
case d: Divide => d.evalMode == EvalMode.ANSI
case m: Multiply => m.evalMode == EvalMode.ANSI
+ case c: Cast => c.evalMode == EvalMode.ANSI
case i: IntegralDivide => i.evalMode == EvalMode.ANSI
case _ => false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@gluten.apache.org
For additional commands, e-mail: commits-help@gluten.apache.org