You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2025/02/18 20:55:43 UTC
(spark) branch branch-4.0 updated: [SPARK-50767][SQL] Remove codegen of `from_json`
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 3000f94e8b4b [SPARK-50767][SQL] Remove codegen of `from_json`
3000f94e8b4b is described below
commit 3000f94e8b4b911aaae8a0f937b657e8df01476a
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Tue Feb 18 21:55:23 2025 +0100
[SPARK-50767][SQL] Remove codegen of `from_json`
### What changes were proposed in this pull request?
This reopens https://github.com/apache/spark/pull/49411 to fix the performance regression in 4.0.
### Why are the changes needed?
It's non-trivial to support CSE for Filter in whole stage codegen. We should not rush but revert the codegen support in 4.0 so that we have more time to get it right in 4.1.
Note: 4.0 also adds codegen support for a few other expressions, but `from_json` is special as it's quite expensive and the performance regression is very significant with it.
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
N/A
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49992 from cloud-fan/json.
Authored-by: Wenchen Fan <we...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
(cherry picked from commit a8b694f4d3dcac684290282c40f52f947b50942b)
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../sql/catalyst/expressions/jsonExpressions.scala | 22 +++-------------------
.../SubExprEliminationBenchmark-jdk21-results.txt | 20 ++++++++++----------
.../SubExprEliminationBenchmark-results.txt | 20 ++++++++++----------
3 files changed, 23 insertions(+), 39 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index 5e6da7ac4125..d2ed7054e0d4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
-import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode}
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, CodegenFallback, ExprCode}
import org.apache.spark.sql.catalyst.expressions.codegen.Block.BlockHelper
import org.apache.spark.sql.catalyst.expressions.json.{GetJsonObjectEvaluator, JsonExpressionUtils, JsonToStructsEvaluator, JsonTupleEvaluator, SchemaOfJsonEvaluator, StructsToJsonEvaluator}
import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, StaticInvoke}
@@ -257,6 +257,7 @@ case class JsonToStructs(
variantAllowDuplicateKeys: Boolean = SQLConf.get.getConf(SQLConf.VARIANT_ALLOW_DUPLICATE_KEYS))
extends UnaryExpression
with TimeZoneAwareExpression
+ with CodegenFallback
with ExpectsInputTypes
with QueryErrorsBase {
@@ -304,7 +305,7 @@ case class JsonToStructs(
copy(timeZoneId = Option(timeZoneId))
@transient
- private val nameOfCorruptRecord = SQLConf.get.getConf(SQLConf.COLUMN_NAME_OF_CORRUPT_RECORD)
+ private lazy val nameOfCorruptRecord = SQLConf.get.getConf(SQLConf.COLUMN_NAME_OF_CORRUPT_RECORD)
@transient
private lazy val evaluator = new JsonToStructsEvaluator(
@@ -312,23 +313,6 @@ case class JsonToStructs(
override def nullSafeEval(json: Any): Any = evaluator.evaluate(json.asInstanceOf[UTF8String])
- override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
- val refEvaluator = ctx.addReferenceObj("evaluator", evaluator)
- val eval = child.genCode(ctx)
- val resultType = CodeGenerator.boxedType(dataType)
- val resultTerm = ctx.freshName("result")
- ev.copy(code =
- code"""
- |${eval.code}
- |$resultType $resultTerm = ($resultType) $refEvaluator.evaluate(${eval.value});
- |boolean ${ev.isNull} = $resultTerm == null;
- |${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
- |if (!${ev.isNull}) {
- | ${ev.value} = $resultTerm;
- |}
- |""".stripMargin)
- }
-
override def inputTypes: Seq[AbstractDataType] =
StringTypeWithCollation(supportsTrimCollation = true) :: Nil
diff --git a/sql/core/benchmarks/SubExprEliminationBenchmark-jdk21-results.txt b/sql/core/benchmarks/SubExprEliminationBenchmark-jdk21-results.txt
index be430feb9780..8cf0ba7224ba 100644
--- a/sql/core/benchmarks/SubExprEliminationBenchmark-jdk21-results.txt
+++ b/sql/core/benchmarks/SubExprEliminationBenchmark-jdk21-results.txt
@@ -3,23 +3,23 @@ Benchmark for performance of subexpression elimination
================================================================================================
Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure
+OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
AMD EPYC 7763 64-Core Processor
from_json as subExpr in Project: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-subExprElimination false, codegen: true 6313 6431 120 0.0 63134831.3 1.0X
-subExprElimination false, codegen: false 6093 6348 288 0.0 60930747.6 1.0X
-subExprElimination true, codegen: true 1387 1425 33 0.0 13872525.5 4.6X
-subExprElimination true, codegen: false 1218 1332 99 0.0 12182992.7 5.2X
+subExprElimination false, codegen: true 6700 7047 301 0.0 67001649.1 1.0X
+subExprElimination false, codegen: false 6719 6837 118 0.0 67191470.6 1.0X
+subExprElimination true, codegen: true 1350 1489 122 0.0 13503842.8 5.0X
+subExprElimination true, codegen: false 1366 1444 96 0.0 13658823.9 4.9X
Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 21.0.4+7-LTS on Linux 6.5.0-1025-azure
+OpenJDK 64-Bit Server VM 21.0.6+7-LTS on Linux 6.8.0-1021-azure
AMD EPYC 7763 64-Core Processor
from_json as subExpr in Filter: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-subExprElimination false, codegen: true 6610 6705 85 0.0 66104698.4 1.0X
-subExprElimination false, codegen: false 6647 6730 76 0.0 66469463.5 1.0X
-subExprElimination true, codegen: true 2077 2126 43 0.0 20769220.1 3.2X
-subExprElimination true, codegen: false 1949 2000 64 0.0 19489004.0 3.4X
+subExprElimination false, codegen: true 7250 7520 384 0.0 72501549.6 1.0X
+subExprElimination false, codegen: false 7255 7366 114 0.0 72554716.3 1.0X
+subExprElimination true, codegen: true 1934 2024 79 0.0 19344228.2 3.7X
+subExprElimination true, codegen: false 1981 2015 29 0.0 19814306.4 3.7X
diff --git a/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt b/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt
index 12d602bec17c..b689fef720f7 100644
--- a/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt
+++ b/sql/core/benchmarks/SubExprEliminationBenchmark-results.txt
@@ -3,23 +3,23 @@ Benchmark for performance of subexpression elimination
================================================================================================
Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure
+OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
AMD EPYC 7763 64-Core Processor
from_json as subExpr in Project: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-subExprElimination false, codegen: true 6438 6551 98 0.0 64378783.5 1.0X
-subExprElimination false, codegen: false 6216 6320 175 0.0 62161826.1 1.0X
-subExprElimination true, codegen: true 1480 1518 39 0.0 14799890.8 4.3X
-subExprElimination true, codegen: false 1321 1429 94 0.0 13212919.6 4.9X
+subExprElimination false, codegen: true 6389 6498 173 0.0 63887225.6 1.0X
+subExprElimination false, codegen: false 6235 6292 81 0.0 62351284.2 1.0X
+subExprElimination true, codegen: true 1328 1368 47 0.0 13284825.9 4.8X
+subExprElimination true, codegen: false 1323 1368 73 0.0 13227629.0 4.8X
Preparing data for benchmarking ...
-OpenJDK 64-Bit Server VM 17.0.12+7-LTS on Linux 6.5.0-1025-azure
+OpenJDK 64-Bit Server VM 17.0.14+7-LTS on Linux 6.8.0-1021-azure
AMD EPYC 7763 64-Core Processor
from_json as subExpr in Filter: Best Time(ms) Avg Time(ms) Stdev(ms) Rate(M/s) Per Row(ns) Relative
------------------------------------------------------------------------------------------------------------------------
-subExprElimination false, codegen: true 7107 7310 207 0.0 71066752.8 1.0X
-subExprElimination false, codegen: false 6738 6781 41 0.0 67375897.0 1.1X
-subExprElimination true, codegen: true 2052 2110 51 0.0 20519152.3 3.5X
-subExprElimination true, codegen: false 2053 2079 33 0.0 20526629.8 3.5X
+subExprElimination false, codegen: true 7081 7177 86 0.0 70813603.9 1.0X
+subExprElimination false, codegen: false 6586 6720 139 0.0 65859888.8 1.1X
+subExprElimination true, codegen: true 1729 1827 117 0.0 17291697.7 4.1X
+subExprElimination true, codegen: false 1726 1789 57 0.0 17255779.7 4.1X
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org