You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/05/28 23:13:52 UTC
(datafusion-comet) branch main updated: chore(audit): audit json expressions across Spark 3.4.3, 3.5.8, 4.0.1, 4.1.1 (#4470)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 6b7d52142a chore(audit): audit json expressions across Spark 3.4.3, 3.5.8, 4.0.1, 4.1.1 (#4470)
6b7d52142a is described below
commit 6b7d52142a97ed35d5bed911663e1c64bbe7a33a
Author: Andy Grove <ag...@apache.org>
AuthorDate: Thu May 28 17:13:49 2026 -0600
chore(audit): audit json expressions across Spark 3.4.3, 3.5.8, 4.0.1, 4.1.1 (#4470)
---
docs/source/contributor-guide/spark_expressions_support.md | 5 +++++
spark/src/main/scala/org/apache/comet/serde/strings.scala | 11 +++++------
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/docs/source/contributor-guide/spark_expressions_support.md b/docs/source/contributor-guide/spark_expressions_support.md
index 6b3d122708..4c69dc49b2 100644
--- a/docs/source/contributor-guide/spark_expressions_support.md
+++ b/docs/source/contributor-guide/spark_expressions_support.md
@@ -511,6 +511,11 @@
- [ ] from_json
- [x] get_json_object
+ - Spark 3.4.3 (audited 2026-05-27): identical to 3.5.8.
+ - Spark 3.5.8 (audited 2026-05-27): baseline. `BinaryExpression with ExpectsInputTypes with CodegenFallback`; `inputTypes = Seq(StringType, StringType) -> StringType`. Eval is inline and uses Jackson with `RawStyle` output. Foldable paths are parsed once. Returns NULL for invalid JSON, missing paths, or `JsonProcessingException`.
+ - Spark 4.0.1 (audited 2026-05-27): the eval is extracted into a `GetJsonObjectEvaluator` helper (no behaviour change). The trait set now mixes in `DefaultStringProducingExpression`, and `inputTypes` is widened to `StringTypeWithCollation(supportsTrimCollation = true)` for both arguments.
+ - Spark 4.1.1 (audited 2026-05-27): identical to 4.0.1.
+ - Known incompatibility: Spark accepts single-quoted JSON and unescaped control characters; Comet's native parser (built on `serde_json`) rejects both, so those inputs require `spark.comet.expression.GetJsonObject.allowIncompatible=true` and may still produce different results. Non-default Spark 4.0 string collations are not propagated (https://github.com/apache/datafusion-comet/issues/2190).
- [ ] json_array_length
- [ ] json_object_keys
- [ ] json_tuple
diff --git a/spark/src/main/scala/org/apache/comet/serde/strings.scala b/spark/src/main/scala/org/apache/comet/serde/strings.scala
index efab2e848a..1d40d527a3 100644
--- a/spark/src/main/scala/org/apache/comet/serde/strings.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/strings.scala
@@ -443,15 +443,14 @@ object CometStringSplit extends CometExpressionSerde[StringSplit] {
object CometGetJsonObject extends CometExpressionSerde[GetJsonObject] {
- override def getIncompatibleReasons(): Seq[String] = Seq(
+ private val incompatReason =
"Spark allows single-quoted JSON and unescaped control characters which Comet does not" +
- " support")
+ " support"
+
+ override def getIncompatibleReasons(): Seq[String] = Seq(incompatReason)
override def getSupportLevel(expr: GetJsonObject): SupportLevel =
- Incompatible(
- Some(
- "Spark allows single-quoted JSON and unescaped control characters " +
- "which Comet does not support"))
+ Incompatible(Some(incompatReason))
override def convert(
expr: GetJsonObject,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org