You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/08/08 11:11:03 UTC
(spark) branch branch-4.3 updated: [SPARK-58643][SQL] Require a constant `create_if_missing` in `variant_set/try_variant_set`
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new b5a69a5180dc [SPARK-58643][SQL] Require a constant `create_if_missing` in `variant_set/try_variant_set`
b5a69a5180dc is described below
commit b5a69a5180dc63cfdd75534ea12ec5f3f8619184
Author: bojana-db <bo...@databricks.com>
AuthorDate: Sat Aug 8 11:10:34 2026 +0000
[SPARK-58643][SQL] Require a constant `create_if_missing` in `variant_set/try_variant_set`
### What changes were proposed in this pull request?
Make the optional `create_if_missing` argument of `variant_set` (and `try_variant_set`) a constant-only argument. `VariantSet.checkInputDataTypes` now rejects a non-foldable `create_if_missing` with `DATATYPE_MISMATCH.NON_FOLDABLE_INPUT`.
### Why are the changes needed?
`create_if_missing` is a mode flag that controls the operation's behavior, not per-row data, so it should be a constant.
### Does this PR introduce _any_ user-facing change?
Yes, within the unreleased `master`/`branch-4.x` line only (`variant_set`/`try_variant_set` are new in 4.3.0 and not yet released; there is no change relative to any released Spark version).
### How was this patch tested?
Unit tests.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code with Claude Opus 4.8
Closes #57855 from bojana-db/reject-non-foldable-flag-variant-set.
Authored-by: bojana-db <bo...@databricks.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
(cherry picked from commit bcd4120fa83e37eff77bd677672e62ca7eaf0b2f)
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../expressions/variant/variantExpressions.scala | 11 +++++++--
.../variant/VariantExpressionSuite.scala | 26 +++++++++++++++++-----
.../scala/org/apache/spark/sql/VariantSuite.scala | 5 +++++
3 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index 61a67bf92648..723af5a4fe92 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -1084,6 +1084,13 @@ case class VariantSet(
val result = super.checkInputDataTypes()
if (result.isFailure) {
result
+ } else if (!createIfMissing.foldable) {
+ DataTypeMismatch(
+ errorSubClass = "NON_FOLDABLE_INPUT",
+ messageParameters = Map(
+ "inputName" -> toSQLId("create_if_missing"),
+ "inputType" -> toSQLType(createIfMissing.dataType),
+ "inputExpr" -> toSQLExpr(createIfMissing)))
} else if (value.dataType == NullType) {
TypeCheckResult.TypeCheckSuccess
} else if (!VariantGet.checkDataType(value.dataType, allowStructsAndMaps = false)) {
@@ -1219,7 +1226,7 @@ abstract class VariantSetExpressionBuilderBase(failOnError: Boolean) extends Exp
path should start with `$` and is followed by one or more segments like `[123]`,
`.name`, `['name']`, or `["name"]`. The root path `$` is not allowed.
* val - Any expression castable to variant.
- * create_if_missing - An optional boolean (default true).
+ * create_if_missing - An optional boolean (default true). Must be a constant.
""",
examples = """
Examples:
@@ -1260,7 +1267,7 @@ object VariantSetExpressionBuilder extends VariantSetExpressionBuilderBase(true)
path should start with `$` and is followed by one or more segments like `[123]`,
`.name`, `['name']`, or `["name"]`. The root path `$` is not allowed.
* val - Any expression castable to variant.
- * create_if_missing - An optional boolean (default true).
+ * create_if_missing - An optional boolean (default true). Must be a constant.
""",
examples = """
Examples:
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index 83a0aaa2e6a6..b3248609ae3e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -1637,20 +1637,34 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
checkSet("""{"a": 1}""", "$.a", Literal.create(null, NullType), null)
Seq(true, false).foreach { failOnError =>
- val dynamic = VariantSet(
+ val dynamicCreate = VariantSet(
Literal(parseJson("""{"a": 1}""")),
BoundReference(0, StringType, nullable = true),
Literal(2),
- BoundReference(1, BooleanType, nullable = true),
+ Literal(true),
failOnError)
checkEvaluation(
- ResolveTimeZone.resolveTimeZones(Cast(dynamic, StringType)),
+ ResolveTimeZone.resolveTimeZones(Cast(dynamicCreate, StringType)),
"""{"a":1,"b":2}""",
- InternalRow(UTF8String.fromString("$.b"), true))
+ InternalRow(UTF8String.fromString("$.b")))
+ val dynamicNoCreate = VariantSet(
+ Literal(parseJson("""{"a": 1}""")),
+ BoundReference(0, StringType, nullable = true),
+ Literal(2),
+ Literal(false),
+ failOnError)
checkEvaluation(
- ResolveTimeZone.resolveTimeZones(Cast(dynamic, StringType)),
+ ResolveTimeZone.resolveTimeZones(Cast(dynamicNoCreate, StringType)),
"""{"a":1}""",
- InternalRow(UTF8String.fromString("$.b"), false))
+ InternalRow(UTF8String.fromString("$.b")))
+ }
+
+ // create_if_missing must be a constant, for both variant_set and try_variant_set.
+ Seq(true, false).foreach { failOnError =>
+ assert(VariantSet(
+ Literal(parseJson("""{"a": 1}""")), Literal("$.a"), Literal(2),
+ BoundReference(0, BooleanType, nullable = true), failOnError)
+ .checkInputDataTypes().isFailure)
}
// Recoverable errors: `variant_set` throws; `try_variant_set` returns NULL. Every shape of
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
index 864d0c9b631b..0c44468b2d89 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/VariantSuite.scala
@@ -486,6 +486,11 @@ class VariantSuite extends SharedSparkSession with ExpressionEvalHelper {
assert(intercept[AnalysisException] {
sql("SELECT variant_set(parse_json('{}'), '$.a', named_struct('x', 1))")
}.getCondition == "DATATYPE_MISMATCH.CAST_WITHOUT_SUGGESTION")
+
+ // A non-constant create_if_missing is rejected at analysis.
+ assert(intercept[AnalysisException] {
+ sql("SELECT variant_set(parse_json('{\"a\": 1}'), '$.a', 2, c) FROM VALUES (true) AS t(c)")
+ }.getCondition == "DATATYPE_MISMATCH.NON_FOLDABLE_INPUT")
}
test("variant_set with dynamic arguments") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org