You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2026/08/12 22:42:05 UTC
(spark) branch branch-4.x updated: [SPARK-58672][SQL] Reject CHAR/VARCHAR map keys in variant conversions
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 15cf04b7a912 [SPARK-58672][SQL] Reject CHAR/VARCHAR map keys in variant conversions
15cf04b7a912 is described below
commit 15cf04b7a912f8bd0889c6b08fa7bb59065682c8
Author: Takuya UESHIN <ue...@happy-camper.st>
AuthorDate: Wed Aug 12 15:41:45 2026 -0700
[SPARK-58672][SQL] Reject CHAR/VARCHAR map keys in variant conversions
### What changes were proposed in this pull request?
Update `VariantGet.checkDataType` to reject map types whose keys are `CHAR` or `VARCHAR`. This validation is shared by `variant_get` and `to_variant_object`.
Add tests covering `STRING`, `CHAR`, and `VARCHAR` as top-level target types and as array elements, map keys/values, and struct fields. The tests cover both `variant_get` target types and `to_variant_object` input types where applicable.
### Why are the changes needed?
`CHAR` and `VARCHAR` extend `StringType`. As a result, the existing `MapType(_: StringType, ...)` pattern incorrectly accepts them as map key types, even though `VariantGet.checkDataType` explicitly rejects `CHAR` and `VARCHAR` in other positions. Since this validation is also used by `to_variant_object`, the same issue affects both variant conversion paths.
### Does this PR introduce _any_ user-facing change?
Yes. `variant_get` and `to_variant_object` now reject maps with `CHAR` or `VARCHAR` key types during analysis. Maps with `STRING` keys remain supported. `CHAR` and `VARCHAR` in array elements, map values, and struct fields were already rejected; the tests now cover these cases explicitly.
### How was this patch tested?
Added unit tests covering positive `STRING` cases and negative `CHAR`/`VARCHAR` cases for `variant_get` and `to_variant_object`, including array elements, map keys/values, and struct fields.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #57878 from ueshin/variant.
Authored-by: Takuya UESHIN <ue...@happy-camper.st>
Signed-off-by: Takuya Ueshin <ue...@happy-camper.st>
(cherry picked from commit 1735a6b23e0fa9ac7fcdbb24999e56c7a68b440c)
Signed-off-by: Takuya Ueshin <ue...@happy-camper.st>
---
.../expressions/variant/variantExpressions.scala | 1 +
.../variant/VariantExpressionSuite.scala | 49 ++++++++++++++++++++++
2 files changed, 50 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
index 79ee3cc4b3eb..fddb88d95395 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/variant/variantExpressions.scala
@@ -526,6 +526,7 @@ case object VariantGet {
VariantType =>
true
case ArrayType(elementType, _) => checkDataType(elementType, allowStructsAndMaps)
+ case MapType(_: CharType | _: VarcharType, _, _) => false
case MapType(_: StringType, valueType, _) if allowStructsAndMaps =>
checkDataType(valueType, allowStructsAndMaps)
case StructType(fields) if allowStructsAndMaps =>
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
index 5b2914c5408c..c69ad520a241 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/variant/VariantExpressionSuite.scala
@@ -708,6 +708,31 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
checkInvalidPath("$[\"\\\"\"]")
}
+ test("SPARK-58672: validate char/varchar target types in variant_get") {
+ def check(dataType: DataType, expected: Boolean): Unit = {
+ assert(
+ variantGet("""{"a": 1}""", "$", dataType)
+ .checkInputDataTypes().isSuccess == expected)
+ }
+
+ def targetTypes(stringType: StringType): Seq[DataType] = Seq(
+ stringType,
+ ArrayType(stringType),
+ MapType(stringType, IntegerType),
+ MapType(StringType, stringType),
+ StructType(Seq(StructField("v", stringType))))
+
+ targetTypes(StringType).foreach { dataType =>
+ check(dataType, expected = true)
+ }
+
+ Seq(CharType(10), VarcharType(10)).foreach { stringType =>
+ targetTypes(stringType).foreach { dataType =>
+ check(dataType, expected = false)
+ }
+ }
+ }
+
test("cast from variant") {
// We do not test too many type combinations, as the cast implementation is mostly the same as
// variant_get.
@@ -986,6 +1011,30 @@ class VariantExpressionSuite extends SparkFunSuite with ExpressionEvalHelper {
checkFailure(Map(1 -> 1), toVariantObject = true)
}
+ test("SPARK-58672: validate char/varchar input types in to_variant_object") {
+ def check(dataType: DataType, expected: Boolean): Unit = {
+ assert(
+ ToVariantObject(Literal.create(null, dataType))
+ .checkInputDataTypes().isSuccess == expected)
+ }
+
+ def nestedTypes(stringType: StringType): Seq[DataType] = Seq(
+ ArrayType(stringType),
+ MapType(stringType, IntegerType),
+ MapType(StringType, stringType),
+ StructType(Seq(StructField("v", stringType))))
+
+ nestedTypes(StringType).foreach { dataType =>
+ check(dataType, expected = true)
+ }
+
+ Seq(CharType(10), VarcharType(10)).foreach { stringType =>
+ nestedTypes(stringType).foreach { dataType =>
+ check(dataType, expected = false)
+ }
+ }
+ }
+
test("variant_from_arrays and variant_from_entries") {
def keysValues(keys: Any, values: Any, valueType: DataType): VariantFromArrays =
VariantFromArrays(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org