You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/08/24 06:36:37 UTC
(spark) branch master updated: [SPARK-58816][SQL] Fix INSERT with column list to resolve structs inside arrays and maps positionally
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fa3ad86e6d4e [SPARK-58816][SQL] Fix INSERT with column list to resolve structs inside arrays and maps positionally
fa3ad86e6d4e is described below
commit fa3ad86e6d4eeb6289d9b70cd02dc06c593c15e9
Author: Anshul Baliga <ba...@gmail.com>
AuthorDate: Mon Aug 24 14:36:16 2026 +0800
[SPARK-58816][SQL] Fix INSERT with column list to resolve structs inside arrays and maps positionally
### What changes were proposed in this pull request?
Fix `ResolveInsertionBase` to resolve struct fields positionally at every nesting level during `INSERT INTO table (column_list)`, including structs inside `ArrayType` and `MapType`.
Fixes https://issues.apache.org/jira/browse/SPARK-58816
### Why are the changes needed?
`createProjectForByNameQuery` only dispatched to the rename path when the top-level column was `StructType`, and `renameFieldsInStruct` only recursed into nested `StructType` fields. Structs inside arrays and maps were left
unrenamed, causing `TableOutputResolver` to resolve them by name instead of position — silently writing values into wrong fields when source and target field order differs.
### Does this PR introduce any user-facing change?
Yes. `INSERT INTO table (column_list)` now consistently resolves struct fields positionally at all nesting levels (direct struct, array-of-struct, map-of-struct), fixing silent data corruption when source and target field order differs.
### How was this patch tested?
Added three regression tests in `SQLInsertTestSuite` covering:
- Array-of-struct positional resolution
- Map-of-struct positional resolution
- Combined direct struct + array-of-struct + map-of-struct (exact JIRA repro)
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #58086 from anshulbaliga7/SPARK-58816-insert-struct-array-map-positional.
Authored-by: Anshul Baliga <ba...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../catalyst/analysis/ResolveInsertionBase.scala | 27 ++++++---
.../org/apache/spark/sql/SQLInsertTestSuite.scala | 67 ++++++++++++++++++++++
2 files changed, 87 insertions(+), 7 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInsertionBase.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInsertionBase.scala
index ad89005a093e..fa8be3aa1f30 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInsertionBase.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveInsertionBase.scala
@@ -21,7 +21,7 @@ import org.apache.spark.sql.catalyst.expressions.{Alias, Cast}
import org.apache.spark.sql.catalyst.plans.logical.{InsertIntoStatement, LogicalPlan, Project}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.errors.QueryCompilationErrors
-import org.apache.spark.sql.types.StructType
+import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType}
import org.apache.spark.sql.util.SchemaUtils
abstract class ResolveInsertionBase extends Rule[LogicalPlan] {
@@ -52,6 +52,10 @@ abstract class ResolveInsertionBase extends Rule[LogicalPlan] {
case (input: StructType, expected: StructType) =>
// Rename inner fields of the input column to pass the by-name INSERT analysis.
Alias(Cast(queryOutputCol, renameFieldsInStruct(input, expected)), resolvedCol.name)()
+ case (input: ArrayType, expected: ArrayType) =>
+ Alias(Cast(queryOutputCol, renameFieldsInType(input, expected)), resolvedCol.name)()
+ case (input: MapType, expected: MapType) =>
+ Alias(Cast(queryOutputCol, renameFieldsInType(input, expected)), resolvedCol.name)()
case _ =>
Alias(queryOutputCol, resolvedCol.name)()
}
@@ -62,16 +66,25 @@ abstract class ResolveInsertionBase extends Rule[LogicalPlan] {
private def renameFieldsInStruct(input: StructType, expected: StructType): StructType = {
if (input.length == expected.length) {
val newFields = input.zip(expected).map { case (f1, f2) =>
- (f1.dataType, f2.dataType) match {
- case (s1: StructType, s2: StructType) =>
- f1.copy(name = f2.name, dataType = renameFieldsInStruct(s1, s2))
- case _ =>
- f1.copy(name = f2.name)
- }
+ f1.copy(name = f2.name, dataType = renameFieldsInType(f1.dataType, f2.dataType))
}
StructType(newFields)
} else {
input
}
}
+
+ // Recursively rename fields so that positional INSERT analysis applies at every nesting level,
+ // including structs inside arrays and maps. See SPARK-58816.
+ private def renameFieldsInType(input: DataType, expected: DataType): DataType =
+ (input, expected) match {
+ case (s1: StructType, s2: StructType) =>
+ renameFieldsInStruct(s1, s2)
+ case (ArrayType(e1, n1), ArrayType(e2, _)) =>
+ ArrayType(renameFieldsInType(e1, e2), n1)
+ case (MapType(k1, v1, n1), MapType(k2, v2, _)) =>
+ MapType(renameFieldsInType(k1, k2), renameFieldsInType(v1, v2), n1)
+ case _ =>
+ input
+ }
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
index 7452917f57d7..7b1fb2e7dcc3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala
@@ -591,6 +591,73 @@ trait SQLInsertTestSuite extends QueryTest with AdaptiveSparkPlanHelper {
}
}
}
+
+ test("SPARK-58816: insert with column list resolves structs inside arrays positionally") {
+ withTable("t") {
+ createTable("t", Seq("arr"), Seq("ARRAY<STRUCT<x: INT, y: INT>>"))
+ // Source has fields in (y, x) order; target expects (x, y).
+ // Positional resolution must rename y->x and x->y so the cast maps by position.
+ sql("INSERT INTO t (arr) SELECT array(named_struct('y', 20, 'x', 10))")
+ checkAnswer(spark.table("t"), Row(Seq(Row(20, 10))))
+
+ // Contrast: INSERT BY NAME must continue to resolve by name, giving {x:10, y:20}.
+ sql("INSERT INTO t BY NAME SELECT array(named_struct('y', 20, 'x', 10)) AS arr")
+ checkAnswer(spark.table("t"), Seq(Row(Seq(Row(20, 10))), Row(Seq(Row(10, 20)))))
+ }
+ }
+
+ test("SPARK-58816: insert with column list resolves structs inside maps positionally") {
+ withTable("t") {
+ // Map value: MAP<STRING, STRUCT<x: INT, y: INT>>
+ createTable("t", Seq("m"), Seq("MAP<STRING, STRUCT<x: INT, y: INT>>"))
+ sql("INSERT INTO t (m) SELECT map('k', named_struct('y', 20, 'x', 10))")
+ checkAnswer(spark.table("t"), Row(Map("k" -> Row(20, 10))))
+ }
+ // Map key: MAP<STRUCT<x: INT, y: INT>, STRING> -- exercises the key recursion branch.
+ withTable("t") {
+ createTable("t", Seq("m"), Seq("MAP<STRUCT<x: INT, y: INT>, STRING>"))
+ sql("INSERT INTO t (m) SELECT map(named_struct('y', 20, 'x', 10), 'v')")
+ checkAnswer(spark.table("t"), Row(Map(Row(20, 10) -> "v")))
+ }
+ }
+
+ test("SPARK-58816: insert with column list resolves structs positionally at all nesting levels") {
+ // All three nesting modes must behave identically: direct struct, array-of-struct,
+ // map-of-struct. Source fields are (y, x); target fields are (x, y).
+ // Positional resolution writes y->x slot and x->y slot for each case.
+ withTable("t") {
+ createTable(
+ "t",
+ Seq("s", "arr", "m"),
+ Seq(
+ "STRUCT<x: INT, y: INT>",
+ "ARRAY<STRUCT<x: INT, y: INT>>",
+ "MAP<STRING, STRUCT<x: INT, y: INT>>"))
+ sql(
+ """INSERT INTO t (s, arr, m)
+ |SELECT
+ | named_struct('y', 20, 'x', 10),
+ | array(named_struct('y', 20, 'x', 10)),
+ | map('k', named_struct('y', 20, 'x', 10))
+ |""".stripMargin)
+ // After positional resolution: first field slot gets 20, second gets 10 in all three cases.
+ checkAnswer(
+ spark.table("t"),
+ Row(Row(20, 10), Seq(Row(20, 10)), Map("k" -> Row(20, 10))))
+ }
+ // Deep mixed nesting: ARRAY<STRUCT<nested: ARRAY<STRUCT<x: INT, y: INT>>>>.
+ // Two recursive transitions (Array->Struct->Array->Struct) must all be renamed positionally.
+ withTable("t") {
+ createTable(
+ "t", Seq("col"),
+ Seq("ARRAY<STRUCT<nested: ARRAY<STRUCT<x: INT, y: INT>>>>"))
+ sql(
+ """INSERT INTO t (col)
+ |SELECT array(named_struct(
+ | 'nested', array(named_struct('y', 20, 'x', 10))))""".stripMargin)
+ checkAnswer(spark.table("t"), Row(Seq(Row(Seq(Row(20, 10))))))
+ }
+ }
}
@SQLUserDefinedType(udt = classOf[MyIntUDT])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org