You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/11/14 20:32:55 UTC
(spark) branch branch-4.1 updated: [SPARK-54172][SQL][FOLLOW-UP] Simplify attribute re-resolution and validate action assignment value resolution for schema evolution
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 4adea7dddc98 [SPARK-54172][SQL][FOLLOW-UP] Simplify attribute re-resolution and validate action assignment value resolution for schema evolution
4adea7dddc98 is described below
commit 4adea7dddc98f02fb1a50ec4aa3af473109e545f
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Fri Nov 14 12:32:37 2025 -0800
[SPARK-54172][SQL][FOLLOW-UP] Simplify attribute re-resolution and validate action assignment value resolution for schema evolution
### What changes were proposed in this pull request?
Addressed remaining review comments in https://github.com/apache/spark/pull/52866
### Why are the changes needed?
Small code simplification, safety
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing unit tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53071 from szehon-ho/merge_schema_evolution_limit_follow.
Authored-by: Szehon Ho <sz...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit dc5bb9641b6e6fb8f55b1c524b3a21d4b5063283)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../sql/catalyst/analysis/ResolveMergeIntoSchemaEvolution.scala | 4 +---
.../org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala | 6 +++++-
2 files changed, 6 insertions(+), 4 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoSchemaEvolution.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoSchemaEvolution.scala
index f317a2efddbe..ea0883f7928f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoSchemaEvolution.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveMergeIntoSchemaEvolution.scala
@@ -48,9 +48,7 @@ object ResolveMergeIntoSchemaEvolution extends Rule[LogicalPlan] {
val newTarget = performSchemaEvolution(r, referencedSourceSchema, changes)
val oldTargetOutput = m.targetTable.output
val newTargetOutput = newTarget.output
- val attributeMapping = oldTargetOutput.map(
- oldAttr => (oldAttr, newTargetOutput.find(_.name == oldAttr.name).getOrElse(oldAttr))
- )
+ val attributeMapping = oldTargetOutput.zip(newTargetOutput)
newTarget -> attributeMapping
}
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index ea9f7c91cf00..c9d6fb61ba69 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -928,9 +928,13 @@ case class MergeIntoTable(
}.flatten
val sourcePaths = MergeIntoTable.extractAllFieldPaths(sourceTable.schema)
+ // Only allow unresolved assignment keys to be candidates for schema evolution
+ // if they are directly assigned from source fields, ie UPDATE SET new = source.new
assignments.forall { assignment =>
assignment.resolved ||
- sourcePaths.exists { path => MergeIntoTable.isEqual(assignment, path) }
+ (assignment.value.resolved && sourcePaths.exists {
+ path => MergeIntoTable.isEqual(assignment, path)
+ })
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org