You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/12/04 16:50:45 UTC
(spark) branch branch-4.1 updated: [SPARK-54578][SQL] Perform Code Cleanup on `AssignmentUtils`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 7012a923fd81 [SPARK-54578][SQL] Perform Code Cleanup on `AssignmentUtils`
7012a923fd81 is described below
commit 7012a923fd81c9cf5740670333241627a95de4fc
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Thu Dec 4 08:50:25 2025 -0800
[SPARK-54578][SQL] Perform Code Cleanup on `AssignmentUtils`
### What changes were proposed in this pull request?
This PR performs a code cleanup on `AssignmentUtils`:
1. Deleted the redundant private methods `getMissingSourcePaths` and `createNullCheckForFieldPath`, which were introduced in https://github.com/apache/spark/pull/53149 but became obsolete after https://github.com/apache/spark/pull/53229.
2. Removed the redundant documentation for the invalid input parameter `missingSourcePaths` in the `alignUpdateAssignments` method.
3. Removed the unused input parameter `updateStar` from the private method `applyAssignments`.
4. Corrected a spelling error in the input parameter `coerceNestedTypes` of the private method `applyFieldAssignments`.
### Why are the changes needed?
Code Cleanup.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass Github Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53302 from LuciferYang/SPARK-54578.
Lead-authored-by: yangjie01 <ya...@baidu.com>
Co-authored-by: YangJie <ya...@baidu.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 8b5d900e0f7899dd17b91b6e5293eaca7947ee4e)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../sql/catalyst/analysis/AssignmentUtils.scala | 59 ++--------------------
1 file changed, 5 insertions(+), 54 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
index c9ed5b86dbde..6c7b0626e81e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentUtils.scala
@@ -21,7 +21,7 @@ import scala.collection.mutable
import org.apache.spark.sql.catalyst.SQLConfHelper
import org.apache.spark.sql.catalyst.analysis.TableOutputResolver.DefaultValueFillMode.{NONE, RECURSE}
-import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateNamedStruct, Expression, GetStructField, IsNull, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Attribute, CreateNamedStruct, Expression, GetStructField, Literal}
import org.apache.spark.sql.catalyst.plans.logical.Assignment
import org.apache.spark.sql.catalyst.types.DataTypeUtils
import org.apache.spark.sql.catalyst.util.CharVarcharUtils
@@ -55,7 +55,6 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
* (preserving existing fields).
* @param coerceNestedTypes whether to coerce nested types to match the target type
* for complex types
- * @param missingSourcePaths paths that exist in target but not in source
* @return aligned update assignments that match table attributes
*/
def alignUpdateAssignments(
@@ -73,8 +72,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
assignments,
addError = err => errors += err,
colPath = Seq(attr.name),
- coerceNestedTypes,
- fromStar)
+ coerceNestedTypes)
}
if (errors.nonEmpty) {
@@ -158,8 +156,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
assignments: Seq[Assignment],
addError: String => Unit,
colPath: Seq[String],
- coerceNestedTypes: Boolean = false,
- updateStar: Boolean = false): Expression = {
+ coerceNestedTypes: Boolean = false): Expression = {
val (exactAssignments, otherAssignments) = assignments.partition { assignment =>
assignment.key.semanticEquals(colExpr)
@@ -197,7 +194,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
assignments: Seq[Assignment],
addError: String => Unit,
colPath: Seq[String],
- coerceNestedTyptes: Boolean): Expression = {
+ coerceNestedTypes: Boolean): Expression = {
col.dataType match {
case structType: StructType =>
@@ -207,7 +204,7 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
}
val updatedFieldExprs = fieldAttrs.zip(fieldExprs).map { case (fieldAttr, fieldExpr) =>
applyAssignments(fieldAttr, fieldExpr, assignments, addError, colPath :+ fieldAttr.name,
- coerceNestedTyptes)
+ coerceNestedTypes)
}
toNamedStruct(structType, updatedFieldExprs)
@@ -226,52 +223,6 @@ object AssignmentUtils extends SQLConfHelper with CastSupport {
CreateNamedStruct(namedStructExprs)
}
- private def getMissingSourcePaths(targetType: StructType,
- sourceType: DataType,
- colPath: Seq[String],
- addError: String => Unit): Seq[Seq[String]] = {
- val nestedTargetPaths = DataTypeUtils.extractLeafFieldPaths(targetType, Seq.empty)
- val nestedSourcePaths = sourceType match {
- case sourceStructType: StructType =>
- DataTypeUtils.extractLeafFieldPaths(sourceStructType, Seq.empty)
- case _ =>
- addError(s"Value for struct type: " +
- s"${colPath.quoted} must be a struct but was ${sourceType.simpleString}")
- Seq()
- }
- nestedSourcePaths.diff(nestedTargetPaths)
- }
-
- /**
- * Creates a null check for a field at the given path within a struct expression.
- * Navigates through the struct hierarchy following the path and returns an IsNull check
- * for the final field.
- *
- * @param rootExpr the root expression to navigate from
- * @param path the field path to navigate (sequence of field names)
- * @return an IsNull expression checking if the field at the path is null
- */
- private def createNullCheckForFieldPath(
- rootExpr: Expression,
- path: Seq[String]): Expression = {
- var currentExpr: Expression = rootExpr
- path.foreach { fieldName =>
- currentExpr.dataType match {
- case st: StructType =>
- st.fields.find(f => conf.resolver(f.name, fieldName)) match {
- case Some(field) =>
- val fieldIndex = st.fieldIndex(field.name)
- currentExpr = GetStructField(currentExpr, fieldIndex, Some(field.name))
- case None =>
- // Field not found, shouldn't happen
- }
- case _ =>
- // Not a struct, shouldn't happen
- }
- }
- IsNull(currentExpr)
- }
-
/**
* Checks whether assignments are aligned and compatible with table columns.
*
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org