You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/09/08 08:34:44 UTC
(spark) branch branch-4.0 updated: [SPARK-53505][CONNECT] Try the eager analysis first for withColumns and withColumnsRenamed in Spark Connect
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 043819a9bd4d [SPARK-53505][CONNECT] Try the eager analysis first for withColumns and withColumnsRenamed in Spark Connect
043819a9bd4d is described below
commit 043819a9bd4dd57f56363bdeeeee2c06f64598dd
Author: Takuya Ueshin <ue...@databricks.com>
AuthorDate: Mon Sep 8 16:34:14 2025 +0800
[SPARK-53505][CONNECT] Try the eager analysis first for withColumns and withColumnsRenamed in Spark Connect
### What changes were proposed in this pull request?
Tries the eager analysis first for `withColumns` and `withColumnsRenamed` in Spark Connect.
### Why are the changes needed?
The planning for `withColumns` and `withColumnsRenamed` in Spark Connect was changed to be lazy at #49386, but it could cause a performance issue as the current analyzer can't resolve it fast enough when they are used many times, which can happen relatively more often than the other APIs.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The existing tests should pass.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52253 from ueshin/issues/SPARK-53505/eager_analysis.
Authored-by: Takuya Ueshin <ue...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit c247483480d3654a6b3ec4045fdc782c611cfae2)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../sql/connect/planner/SparkConnectPlanner.scala | 43 ++++++++++++++++------
1 file changed, 32 insertions(+), 11 deletions(-)
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index bf1b6e7e00eb..a0db7a9627a2 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -44,7 +44,7 @@ import org.apache.spark.connect.proto.WriteStreamOperationStart.TriggerCase
import org.apache.spark.internal.{Logging, LogKeys, MDC}
import org.apache.spark.internal.LogKeys.{DATAFRAME_ID, SESSION_ID}
import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile, TaskResourceProfile, TaskResourceRequest}
-import org.apache.spark.sql.{Column, Encoders, ForeachWriter, Observation, Row}
+import org.apache.spark.sql.{AnalysisException, Column, Encoders, ForeachWriter, Observation, Row}
import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier, QueryPlanningTracker}
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedStarWithColumns, UnresolvedStarWithColumnsRenames, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunction, Unresolved [...]
import org.apache.spark.sql.catalyst.encoders.{encoderFor, AgnosticEncoder, ExpressionEncoder, RowEncoder}
@@ -1091,9 +1091,20 @@ class SparkConnectPlanner(
// for backward compatibility
rel.getRenameColumnsMapMap.asScala.toSeq.unzip
}
- Project(
- Seq(UnresolvedStarWithColumnsRenames(existingNames = colNames, newNames = newColNames)),
- transformRelation(rel.getInput))
+
+ val child = transformRelation(rel.getInput)
+ try {
+ // Try the eager analysis first.
+ Dataset
+ .ofRows(session, child)
+ .withColumnsRenamed(colNames, newColNames)
+ .logicalPlan
+ } catch {
+ case _: AnalysisException | _: SparkException =>
+ Project(
+ Seq(UnresolvedStarWithColumnsRenames(existingNames = colNames, newNames = newColNames)),
+ child)
+ }
}
private def transformWithColumns(rel: proto.WithColumns): LogicalPlan = {
@@ -1113,13 +1124,23 @@ class SparkConnectPlanner(
(alias.getName(0), transformExpression(alias.getExpr), metadata)
}.unzip3
- Project(
- Seq(
- UnresolvedStarWithColumns(
- colNames = colNames,
- exprs = exprs,
- explicitMetadata = Some(metadata))),
- transformRelation(rel.getInput))
+ val child = transformRelation(rel.getInput)
+ try {
+ // Try the eager analysis first.
+ Dataset
+ .ofRows(session, child)
+ .withColumns(colNames, exprs.map(expr => Column(expr)), metadata)
+ .logicalPlan
+ } catch {
+ case _: AnalysisException | _: SparkException =>
+ Project(
+ Seq(
+ UnresolvedStarWithColumns(
+ colNames = colNames,
+ exprs = exprs,
+ explicitMetadata = Some(metadata))),
+ child)
+ }
}
private def transformWithWatermark(rel: proto.WithWatermark): LogicalPlan = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org