You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/04/22 21:47:51 UTC
(spark) branch master updated: [SPARK-56547][SQL][CONNECT] Improve error message when DataFrame column is referenced from a wrong DataFrame
This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b4aff0b9abee [SPARK-56547][SQL][CONNECT] Improve error message when DataFrame column is referenced from a wrong DataFrame
b4aff0b9abee is described below
commit b4aff0b9abeed43ad966c94b4429dc5fe3a79f3a
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Apr 23 06:47:39 2026 +0900
[SPARK-56547][SQL][CONNECT] Improve error message when DataFrame column is referenced from a wrong DataFrame
### What changes were proposed in this pull request?
In `CheckAnalysis.failUnresolvedAttribute`, when the unresolved `Attribute` carries `PLAN_ID_TAG` (i.e. a Spark Connect DataFrame column reference like `df.col_name` or `df["col_name"]`), raise the existing `CANNOT_RESOLVE_DATAFRAME_COLUMN` error instead of the generic `UNRESOLVED_COLUMN`. The tag reliably identifies DataFrame column references that the plan-id-based resolution already tried and failed, so the dedicated error's message ("illegal references like `df1.select(df2.col("a" [...]
### Why are the changes needed?
The previous error was misleading. For example:
```python
from pyspark.sql import functions as sf
df = spark.range(10)
df2 = df.withColumn("id", sf.col("id") + 1)
df2.select(df["id"]).show()
```
produced:
```
AnalysisException: [UNRESOLVED_COLUMN.WITH_SUGGESTION] A column, variable, or function parameter with name `id` cannot be resolved. Did you mean one of the following? [`id`]. SQLSTATE: 42703;
'Project ['id]
+- Project [(id#1L + cast(1 as bigint)) AS id#3L]
+- Range (0, 10, step=1, splits=Some(1))
```
The message says `id` can't be resolved and suggests `id` as a fix, with no hint that the real issue is a reference to a column from a different DataFrame whose value was overwritten by `withColumn`.
### Does this PR introduce _any_ user-facing change?
Yes. For DataFrame column references that fail to resolve in Spark Connect, the error class changes from `UNRESOLVED_COLUMN.WITH_SUGGESTION` (or `UNRESOLVED_COLUMN.WITHOUT_SUGGESTION`) to `CANNOT_RESOLVE_DATAFRAME_COLUMN`:
```
[CANNOT_RESOLVE_DATAFRAME_COLUMN] Cannot resolve dataframe column "id". It's probably because of illegal references like `df1.select(df2.col("a"))`. SQLSTATE: 42704
```
### How was this patch tested?
Updated `test_invalid_column` in `python/pyspark/sql/tests/connect/test_connect_error.py`:
- `cdf3.select(cdf1.b)` where `cdf3 = cdf1.select(cdf1.a)` — existing case, error class updated.
- `cdf3.select(cdf1.a)` where `cdf3 = cdf1.withColumn("a", F.lit(0))` — new case: column shadowed via `withColumn` on a `createDataFrame` source.
- `df2.select(df["id"])` where `df = spark.range(10)` and `df2 = df.withColumn("id", F.col("id") + 1)` — new regression test mirroring the exact reproduction from the bug report.
Also updated `ClientE2ETestSuite."SPARK-45509: ambiguous column reference"` in `sql/connect/client/jvm/src/test/scala/.../ClientE2ETestSuite.scala`, which hits the same path.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.7)
Closes #55421 from zhengruifeng/better-error-message.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../sql/tests/connect/test_connect_error.py | 24 ++++++++++++++++++++--
.../sql/catalyst/analysis/CheckAnalysis.scala | 8 ++++++++
.../spark/sql/connect/ClientE2ETestSuite.scala | 2 +-
3 files changed, 31 insertions(+), 3 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_connect_error.py b/python/pyspark/sql/tests/connect/test_connect_error.py
index 2213d1b1ed29..50c6900cabc0 100644
--- a/python/pyspark/sql/tests/connect/test_connect_error.py
+++ b/python/pyspark/sql/tests/connect/test_connect_error.py
@@ -64,14 +64,34 @@ class SparkConnectErrorTests(ReusedConnectTestCase):
with self.assertRaises(AnalysisException):
cdf2.withColumn("x", cdf1.a + 1).schema
- # Can find the target plan node, but fail to resolve with it
+ # Can find the target plan node, but the resolved attribute is not part of the outer
+ # operator's output (e.g. it was dropped by a projection).
with self.assertRaisesRegex(
AnalysisException,
- "UNRESOLVED_COLUMN.WITH_SUGGESTION",
+ "CANNOT_RESOLVE_DATAFRAME_COLUMN",
):
cdf3 = cdf1.select(cdf1.a)
cdf3.select(cdf1.b).schema
+ # Same pattern: the column from a different DataFrame is overwritten by `withColumn`.
+ with self.assertRaisesRegex(
+ AnalysisException,
+ "CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ ):
+ cdf3 = cdf1.withColumn("a", F.lit(0))
+ cdf3.select(cdf1.a).schema
+
+ # SPARK-56547: `df["id"]` references the original `id`, but `df2`'s `id` was shadowed
+ # by `withColumn`. Previously surfaced a misleading
+ # `UNRESOLVED_COLUMN.WITH_SUGGESTION` error suggesting the same column name.
+ df = self.spark.range(10)
+ df2 = df.withColumn("id", F.col("id") + 1)
+ with self.assertRaisesRegex(
+ AnalysisException,
+ "CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ ):
+ df2.select(df["id"]).schema
+
# Can not find the target plan node by plan id
with self.assertRaisesRegex(
AnalysisException,
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
index 17fac640d483..9d48955cbc71 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala
@@ -214,6 +214,14 @@ trait CheckAnalysis extends LookupCatalog with QueryErrorsBase with PlanToString
operator: LogicalPlan,
a: Attribute,
errorClass: String): Nothing = {
+ // An unresolved attribute tagged with `PLAN_ID_TAG` is a Spark Connect DataFrame column
+ // reference (`df.col_name` or `df["col_name"]`). If it ends up unresolved, it means the
+ // referenced column is not reachable from the current operator — typically because the
+ // user referenced a column from the wrong DataFrame. Raise the dedicated error so the
+ // message points at the wrong DataFrame reference instead of the generic unresolved column.
+ if (a.containsTag(LogicalPlan.PLAN_ID_TAG)) {
+ throw QueryCompilationErrors.cannotResolveDataFrameColumn(a)
+ }
val missingCol = a.sql
val candidates = operator.inputSet.toSeq
.map(attr => attr.qualifier :+ attr.name)
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
index ed227a24433b..0e8cb9348c7f 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
@@ -970,7 +970,7 @@ class ClientE2ETestSuite
// df1("i") is not ambiguous, but it's not valid in the projected df.
df1.select((df1("i") + 1).as("plus")).select(df1("i")).collect()
}
- assert(e1.getMessage.contains("UNRESOLVED_COLUMN.WITH_SUGGESTION"))
+ assert(e1.getMessage.contains("CANNOT_RESOLVE_DATAFRAME_COLUMN"))
checkSameResult(
Seq(Row(1, "a")),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org