You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/10/13 00:26:51 UTC
(spark) branch master updated: [MINOR][PYTHON][TEST] Make column resolution error differences explicit
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f70e229b63c9 [MINOR][PYTHON][TEST] Make column resolution error differences explicit
f70e229b63c9 is described below
commit f70e229b63c954688347dfcc534aca580f0eb931
Author: marko-radoicic-db <ma...@databricks.com>
AuthorDate: Tue Oct 13 08:26:22 2026 +0800
[MINOR][PYTHON][TEST] Make column resolution error differences explicit
### What changes were proposed in this pull request?
This follow-up to #58777 removes the API-dependent `assert_column_resolution_error` helper from the shared column tests. Classic error conditions are asserted inline, while the Connect tests with intentionally different public errors are explicitly overridden and documented in `ColumnParityTests`.
This addresses the concern raised in https://github.com/apache/spark/pull/58777#issuecomment-5699678816.
### Why are the changes needed?
An inherited Connect parity test normally communicates identical Classic and Connect behavior. Selecting different expected errors through an overridden helper hid observable error-condition differences inside otherwise inherited tests. Explicit Connect overrides make those differences visible and prevent future parity analysis from treating them as identical behavior.
This follows the existing `master` convention for tests that produce different exceptions in Classic and Connect. The shared [`test_udtf_with_invalid_return_type`](https://github.com/apache/spark/blob/e985bd4b9cbfa02f937d7be2c585586a3375dda9/python/pyspark/sql/tests/test_udtf.py#L706-L719) expects `PySparkTypeError` with `UDTF_RETURN_TYPE_MISMATCH`, while [`UDTFParityTests` explicitly overrides the same test](https://github.com/apache/spark/blob/e985bd4b9cbfa02f937d7be2c585586a3375dda [...]
### Does this PR introduce _any_ user-facing change?
No. This is a test-only refactoring.
### How was this patch tested?
The following commands passed:
```
build/sbt -Phive package
.venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.test_column ColumnTests"
.venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.connect.test_parity_column ColumnParityTests"
.venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.connect.test_parity_column ColumnParityTestsWithNonStrictDFColResolution"
```
Both Connect suites had one expected skip: `test_validate_column_types`, which requires JVM access. The changed files also pass `git diff --check` and an in-memory Python syntax compilation check.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #59040 from marko-radoicic-db/column-resolution-error-parity.
Authored-by: marko-radoicic-db <ma...@databricks.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../sql/tests/connect/test_parity_column.py | 62 ++++++++++++++++++++--
python/pyspark/sql/tests/test_column.py | 59 ++++++++------------
2 files changed, 81 insertions(+), 40 deletions(-)
diff --git a/python/pyspark/sql/tests/connect/test_parity_column.py b/python/pyspark/sql/tests/connect/test_parity_column.py
index a1f2122fb045..f480294b933d 100644
--- a/python/pyspark/sql/tests/connect/test_parity_column.py
+++ b/python/pyspark/sql/tests/connect/test_parity_column.py
@@ -24,9 +24,6 @@ from pyspark.testing.connectutils import ReusedConnectTestCase
class ColumnParityTests(ColumnTestsMixin, ReusedConnectTestCase):
- def assert_column_resolution_error(self, exception, *, classic_condition, connect_condition):
- self.assertEqual(exception.getCondition(), connect_condition)
-
@classmethod
def setUpClass(cls):
super().setUpClass()
@@ -43,6 +40,65 @@ class ColumnParityTests(ColumnTestsMixin, ReusedConnectTestCase):
def test_validate_column_types(self):
super().test_validate_column_types()
+ # Connect resolves tagged DataFrame columns by plan id, while Classic tracks
+ # attribute ids. Keep these tests explicit so the resulting public error
+ # differences are not hidden by inheritance.
+
+ def test_resolve_after_chained_withcolumn_shadow(self):
+ df = self.spark.sql("SELECT 1 AS c")
+ with self.assertRaises(AnalysisException) as error:
+ df.withColumn("c", sf.col("c").cast("string")).withColumn(
+ "c", sf.col("c").cast("int")
+ ).select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_after_select_alias_shadow(self):
+ df = self.spark.sql("SELECT 1 AS c")
+ with self.assertRaises(AnalysisException) as error:
+ df.select(df.c.cast("string").alias("c")).select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_after_withcolumnrenamed(self):
+ df = self.spark.sql("SELECT 1 AS c")
+ with self.assertRaises(AnalysisException) as error:
+ df.withColumnRenamed("c", "c2").select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_after_drop(self):
+ df = self.spark.sql("SELECT 1 AS c, 2 AS d")
+ with self.assertRaises(AnalysisException) as error:
+ df.drop("c").select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_generator_after_projection(self):
+ df = self.spark.sql("SELECT array(1, 2) AS arr")
+ projected = df.select(sf.lit(0).alias("keep"))
+ with self.assertRaises(AnalysisException) as error:
+ projected.select(sf.explode(df.arr)).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_after_agg_alias_shadow(self):
+ df = self.spark.sql("SELECT 1 AS c")
+ with self.assertRaises(AnalysisException) as error:
+ df.groupBy().agg(sf.sum("c").alias("c")).select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+ def test_resolve_self_join_alias(self):
+ # Both sides match the tagged plan id at the same depth. Connect reports
+ # AMBIGUOUS_COLUMN_REFERENCE instead of Classic's legacy self-join error.
+ df = self.spark.sql("SELECT 1 AS c UNION ALL SELECT 2 AS c")
+ a, b = df.alias("a"), df.alias("b")
+ with self.assertRaises(AnalysisException) as error:
+ a.join(b, a.c == b.c).select(df.c).collect()
+ self.assertEqual(error.exception.getCondition(), "AMBIGUOUS_COLUMN_REFERENCE")
+
+ def test_resolve_cross_dataframe_illegal_reference(self):
+ df1 = self.spark.range(3)
+ df2 = self.spark.range(5)
+ with self.assertRaises(AnalysisException) as error:
+ df1.select(df2.id).collect()
+ self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
def test_resolve_after_union(self):
# Connect diverges from Classic here: Union is treated as a leaf when
# walking the plan tree for plan-id resolution, so the left-side plan
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index 8de94a34d66f..410a9eeb1159 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -31,10 +31,6 @@ from pyspark.testing.utils import have_pandas, pandas_requirement_message
class ColumnTestsMixin:
- def assert_column_resolution_error(self, exception, *, classic_condition, connect_condition):
- """Assert the API-specific condition for a shared Classic/Connect test."""
- self.assertEqual(exception.getCondition(), classic_condition)
-
def test_column_name_encoding(self):
"""Ensure that created columns has `str` type consistently."""
columns = self.spark.createDataFrame([("Alice", 1)], ["name", "age"]).columns
@@ -838,10 +834,9 @@ class ColumnTestsMixin:
df.withColumn("c", sf.col("c").cast("string")).withColumn(
"c", sf.col("c").cast("int")
).select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
)
def test_resolve_after_select_alias_shadow(self):
@@ -851,10 +846,9 @@ class ColumnTestsMixin:
df = self.spark.sql("SELECT 1 AS c")
with self.assertRaises(AnalysisException) as error:
df.select(df.c.cast("string").alias("c")).select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
)
def test_resolve_after_withcolumnrenamed(self):
@@ -864,10 +858,9 @@ class ColumnTestsMixin:
df = self.spark.sql("SELECT 1 AS c")
with self.assertRaises(AnalysisException) as error:
df.withColumnRenamed("c", "c2").select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
)
def test_resolve_after_drop(self):
@@ -876,10 +869,9 @@ class ColumnTestsMixin:
df = self.spark.sql("SELECT 1 AS c, 2 AS d")
with self.assertRaises(AnalysisException) as error:
df.drop("c").select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
)
def test_resolve_generator_after_projection(self):
@@ -888,10 +880,9 @@ class ColumnTestsMixin:
projected = df.select(sf.lit(0).alias("keep"))
with self.assertRaises(AnalysisException) as error:
projected.select(sf.explode(df.arr)).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
)
def test_resolve_through_filter(self):
@@ -929,10 +920,9 @@ class ColumnTestsMixin:
df = self.spark.sql("SELECT 1 AS c")
with self.assertRaises(AnalysisException) as error:
df.groupBy().agg(sf.sum("c").alias("c")).select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
)
def test_resolve_after_pivot(self):
@@ -1101,11 +1091,7 @@ class ColumnTestsMixin:
a, b = df.alias("a"), df.alias("b")
with self.assertRaises(AnalysisException) as error:
a.join(b, a.c == b.c).select(df.c).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="_LEGACY_ERROR_TEMP_1182",
- connect_condition="AMBIGUOUS_COLUMN_REFERENCE",
- )
+ self.assertEqual(error.exception.getCondition(), "_LEGACY_ERROR_TEMP_1182")
def test_resolve_after_subquery_view(self):
# Persisting the DataFrame as a temp view and reading it back via
@@ -1128,10 +1114,9 @@ class ColumnTestsMixin:
df2 = self.spark.range(5)
with self.assertRaises(AnalysisException) as error:
df1.select(df2.id).collect()
- self.assert_column_resolution_error(
- error.exception,
- classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
- connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+ self.assertEqual(
+ error.exception.getCondition(),
+ "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
)
def test_resolve_df_star(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org