You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/10/13 00:26:51 UTC

(spark) branch master updated: [MINOR][PYTHON][TEST] Make column resolution error differences explicit

This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new f70e229b63c9 [MINOR][PYTHON][TEST] Make column resolution error differences explicit
f70e229b63c9 is described below

commit f70e229b63c954688347dfcc534aca580f0eb931
Author: marko-radoicic-db <ma...@databricks.com>
AuthorDate: Tue Oct 13 08:26:22 2026 +0800

    [MINOR][PYTHON][TEST] Make column resolution error differences explicit
    
    ### What changes were proposed in this pull request?
    
    This follow-up to #58777 removes the API-dependent `assert_column_resolution_error` helper from the shared column tests. Classic error conditions are asserted inline, while the Connect tests with intentionally different public errors are explicitly overridden and documented in `ColumnParityTests`.
    
    This addresses the concern raised in https://github.com/apache/spark/pull/58777#issuecomment-5699678816.
    
    ### Why are the changes needed?
    
    An inherited Connect parity test normally communicates identical Classic and Connect behavior. Selecting different expected errors through an overridden helper hid observable error-condition differences inside otherwise inherited tests. Explicit Connect overrides make those differences visible and prevent future parity analysis from treating them as identical behavior.
    
    This follows the existing `master` convention for tests that produce different exceptions in Classic and Connect. The shared [`test_udtf_with_invalid_return_type`](https://github.com/apache/spark/blob/e985bd4b9cbfa02f937d7be2c585586a3375dda9/python/pyspark/sql/tests/test_udtf.py#L706-L719) expects `PySparkTypeError` with `UDTF_RETURN_TYPE_MISMATCH`, while [`UDTFParityTests` explicitly overrides the same test](https://github.com/apache/spark/blob/e985bd4b9cbfa02f937d7be2c585586a3375dda [...]
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. This is a test-only refactoring.
    
    ### How was this patch tested?
    
    The following commands passed:
    
    ```
    build/sbt -Phive package
    .venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.test_column ColumnTests"
    .venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.connect.test_parity_column ColumnParityTests"
    .venv/bin/python python/run-tests.py --testnames "pyspark.sql.tests.connect.test_parity_column ColumnParityTestsWithNonStrictDFColResolution"
    ```
    
    Both Connect suites had one expected skip: `test_validate_column_types`, which requires JVM access. The changed files also pass `git diff --check` and an in-memory Python syntax compilation check.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-5)
    
    Closes #59040 from marko-radoicic-db/column-resolution-error-parity.
    
    Authored-by: marko-radoicic-db <ma...@databricks.com>
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
 .../sql/tests/connect/test_parity_column.py        | 62 ++++++++++++++++++++--
 python/pyspark/sql/tests/test_column.py            | 59 ++++++++------------
 2 files changed, 81 insertions(+), 40 deletions(-)

diff --git a/python/pyspark/sql/tests/connect/test_parity_column.py b/python/pyspark/sql/tests/connect/test_parity_column.py
index a1f2122fb045..f480294b933d 100644
--- a/python/pyspark/sql/tests/connect/test_parity_column.py
+++ b/python/pyspark/sql/tests/connect/test_parity_column.py
@@ -24,9 +24,6 @@ from pyspark.testing.connectutils import ReusedConnectTestCase
 
 
 class ColumnParityTests(ColumnTestsMixin, ReusedConnectTestCase):
-    def assert_column_resolution_error(self, exception, *, classic_condition, connect_condition):
-        self.assertEqual(exception.getCondition(), connect_condition)
-
     @classmethod
     def setUpClass(cls):
         super().setUpClass()
@@ -43,6 +40,65 @@ class ColumnParityTests(ColumnTestsMixin, ReusedConnectTestCase):
     def test_validate_column_types(self):
         super().test_validate_column_types()
 
+    # Connect resolves tagged DataFrame columns by plan id, while Classic tracks
+    # attribute ids. Keep these tests explicit so the resulting public error
+    # differences are not hidden by inheritance.
+
+    def test_resolve_after_chained_withcolumn_shadow(self):
+        df = self.spark.sql("SELECT 1 AS c")
+        with self.assertRaises(AnalysisException) as error:
+            df.withColumn("c", sf.col("c").cast("string")).withColumn(
+                "c", sf.col("c").cast("int")
+            ).select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_after_select_alias_shadow(self):
+        df = self.spark.sql("SELECT 1 AS c")
+        with self.assertRaises(AnalysisException) as error:
+            df.select(df.c.cast("string").alias("c")).select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_after_withcolumnrenamed(self):
+        df = self.spark.sql("SELECT 1 AS c")
+        with self.assertRaises(AnalysisException) as error:
+            df.withColumnRenamed("c", "c2").select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_after_drop(self):
+        df = self.spark.sql("SELECT 1 AS c, 2 AS d")
+        with self.assertRaises(AnalysisException) as error:
+            df.drop("c").select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_generator_after_projection(self):
+        df = self.spark.sql("SELECT array(1, 2) AS arr")
+        projected = df.select(sf.lit(0).alias("keep"))
+        with self.assertRaises(AnalysisException) as error:
+            projected.select(sf.explode(df.arr)).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_after_agg_alias_shadow(self):
+        df = self.spark.sql("SELECT 1 AS c")
+        with self.assertRaises(AnalysisException) as error:
+            df.groupBy().agg(sf.sum("c").alias("c")).select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
+    def test_resolve_self_join_alias(self):
+        # Both sides match the tagged plan id at the same depth. Connect reports
+        # AMBIGUOUS_COLUMN_REFERENCE instead of Classic's legacy self-join error.
+        df = self.spark.sql("SELECT 1 AS c UNION ALL SELECT 2 AS c")
+        a, b = df.alias("a"), df.alias("b")
+        with self.assertRaises(AnalysisException) as error:
+            a.join(b, a.c == b.c).select(df.c).collect()
+        self.assertEqual(error.exception.getCondition(), "AMBIGUOUS_COLUMN_REFERENCE")
+
+    def test_resolve_cross_dataframe_illegal_reference(self):
+        df1 = self.spark.range(3)
+        df2 = self.spark.range(5)
+        with self.assertRaises(AnalysisException) as error:
+            df1.select(df2.id).collect()
+        self.assertEqual(error.exception.getCondition(), "CANNOT_RESOLVE_DATAFRAME_COLUMN")
+
     def test_resolve_after_union(self):
         # Connect diverges from Classic here: Union is treated as a leaf when
         # walking the plan tree for plan-id resolution, so the left-side plan
diff --git a/python/pyspark/sql/tests/test_column.py b/python/pyspark/sql/tests/test_column.py
index 8de94a34d66f..410a9eeb1159 100644
--- a/python/pyspark/sql/tests/test_column.py
+++ b/python/pyspark/sql/tests/test_column.py
@@ -31,10 +31,6 @@ from pyspark.testing.utils import have_pandas, pandas_requirement_message
 
 
 class ColumnTestsMixin:
-    def assert_column_resolution_error(self, exception, *, classic_condition, connect_condition):
-        """Assert the API-specific condition for a shared Classic/Connect test."""
-        self.assertEqual(exception.getCondition(), classic_condition)
-
     def test_column_name_encoding(self):
         """Ensure that created columns has `str` type consistently."""
         columns = self.spark.createDataFrame([("Alice", 1)], ["name", "age"]).columns
@@ -838,10 +834,9 @@ class ColumnTestsMixin:
             df.withColumn("c", sf.col("c").cast("string")).withColumn(
                 "c", sf.col("c").cast("int")
             ).select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
         )
 
     def test_resolve_after_select_alias_shadow(self):
@@ -851,10 +846,9 @@ class ColumnTestsMixin:
         df = self.spark.sql("SELECT 1 AS c")
         with self.assertRaises(AnalysisException) as error:
             df.select(df.c.cast("string").alias("c")).select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
         )
 
     def test_resolve_after_withcolumnrenamed(self):
@@ -864,10 +858,9 @@ class ColumnTestsMixin:
         df = self.spark.sql("SELECT 1 AS c")
         with self.assertRaises(AnalysisException) as error:
             df.withColumnRenamed("c", "c2").select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
         )
 
     def test_resolve_after_drop(self):
@@ -876,10 +869,9 @@ class ColumnTestsMixin:
         df = self.spark.sql("SELECT 1 AS c, 2 AS d")
         with self.assertRaises(AnalysisException) as error:
             df.drop("c").select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
         )
 
     def test_resolve_generator_after_projection(self):
@@ -888,10 +880,9 @@ class ColumnTestsMixin:
         projected = df.select(sf.lit(0).alias("keep"))
         with self.assertRaises(AnalysisException) as error:
             projected.select(sf.explode(df.arr)).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_MISSING_FROM_INPUT",
         )
 
     def test_resolve_through_filter(self):
@@ -929,10 +920,9 @@ class ColumnTestsMixin:
         df = self.spark.sql("SELECT 1 AS c")
         with self.assertRaises(AnalysisException) as error:
             df.groupBy().agg(sf.sum("c").alias("c")).select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
         )
 
     def test_resolve_after_pivot(self):
@@ -1101,11 +1091,7 @@ class ColumnTestsMixin:
         a, b = df.alias("a"), df.alias("b")
         with self.assertRaises(AnalysisException) as error:
             a.join(b, a.c == b.c).select(df.c).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="_LEGACY_ERROR_TEMP_1182",
-            connect_condition="AMBIGUOUS_COLUMN_REFERENCE",
-        )
+        self.assertEqual(error.exception.getCondition(), "_LEGACY_ERROR_TEMP_1182")
 
     def test_resolve_after_subquery_view(self):
         # Persisting the DataFrame as a temp view and reading it back via
@@ -1128,10 +1114,9 @@ class ColumnTestsMixin:
         df2 = self.spark.range(5)
         with self.assertRaises(AnalysisException) as error:
             df1.select(df2.id).collect()
-        self.assert_column_resolution_error(
-            error.exception,
-            classic_condition="MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
-            connect_condition="CANNOT_RESOLVE_DATAFRAME_COLUMN",
+        self.assertEqual(
+            error.exception.getCondition(),
+            "MISSING_ATTRIBUTES.RESOLVED_ATTRIBUTE_APPEAR_IN_OPERATION",
         )
 
     def test_resolve_df_star(self):


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org