(spark) branch branch-4.2 updated: [SPARK-56714][SQL] Remove `__metadata_col` metadata from aggregated access only columns

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new 8485a91ed1b9 [SPARK-56714][SQL] Remove `__metadata_col` metadata from aggregated access only columns
8485a91ed1b9 is described below

commit 8485a91ed1b9711bdf50174d28c2c63fa10e609d
Author: Mihailo Aleksic <mi...@databricks.com>
AuthorDate: Wed May 6 00:21:59 2026 +0800

    [SPARK-56714][SQL] Remove `__metadata_col` metadata from aggregated access only columns
    
    ### What changes were proposed in this pull request?
    In this PR I propose to remove `__metadata_col` metadata from aggregated access only columns. It's not semantically needed and it blocks metadata column resolution work in single-pass resolver.
    
    ### Why are the changes needed?
    To make aggregated access only metadata semantically sound.
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Existing tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No.
    
    Closes #55667 from mihailoale-db/removemetadataforqualified.
    
    Authored-by: Mihailo Aleksic <mi...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../scala/org/apache/spark/sql/catalyst/util/package.scala   | 12 +++++-------
 .../apache/spark/sql/catalyst/analysis/AnalysisSuite.scala   |  6 ++++++
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
index 67423e1b50d3..2e7e88633bfa 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/package.scala
@@ -187,10 +187,10 @@ package object util extends Logging {
   val QUALIFIED_ACCESS_ONLY = "__qualified_access_only"
 
   /**
-   * If set, this metadata column can only be accessed under [[AggregateExpression]]. This is
-   * important when resolving columns in ORDER BY and HAVING clauses on top of [[Aggregate]].
-   * In this case we can only reference attributes from grouping expressions, or attributes marked
-   * as "__aggregated_access_only" under [[AggregateExpression]].
+   * If set, this column can only be accessed under [[AggregateExpression]]. This is important when
+   * resolving columns in ORDER BY and HAVING clauses on top of [[Aggregate]]. In this case we can
+   * only reference attributes from grouping expressions, or attributes marked as
+   * "__aggregated_access_only" under [[AggregateExpression]].
    */
   val AGGREGATED_ACCESS_ONLY = "__aggregated_access_only"
 
@@ -202,8 +202,7 @@ package object util extends Logging {
       attr.metadata.contains(QUALIFIED_ACCESS_ONLY) &&
       attr.metadata.getBoolean(QUALIFIED_ACCESS_ONLY)
 
-    def aggregatedAccessOnly: Boolean = attr.isMetadataCol &&
-      attr.metadata.contains(AGGREGATED_ACCESS_ONLY) &&
+    def aggregatedAccessOnly: Boolean = attr.metadata.contains(AGGREGATED_ACCESS_ONLY) &&
       attr.metadata.getBoolean(AGGREGATED_ACCESS_ONLY)
 
     def markAsQualifiedAccessOnly(): Attribute = attr.withMetadata(
@@ -217,7 +216,6 @@ package object util extends Logging {
     def markAsAggregatedAccessOnly(): Attribute = attr.withMetadata(
       new MetadataBuilder()
         .withMetadata(attr.metadata)
-        .putString(METADATA_COL_ATTR_KEY, attr.name)
         .putBoolean(AGGREGATED_ACCESS_ONLY, true)
         .build()
     )
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
index e25777b230ce..4a7adcc050a0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
@@ -1764,6 +1764,12 @@ class AnalysisSuite extends AnalysisTest with Matchers {
     checkAnalysis(rel.select($"a"), rel.select(attr.markAsAllowAnyAccess()))
   }
 
+  test("SPARK-56714: __aggregated_access_only should not imply metadata column") {
+    val attr = $"a".int.markAsAggregatedAccessOnly()
+    assert(!attr.isMetadataCol)
+    assert(attr.aggregatedAccessOnly)
+  }
+
   test("SPARK-43030: deduplicate relations with duplicate aliases") {
     // Should not fail with the assertion failure: Found duplicate rewrite attributes.
     val alias = Alias($"a", "x")()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org