You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/09/15 18:29:10 UTC
[spark] branch branch-3.3 updated: [SPARK-40429][SQL][3.3] Only set KeyGroupedPartitioning when the referenced column is in the output
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new d7483b51f56 [SPARK-40429][SQL][3.3] Only set KeyGroupedPartitioning when the referenced column is in the output
d7483b51f56 is described below
commit d7483b51f562d361be3dc995dfefcc4ac8d2e45f
Author: huaxingao <hu...@apple.com>
AuthorDate: Thu Sep 15 11:28:20 2022 -0700
[SPARK-40429][SQL][3.3] Only set KeyGroupedPartitioning when the referenced column is in the output
### What changes were proposed in this pull request?
back porting [PR](https://github.com/apache/spark/pull/37886) to 3.3.
Only set `KeyGroupedPartitioning` when the referenced column is in the output
### Why are the changes needed?
bug fixing
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
New test
Closes #37901 from huaxingao/3.3.
Authored-by: huaxingao <hu...@apple.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../execution/datasources/v2/V2ScanPartitioning.scala | 14 ++++++++++++--
.../apache/spark/sql/connector/MetadataColumnSuite.scala | 16 ++++++++++++++++
2 files changed, 28 insertions(+), 2 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanPartitioning.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanPartitioning.scala
index 9a5a7e6aab6..64e80081018 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanPartitioning.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/V2ScanPartitioning.scala
@@ -39,8 +39,18 @@ object V2ScanPartitioning extends Rule[LogicalPlan] with SQLConfHelper {
}
val catalystPartitioning = scan.outputPartitioning() match {
- case kgp: KeyGroupedPartitioning => sequenceToOption(kgp.keys().map(
- V2ExpressionUtils.toCatalystOpt(_, relation, funCatalogOpt)))
+ case kgp: KeyGroupedPartitioning =>
+ val partitioning = sequenceToOption(kgp.keys().map(
+ V2ExpressionUtils.toCatalystOpt(_, relation, funCatalogOpt)))
+ if (partitioning.isEmpty) {
+ None
+ } else {
+ if (partitioning.get.forall(p => p.references.subsetOf(d.outputSet))) {
+ partitioning
+ } else {
+ None
+ }
+ }
case _: UnknownPartitioning => None
case p => throw new IllegalArgumentException("Unsupported data source V2 partitioning " +
"type: " + p.getClass.getSimpleName)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/MetadataColumnSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/MetadataColumnSuite.scala
index 95b9c4f7235..7f0e74f6bc7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/MetadataColumnSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/MetadataColumnSuite.scala
@@ -216,4 +216,20 @@ class MetadataColumnSuite extends DatasourceV2SQLBase {
.withColumn("right_all", struct($"right.*"))
checkAnswer(dfQuery, Row(1, "a", "b", Row(1, "a"), Row(1, "b")))
}
+
+ test("SPARK-40429: Only set KeyGroupedPartitioning when the referenced column is in the output") {
+ withTable(tbl) {
+ sql(s"CREATE TABLE $tbl (id bigint, data string) PARTITIONED BY (id)")
+ sql(s"INSERT INTO $tbl VALUES (1, 'a'), (2, 'b'), (3, 'c')")
+ checkAnswer(
+ spark.table(tbl).select("index", "_partition"),
+ Seq(Row(0, "3"), Row(0, "2"), Row(0, "1"))
+ )
+
+ checkAnswer(
+ spark.table(tbl).select("id", "index", "_partition"),
+ Seq(Row(3, 0, "3"), Row(2, 0, "2"), Row(1, 0, "1"))
+ )
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org