You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2025/04/04 15:57:36 UTC
(spark) branch master updated: [SPARK-51704][SQL] Eliminate unnecessary collect operation
This is an automated email from the ASF dual-hosted git repository.
yangjie01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new eae5ca789f61 [SPARK-51704][SQL] Eliminate unnecessary collect operation
eae5ca789f61 is described below
commit eae5ca789f61f44cde75fc4d1c2230a01d9ebbd8
Author: summaryzb <su...@gmail.com>
AuthorDate: Fri Apr 4 23:57:25 2025 +0800
[SPARK-51704][SQL] Eliminate unnecessary collect operation
### What changes were proposed in this pull request?
Change operation for `TreeNode` from `collect` to `collectFirst` when in below scenarios:
- The final purpose is to find the first qualified node in a pre order way
- The final purpose is to verify that there is no one or at least one node which satisfy the requirement
Two factors should be satisfied:
- Apply certain operation recursively on the `TreeNode` including itself
- Partial function applied should not effect the node or other related object
### Why are the changes needed?
Avoid unnecessary Traversal of the `TreeNode`
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
- Pass GitHub Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #50494 from summaryzb/sql_collect.
Authored-by: summaryzb <su...@gmail.com>
Signed-off-by: yangjie01 <ya...@baidu.com>
---
.../spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala | 2 +-
.../org/apache/spark/sql/kafka010/KafkaRelationSuite.scala | 4 ++--
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 2 +-
.../spark/sql/execution/streaming/ProgressReporter.scala | 2 +-
.../test/scala/org/apache/spark/sql/CTEInlineSuite.scala | 4 ++--
.../test/scala/org/apache/spark/sql/CachedTableSuite.scala | 4 ++--
.../org/apache/spark/sql/DatasetOptimizationSuite.scala | 14 ++++++++------
.../src/test/scala/org/apache/spark/sql/QueryTest.scala | 2 +-
.../test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 10 +++++-----
.../test/scala/org/apache/spark/sql/SubquerySuite.scala | 2 +-
.../test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala | 14 +++++++-------
.../org/apache/spark/sql/sources/BucketedReadSuite.scala | 4 ++--
12 files changed, 33 insertions(+), 31 deletions(-)
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
index 22eeae97874b..e738abf21f59 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaMicroBatchSourceSuite.scala
@@ -1614,7 +1614,7 @@ abstract class KafkaMicroBatchV1SourceSuite extends KafkaMicroBatchSourceSuiteBa
testStream(kafka)(
makeSureGetOffsetCalled,
AssertOnQuery { query =>
- query.logicalPlan.collect {
+ query.logicalPlan.collectFirst {
case StreamingExecutionRelation(_: KafkaSource, _, _) => true
}.nonEmpty
}
diff --git a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
index 6087447fa304..1a884533e818 100644
--- a/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
+++ b/connector/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaRelationSuite.scala
@@ -637,7 +637,7 @@ class KafkaRelationSuiteV1 extends KafkaRelationSuiteBase {
test("V1 Source is used when set through SQLConf") {
val topic = newTopic()
val df = createDF(topic)
- assert(df.logicalPlan.collect {
+ assert(df.logicalPlan.collectFirst {
case _: LogicalRelation => true
}.nonEmpty)
}
@@ -652,7 +652,7 @@ class KafkaRelationSuiteV2 extends KafkaRelationSuiteBase {
test("V2 Source is used when set through SQLConf") {
val topic = newTopic()
val df = createDF(topic)
- assert(df.logicalPlan.collect {
+ assert(df.logicalPlan.collectFirst {
case _: DataSourceV2Relation => true
}.nonEmpty)
}
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index 47db416e2d78..beef278a3dfe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1833,7 +1833,7 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
* Returns true if `exprs` contains a [[Star]].
*/
def containsStar(exprs: Seq[Expression]): Boolean =
- exprs.exists(_.collect { case _: Star => true }.nonEmpty)
+ exprs.exists(_.collectFirst { case _: Star => true }.nonEmpty)
private def extractStar(exprs: Seq[Expression]): Seq[Star] =
exprs.flatMap(_.collect { case s: Star => s })
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
index 3ac07cf1d730..d814a86c84c7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/ProgressReporter.scala
@@ -559,7 +559,7 @@ abstract class ProgressContext(
hasNewData: Boolean,
sourceToNumInputRows: Map[SparkDataStream, Long],
lastExecution: IncrementalExecution): ExecutionStats = {
- val hasEventTime = progressReporter.logicalPlan().collect {
+ val hasEventTime = progressReporter.logicalPlan().collectFirst {
case e: EventTimeWatermark => e
}.nonEmpty
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
index 80450b79a9a7..2333b82e11d4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CTEInlineSuite.scala
@@ -170,7 +170,7 @@ abstract class CTEInlineSuiteBase
}.head.length == 2,
"With-CTE should contain 2 CTE defs after analysis.")
assert(
- df.queryExecution.optimizedPlan.collect {
+ df.queryExecution.optimizedPlan.collectFirst {
case r: RepartitionOperation => r
}.isEmpty,
"CTEs with one reference should all be inlined after optimization.")
@@ -255,7 +255,7 @@ abstract class CTEInlineSuiteBase
}.head.length == 2,
"With-CTE should contain 2 CTE defs after analysis.")
assert(
- df.queryExecution.optimizedPlan.collect {
+ df.queryExecution.optimizedPlan.collectFirst {
case r: RepartitionOperation => r
}.isEmpty,
"Deterministic CTEs should all be inlined after optimization.")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index e0ad3feda3ac..68299804ea87 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -508,7 +508,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
|join abc c on a.key=c.key""".stripMargin).queryExecution.sparkPlan
assert(sparkPlan.collect { case e: InMemoryTableScanExec => e }.size === 3)
- assert(sparkPlan.collect { case e: RDDScanExec => e }.size === 0)
+ assert(sparkPlan.collectFirst { case e: RDDScanExec => e }.isEmpty)
}
}
@@ -923,7 +923,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
withSQLConf(SQLConf.CACHE_VECTORIZED_READER_ENABLED.key -> vectorized.toString) {
val cache = spark.range(10).cache()
val df = cache.filter($"id" > 0)
- val columnarToRow = df.queryExecution.executedPlan.collect {
+ val columnarToRow = df.queryExecution.executedPlan.collectFirst {
case c: ColumnarToRowExec => c
}
assert(columnarToRow.isEmpty)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
index 5db3990b67c8..8022bfdf99cc 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetOptimizationSuite.scala
@@ -32,10 +32,11 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
test("SPARK-26619: Prune the unused serializers from SerializeFromObject") {
val data = Seq(("a", 1), ("b", 2), ("c", 3))
val ds = data.toDS().map(t => (t._1, t._2 + 1)).select("_1")
- val serializer = ds.queryExecution.optimizedPlan.collect {
+ val serializerOpt = ds.queryExecution.optimizedPlan.collectFirst {
case s: SerializeFromObject => s
- }.head
- assert(serializer.serializer.size == 1)
+ }
+ assert(serializerOpt.isDefined)
+ assert(serializerOpt.get.serializer.size == 1)
checkAnswer(ds, Seq(Row("a"), Row("b"), Row("c")))
}
@@ -45,15 +46,16 @@ class DatasetOptimizationSuite extends QueryTest with SharedSparkSession {
// serializers. The first `structFields` is aligned with first serializer and ditto
// for other `structFields`.
private def testSerializer(df: DataFrame, structFields: Seq[Seq[String]]*): Unit = {
- val serializer = df.queryExecution.optimizedPlan.collect {
+ val serializerOpt = df.queryExecution.optimizedPlan.collectFirst {
case s: SerializeFromObject => s
- }.head
+ }
def collectNamedStruct: PartialFunction[Expression, Seq[CreateNamedStruct]] = {
case c: CreateNamedStruct => Seq(c)
}
- serializer.serializer.zip(structFields).foreach { case (ser, fields) =>
+ assert(serializerOpt.isDefined)
+ serializerOpt.get.serializer.zip(structFields).foreach { case (ser, fields) =>
val structs: Seq[CreateNamedStruct] = ser.collect(collectNamedStruct).flatten
assert(structs.size == fields.size)
structs.zip(fields).foreach { case (struct, fieldNames) =>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
index 71af79895d20..37233dbd7b5c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/QueryTest.scala
@@ -285,7 +285,7 @@ object QueryTest extends Assertions {
df: DataFrame,
expectedAnswer: Seq[Row],
checkToRDD: Boolean = true): Option[String] = {
- val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty
+ val isSorted = df.logicalPlan.collectFirst { case s: logical.Sort => s }.nonEmpty
if (checkToRDD) {
SQLExecution.withSQLConfPropagated(df.sparkSession) {
df.materializedRdd.count() // Also attempt to deserialize as an RDD [SPARK-15791]
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index ce6440e90103..3fc440983a68 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -821,11 +821,11 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
case cp: CartesianProductExec => cp
}
assert(cp.isEmpty, "should not use CartesianProduct for null-safe join")
- val smj = df.queryExecution.sparkPlan.collect {
+ val smj = df.queryExecution.sparkPlan.collectFirst {
case smj: SortMergeJoinExec => smj
case j: BroadcastHashJoinExec => j
}
- assert(smj.size > 0, "should use SortMergeJoin or BroadcastHashJoin")
+ assert(smj.nonEmpty, "should use SortMergeJoin or BroadcastHashJoin")
checkAnswer(df, Row(100) :: Nil)
}
@@ -3815,7 +3815,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
Seq(1, "1, 2", null, "version()").foreach { expr =>
val plan = sql(s"select * from values (1), (2), (3) t(a) distribute by $expr")
.queryExecution.optimizedPlan
- val res = plan.collect {
+ val res = plan.collectFirst {
case r: RepartitionByExpression if r.numPartitions == 1 => true
}
assert(res.nonEmpty)
@@ -3827,7 +3827,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
withSQLConf((SQLConf.SHUFFLE_PARTITIONS.key, "5")) {
val df = spark.range(1).hint("REPARTITION_BY_RANGE")
val plan = df.queryExecution.optimizedPlan
- val res = plan.collect {
+ val res = plan.collectFirst {
case r: RepartitionByExpression if r.numPartitions == 5 => true
}
assert(res.nonEmpty)
@@ -3839,7 +3839,7 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark
Seq(1, "1, 2", null, "version()").foreach { expr =>
val plan = sql(s"select * from values (1), (2), (3) t(a) distribute by $expr")
.queryExecution.analyzed
- val res = plan.collect {
+ val res = plan.collectFirst {
case r: RepartitionByExpression if r.numPartitions == 2 => true
}
assert(res.nonEmpty)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
index e7e41f6570d3..7e8dd439161b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SubquerySuite.scala
@@ -2722,7 +2722,7 @@ class SubquerySuite extends QueryTest
|SELECT * FROM v1 WHERE kind = (SELECT kind FROM v1 WHERE kind = 'foo')
|""".stripMargin)
val df = sql("SELECT * FROM v1 JOIN v2 ON v1.id = v2.id")
- val filter = df.queryExecution.optimizedPlan.collect {
+ val filter = df.queryExecution.optimizedPlan.collectFirst {
case f: Filter => f
}
assert(filter.isEmpty,
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
index 0b729147c120..ac087562824d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
@@ -300,7 +300,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}
private def checkLimitRemoved(df: DataFrame, removed: Boolean = true): Unit = {
- val limits = df.queryExecution.optimizedPlan.collect {
+ val limits = df.queryExecution.optimizedPlan.collectFirst {
case g: GlobalLimit => g
case limit: LocalLimit => limit
}
@@ -382,7 +382,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}
private def checkOffsetRemoved(df: DataFrame, removed: Boolean = true): Unit = {
- val offsets = df.queryExecution.optimizedPlan.collect {
+ val offsets = df.queryExecution.optimizedPlan.collectFirst {
case offset: Offset => offset
}
if (removed) {
@@ -814,7 +814,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}
private def checkSortRemoved(df: DataFrame, removed: Boolean = true): Unit = {
- val sorts = df.queryExecution.optimizedPlan.collect {
+ val sorts = df.queryExecution.optimizedPlan.collectFirst {
case s: Sort => s
}
if (removed) {
@@ -1851,7 +1851,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}
private def checkAggregateRemoved(df: DataFrame, removed: Boolean = true): Unit = {
- val aggregates = df.queryExecution.optimizedPlan.collect {
+ val aggregates = df.queryExecution.optimizedPlan.collectFirst {
case agg: Aggregate => agg
}
if (removed) {
@@ -1948,7 +1948,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
}
private def checkFiltersRemoved(df: DataFrame, removed: Boolean = true): Unit = {
- val filters = df.queryExecution.optimizedPlan.collect {
+ val filters = df.queryExecution.optimizedPlan.collectFirst {
case f: Filter => f
}
if (removed) {
@@ -2245,7 +2245,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
test("scan with aggregate push-down: with concat multiple group key in project") {
val df1 = sql("SELECT concat_ws('#', DEPT, NAME), MAX(SALARY) FROM h2.test.employee" +
" WHERE dept > 0 GROUP BY DEPT, NAME")
- val filters1 = df1.queryExecution.optimizedPlan.collect {
+ val filters1 = df1.queryExecution.optimizedPlan.collectFirst {
case f: Filter => f
}
assert(filters1.isEmpty)
@@ -2259,7 +2259,7 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
val df2 = sql("SELECT concat_ws('#', DEPT, NAME), MAX(SALARY) + MIN(BONUS)" +
" FROM h2.test.employee WHERE dept > 0 GROUP BY DEPT, NAME")
- val filters2 = df2.queryExecution.optimizedPlan.collect {
+ val filters2 = df2.queryExecution.optimizedPlan.collectFirst {
case f: Filter => f
}
assert(filters2.isEmpty)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
index c4b09c4b289e..83ebd24384c2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/BucketedReadSuite.scala
@@ -690,7 +690,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
val t1 = spark.table("t")
val t2 = t1.selectExpr("i as ii")
val plan = t1.join(t2, t1("i") === t2("ii")).queryExecution.executedPlan
- assert(plan.collect { case sort: SortExec => sort }.isEmpty)
+ assert(plan.collectFirst { case sort: SortExec => sort }.isEmpty)
}
}
}
@@ -703,7 +703,7 @@ abstract class BucketedReadSuite extends QueryTest with SQLTestUtils with Adapti
sql("CREATE VIEW v AS SELECT * FROM t").collect()
val plan = sql("SELECT * FROM t a JOIN v b ON a.i = b.i").queryExecution.executedPlan
- assert(plan.collect { case exchange: ShuffleExchangeExec => exchange }.isEmpty)
+ assert(plan.collectFirst { case exchange: ShuffleExchangeExec => exchange }.isEmpty)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org