You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by vi...@apache.org on 2025/07/16 07:09:38 UTC
(spark) branch master updated: [SPARK-52809][SQL] Don't hold reader and iterator references for all partitions in task completion listeners for metric update
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cea0051f71e8 [SPARK-52809][SQL] Don't hold reader and iterator references for all partitions in task completion listeners for metric update
cea0051f71e8 is described below
commit cea0051f71e8b50e8b9e19fb0ff4fb100e82e3d0
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Wed Jul 16 00:09:26 2025 -0700
[SPARK-52809][SQL] Don't hold reader and iterator references for all partitions in task completion listeners for metric update
### What changes were proposed in this pull request?
This patch adds only one task completion listener for metric updating for `DataSourceRDD`, instead of adding separate one for each partition iterator.
### Why are the changes needed?
For each partition iterator, currently we add one task completion listener used to update final metrics if the task is stopped early. In the listener, the reader and iterator are held. So if the partition is normally exhausted, the references cannot be released early. It is a problem especially if the references are heavy as reported by https://github.com/apache/iceberg/issues/13297.
Since the purpose of the callback is to update the final metrics if the task is stopped early, it means that we only need to do it for the last partition iterator. So we don't need set up a listener for each partition iterator. Thus, we can set up just one listener for all iterator. Once we advance to next partition iterator, we can update the update target (reader and partition iterator).
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing tests.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #51503 from viirya/fix_metric_callback.
Authored-by: Liang-Chi Hsieh <vi...@gmail.com>
Signed-off-by: Liang-Chi Hsieh <vi...@gmail.com>
---
.../execution/datasources/v2/DataSourceRDD.scala | 42 +++++++++++++++++-----
1 file changed, 34 insertions(+), 8 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
index e46c0806ba2d..6a07d3c3931a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala
@@ -61,6 +61,14 @@ class DataSourceRDD(
private var currentIter: Option[Iterator[Object]] = None
private var currentIndex: Int = 0
+ private val partitionMetricCallback = new PartitionMetricCallback(customMetrics)
+
+ // In case of early stopping before consuming the entire iterator,
+ // we need to do one more metric update at the end of the task.
+ context.addTaskCompletionListener[Unit] { _ =>
+ partitionMetricCallback.execute()
+ }
+
override def hasNext: Boolean = currentIter.exists(_.hasNext) || advanceToNextIter()
override def next(): Object = {
@@ -87,14 +95,10 @@ class DataSourceRDD(
new PartitionIterator[InternalRow](rowReader, customMetrics))
(iter, rowReader)
}
- context.addTaskCompletionListener[Unit] { _ =>
- // In case of early stopping before consuming the entire iterator,
- // we need to do one more metric update at the end of the task.
- CustomMetrics
- .updateMetrics(reader.currentMetricsValues.toImmutableArraySeq, customMetrics)
- iter.forceUpdateMetrics()
- reader.close()
- }
+
+ // Once we advance to the next partition, update the metric callback for early finish
+ partitionMetricCallback.advancePartition(iter, reader)
+
currentIter = Some(iter)
hasNext
}
@@ -109,6 +113,28 @@ class DataSourceRDD(
}
}
+private class PartitionMetricCallback
+ (customMetrics: Map[String, SQLMetric]) {
+ private var iter: MetricsIterator[_] = null
+ private var reader: PartitionReader[_] = null
+
+ def advancePartition(iter: MetricsIterator[_], reader: PartitionReader[_]): Unit = {
+ execute()
+
+ this.iter = iter
+ this.reader = reader
+ }
+
+ def execute(): Unit = {
+ if (iter != null && reader != null) {
+ CustomMetrics
+ .updateMetrics(reader.currentMetricsValues.toImmutableArraySeq, customMetrics)
+ iter.forceUpdateMetrics()
+ reader.close()
+ }
+ }
+}
+
private class PartitionIterator[T](
reader: PartitionReader[T],
customMetrics: Map[String, SQLMetric]) extends Iterator[T] {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org