You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/26 03:04:06 UTC
(datafusion) 01/01: Fix topk emitted batches metric (#24481)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24481-1f5be0e70d0ea9705544b74f7ad83bd2130bbcb4
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 54557c33d81a0bc8c3683fd3126533a98dfa36e5
Author: Peter L <ce...@hotmail.com>
AuthorDate: Wed Aug 26 03:03:41 2026 +0000
Fix topk emitted batches metric (#24481)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes https://github.com/apache/datafusion/issues/24468
- Closes https://github.com/apache/datafusion/issues/24470
## Rationale for this change
Fixes some issues around metrics and TopK execution nodes
## What changes are included in this PR?
Adjusts to include metrics in PartitionedTopKExec and also fixes the
batch count in TopK
## Are these changes tested?
Yes new tests added.
## Are there any user-facing changes?
No new traits, etc.. just fixes some bugs on the physical plan side
---
.../core/tests/physical_optimizer/window_topn.rs | 69 +++++++++++++
.../physical-plan/src/sorts/partitioned_topk.rs | 6 +-
datafusion/physical-plan/src/topk/mod.rs | 113 +++++++++++++++++++--
3 files changed, 180 insertions(+), 8 deletions(-)
diff --git a/datafusion/core/tests/physical_optimizer/window_topn.rs b/datafusion/core/tests/physical_optimizer/window_topn.rs
index a31e7a45e5..85801e84c9 100644
--- a/datafusion/core/tests/physical_optimizer/window_topn.rs
+++ b/datafusion/core/tests/physical_optimizer/window_topn.rs
@@ -20,6 +20,7 @@
use std::sync::Arc;
use arrow::datatypes::{DataType, Field, Schema};
+use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_common::Result;
use datafusion_common::ScalarValue;
use datafusion_common::config::ConfigOptions;
@@ -32,10 +33,13 @@ use datafusion_physical_expr::window::StandardWindowExpr;
use datafusion_physical_expr_common::sort_expr::{LexOrdering, PhysicalSortExpr};
use datafusion_physical_optimizer::PhysicalOptimizerRule;
use datafusion_physical_optimizer::window_topn::WindowTopN;
+use datafusion_physical_plan::collect;
use datafusion_physical_plan::displayable;
use datafusion_physical_plan::filter::FilterExec;
+use datafusion_physical_plan::metrics::MetricValue;
use datafusion_physical_plan::placeholder_row::PlaceholderRowExec;
use datafusion_physical_plan::projection::ProjectionExec;
+use datafusion_physical_plan::sorts::partitioned_topk::PartitionedTopKExec;
use datafusion_physical_plan::sorts::sort::SortExec;
use datafusion_physical_plan::windows::{BoundedWindowAggExec, create_udwf_window_expr};
use datafusion_physical_plan::{ExecutionPlan, InputOrderMode};
@@ -658,3 +662,68 @@ fn predicate_lt_1_no_change() -> Result<()> {
}
Ok(())
}
+
+// ----------------------------------------------------------------------
+// Execution: metrics exposure
+// ----------------------------------------------------------------------
+
+/// Recursively finds the `PartitionedTopKExec` node in `plan`.
+fn find_partitioned_topk(
+ plan: &Arc<dyn ExecutionPlan>,
+) -> Option<Arc<dyn ExecutionPlan>> {
+ if plan.is::<PartitionedTopKExec>() {
+ return Some(Arc::clone(plan));
+ }
+ plan.children().into_iter().find_map(find_partitioned_topk)
+}
+
+/// Regression test for #24470: `PartitionedTopKExec` builds an
+/// `ExecutionPlanMetricsSet` and hands it to the top-K state, but used to omit
+/// `ExecutionPlan::metrics()`, so nothing it recorded was ever observable (and
+/// `EXPLAIN ANALYZE` showed no metrics for the operator).
+///
+/// The `output_batches` assertion also covers the emit-side counting fixed for
+/// #24468: the per-partition heaps are coalesced into `batch_size` batches, so
+/// the metric must count the coalesced output, not the per-partition inputs.
+#[tokio::test]
+async fn partitioned_topk_exec_exposes_metrics() -> Result<()> {
+ let mut config = SessionConfig::new()
+ .with_batch_size(10)
+ .with_target_partitions(1);
+ config.options_mut().optimizer.enable_window_topn = true;
+ let ctx = SessionContext::new_with_config(config);
+
+ // 10 partition keys x 5 rows each; top-5 per key keeps all 50 rows, which
+ // at batch_size 10 must be emitted as 5 batches.
+ ctx.sql(
+ "CREATE TABLE t AS
+ SELECT v % 10 AS pk, v AS val FROM (SELECT unnest(range(0, 50)) AS v)",
+ )
+ .await?
+ .collect()
+ .await?;
+
+ let df = ctx
+ .sql(
+ "SELECT pk, val FROM (
+ SELECT *, ROW_NUMBER() OVER (PARTITION BY pk ORDER BY val) AS rn FROM t
+ ) WHERE rn <= 5",
+ )
+ .await?;
+ let plan = df.create_physical_plan().await?;
+ let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?;
+ assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 50);
+
+ let topk = find_partitioned_topk(&plan).expect("plan has a PartitionedTopKExec");
+ let metrics = topk
+ .metrics()
+ .expect("PartitionedTopKExec should expose metrics");
+ assert_eq!(metrics.output_rows(), Some(50));
+ let output_batches = metrics
+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
+ .expect("output_batches metric")
+ .as_usize();
+ assert_eq!(output_batches, 5);
+
+ Ok(())
+}
diff --git a/datafusion/physical-plan/src/sorts/partitioned_topk.rs b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
index cd8bd340d2..f87a99944f 100644
--- a/datafusion/physical-plan/src/sorts/partitioned_topk.rs
+++ b/datafusion/physical-plan/src/sorts/partitioned_topk.rs
@@ -46,7 +46,7 @@ use futures::StreamExt;
use futures::TryStreamExt;
use crate::execution_plan::{Boundedness, EmissionType};
-use crate::metrics::ExecutionPlanMetricsSet;
+use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet};
use crate::topk::{
PartitionedTopK, PartitionedTopKDenseRank, PartitionedTopKRank, build_sort_fields,
};
@@ -463,6 +463,10 @@ impl ExecutionPlan for PartitionedTopKExec {
stream,
)))
}
+
+ fn metrics(&self) -> Option<MetricsSet> {
+ Some(self.metrics_set.clone_inner())
+ }
}
/// Read all input, feed each batch into a per-partition top-K state
diff --git a/datafusion/physical-plan/src/topk/mod.rs b/datafusion/physical-plan/src/topk/mod.rs
index 030614eed0..940033857d 100644
--- a/datafusion/physical-plan/src/topk/mod.rs
+++ b/datafusion/physical-plan/src/topk/mod.rs
@@ -810,14 +810,15 @@ impl TopK {
// break into record batches as needed
let mut batches = vec![];
if let Some(mut batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
-
loop {
if batch.num_rows() <= batch_size {
+ (&batch).record_output(&metrics.baseline);
batches.push(Ok(batch));
break;
} else {
- batches.push(Ok(batch.slice(0, batch_size)));
+ let head = batch.slice(0, batch_size);
+ (&head).record_output(&metrics.baseline);
+ batches.push(Ok(head));
let remaining_length = batch.num_rows() - batch_size;
batch = batch.slice(batch_size, remaining_length);
}
@@ -1430,7 +1431,6 @@ impl PartitionedTopK {
for pk in sorted_pks {
let mut heap = heaps.remove(&pk).expect("key from heaps.keys()");
if let Some(batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
coalescer.push_batch(batch)?;
}
}
@@ -1438,6 +1438,7 @@ impl PartitionedTopK {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -1775,13 +1776,11 @@ impl PartitionedTopKRank {
let RankPartitionState { mut heap, ties } =
states.remove(&pk).expect("key from states.keys()");
if let Some(batch) = heap.emit()? {
- (&batch).record_output(&metrics.baseline);
coalescer.push_batch(batch)?;
}
for tie in ties {
let indices = UInt32Array::from(tie.row_indices);
let tie_batch = take_record_batch(&tie.batch, &indices)?;
- (&tie_batch).record_output(&metrics.baseline);
coalescer.push_batch(tie_batch)?;
}
}
@@ -1789,6 +1788,7 @@ impl PartitionedTopKRank {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -2206,7 +2206,6 @@ impl PartitionedTopKDenseRank {
.batch;
let indices = UInt32Array::from(entry.row_indices);
let sub = take_record_batch(batch, &indices)?;
- (&sub).record_output(&metrics.baseline);
coalescer.push_batch(sub)?;
}
}
@@ -2215,6 +2214,7 @@ impl PartitionedTopKDenseRank {
let mut out: Vec<Result<RecordBatch>> = Vec::new();
while let Some(b) = coalescer.next_completed_batch() {
+ (&b).record_output(&metrics.baseline);
out.push(Ok(b));
}
@@ -2258,6 +2258,7 @@ impl PartitionedTopKDenseRank {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::metrics::MetricValue;
use arrow::array::{BooleanArray, Float64Array, Int32Array, StringArray};
use arrow::datatypes::{DataType, Field, Schema};
use arrow_schema::SortOptions;
@@ -2361,6 +2362,54 @@ mod tests {
)
}
+ /// Reads the `output_batches` and `output_rows` metrics recorded by an emit.
+ fn output_batches_and_rows(metrics: &ExecutionPlanMetricsSet) -> (usize, usize) {
+ let metrics = metrics.clone_inner();
+ let batches = metrics
+ .sum(|m| matches!(m.value(), MetricValue::OutputBatches(_)))
+ .expect("output_batches metric")
+ .as_usize();
+ (batches, metrics.output_rows().expect("output_rows metric"))
+ }
+
+ /// Regression test for #24468: `emit` splits the heap's single batch into
+ /// `batch_size` chunks, so `output_batches` must count each emitted chunk
+ /// rather than the one pre-split batch.
+ #[tokio::test]
+ async fn test_topk_output_batches_metric_counts_emitted_batches() -> Result<()> {
+ let schema = make_ab_schema();
+ let metrics = ExecutionPlanMetricsSet::new();
+ let sort_expr = PhysicalSortExpr {
+ expr: col("a", schema.as_ref())?,
+ options: SortOptions::default(),
+ };
+ // k = 5 with batch_size = 2 => emitted batches of [2, 2, 1]
+ let mut topk = TopK::try_new(
+ 0,
+ Arc::clone(&schema),
+ vec![],
+ LexOrdering::from([sort_expr]),
+ 5,
+ 2,
+ Arc::new(RuntimeEnv::default()),
+ &metrics,
+ make_topk_filter(),
+ )?;
+
+ topk.insert_batch(make_ab_batch(
+ Arc::clone(&schema),
+ &[Some(5), Some(4), Some(3), Some(2), Some(1), Some(0)],
+ &[0.0, 1.0, 2.0, 3.0, 4.0, 5.0],
+ )?)?;
+
+ let results: Vec<_> = topk.emit()?.try_collect().await?;
+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
+ assert_eq!(row_counts, vec![2, 2, 1]);
+ assert_eq!(output_batches_and_rows(&metrics), (3, 5));
+
+ Ok(())
+ }
+
fn make_ab_batch(
schema: SchemaRef,
a: &[Option<i32>],
@@ -2980,6 +3029,56 @@ mod tests {
)?)
}
+ /// Companion to [`test_topk_output_batches_metric_counts_emitted_batches`] for
+ /// the partitioned emit path: rows from several per-partition heaps are
+ /// coalesced into `batch_size` batches, so the metric must count the
+ /// coalesced output rather than the per-partition inputs.
+ #[tokio::test]
+ async fn test_partitioned_topk_output_batches_metric_counts_emitted_batches()
+ -> Result<()> {
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("pk", DataType::Int32, false),
+ Field::new("val", DataType::Int32, false),
+ ]));
+ let pk_expr: Arc<dyn PhysicalExpr> = col("pk", schema.as_ref())?;
+ let partition_sort_fields = build_sort_fields(
+ &[PhysicalSortExpr {
+ expr: Arc::clone(&pk_expr),
+ options: SortOptions::default(),
+ }],
+ &schema,
+ )?;
+ let metrics = ExecutionPlanMetricsSet::new();
+ // 3 per-partition heaps of 2 rows each, coalesced into 2 batches of 3
+ let mut state = PartitionedTopK::try_new(
+ 0,
+ Arc::clone(&schema),
+ vec![pk_expr],
+ partition_sort_fields,
+ LexOrdering::from([PhysicalSortExpr {
+ expr: col("val", schema.as_ref())?,
+ options: SortOptions::default(),
+ }]),
+ 2,
+ 3, // batch_size
+ &Arc::new(RuntimeEnv::default()),
+ &metrics,
+ )?;
+
+ state.insert_batch(&pk_val_batch(
+ &schema,
+ vec![1, 1, 2, 2, 3, 3],
+ vec![10, 5, 20, 15, 30, 25],
+ )?)?;
+
+ let results: Vec<_> = state.emit()?.try_collect().await?;
+ let row_counts: Vec<usize> = results.iter().map(|b| b.num_rows()).collect();
+ assert_eq!(row_counts, vec![3, 3]);
+ assert_eq!(output_batches_and_rows(&metrics), (2, 6));
+
+ Ok(())
+ }
+
/// Multiple distinct partition keys interleaved within a single
/// input batch — grouping rows by partition key, per-partition heap eviction,
/// and partition-key-ordered emit must all behave correctly.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org