You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/08/11 12:17:20 UTC
(datafusion) 01/01: fix: avoid buffering unbounded repartition output indefinitely (#24193)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24193-d5868e683435042aa5fa72465a8a2d667d8cc468
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 047f531488f6617e88c76769baf2e2d6ac8e1c0e
Author: Goutam Adwant <86...@users.noreply.github.com>
AuthorDate: Tue Aug 11 05:16:56 2026 -0700
fix: avoid buffering unbounded repartition output indefinitely (#24193)
## Which issue does this PR close?
- Closes #24044.
## Rationale for this change
`RepartitionExec` coalesces small batches on the producer side until the
configured batch size is reached or all input senders finish. An
unbounded input may never finish, so an available partial batch can be
withheld indefinitely. This prevents an incremental query from emitting
rows that are ready for downstream processing.
## What changes are included in this PR?
- Skip producer-side batch coalescing when the input is unbounded.
- Preserve the existing coalescing behavior for bounded inputs and
preserve-order execution.
- Add a regression test with an unbounded source that emits one partial
batch and then remains open.
## Are these changes tested?
Yes.
- `cargo fmt --all -- --check`
- `cargo test -p datafusion-physical-plan`
- `cargo clippy --all-targets --all-features -- -D warnings`
- The extended workspace test command required by `AGENTS.md`, including
all 503 sqllogictest files
The regression test fails on the previous implementation because the
partial batch is never emitted, and passes with this change. The
existing bounded-input coalescing test also continues to pass.
## Are there any user-facing changes?
Yes. Repartitioning an unbounded input can now emit partial batches
promptly instead of waiting indefinitely to fill the configured batch
size. This may expose smaller batches to downstream operators. There are
no public API changes.
Signed-off-by: goutamadwant <wo...@gmail.com>
---
datafusion/physical-plan/src/repartition/mod.rs | 64 +++++++++++++++++++++++--
1 file changed, 60 insertions(+), 4 deletions(-)
diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs
index 2863524f16..51e350b2f0 100644
--- a/datafusion/physical-plan/src/repartition/mod.rs
+++ b/datafusion/physical-plan/src/repartition/mod.rs
@@ -456,6 +456,7 @@ impl RepartitionExecState {
let num_input_partitions = streams_and_metrics.len();
let num_output_partitions = partitioning.partition_count();
+ let coalesce_batches = !preserve_order && !input.boundedness().is_unbounded();
let spill_manager = Arc::new(spill_manager);
@@ -524,9 +525,10 @@ impl RepartitionExecState {
// Coalesce on the producer side, before the channel's gate, so
// the consumer never sees the per-input-task small batches.
- // Skip in preserve-order mode: each input has its own dedicated
- // channel and `StreamingMergeBuilder` handles batching.
- let shared_coalescer = (!preserve_order).then(|| {
+ // Skip in preserve-order mode, where `StreamingMergeBuilder`
+ // handles batching, and for unbounded inputs, where a residual
+ // batch could otherwise be withheld indefinitely.
+ let shared_coalescer = coalesce_batches.then(|| {
SharedCoalescer::new(
input.schema(),
context.session_config().batch_size(),
@@ -1127,7 +1129,8 @@ impl BatchPartitioner {
/// Repartitioning one [`RecordBatch`] implies creating multiple smaller batches, potentially
/// as many as the number of output partitions. [`RepartitionExec`] makes sure that the returned
/// batches adhere to the configured `datafusion.execution.batch_size` for efficient operations,
-/// and for that, it will automatically coalesce batches right after repartitioning.
+/// and for that, it will automatically coalesce batches right after repartitioning for bounded
+/// inputs. Coalescing is skipped for unbounded inputs so partial batches are emitted promptly.
///
/// For this, one shared [`LimitedBatchCoalescer`] per output partition is used:
///
@@ -2239,6 +2242,7 @@ mod tests {
use super::*;
use crate::empty::EmptyExec;
use crate::projection::ProjectionExpr;
+ use crate::streaming::{PartitionStream, StreamingTableExec};
use crate::test::TestMemoryExec;
use crate::{
test::{
@@ -2263,6 +2267,27 @@ mod tests {
use datafusion_physical_expr::{PhysicalSortExpr, RangePartitioning, SplitPoint};
use insta::assert_snapshot;
+ #[derive(Debug)]
+ struct UnboundedTestPartition {
+ schema: SchemaRef,
+ batch: RecordBatch,
+ }
+
+ impl PartitionStream for UnboundedTestPartition {
+ fn schema(&self) -> &SchemaRef {
+ &self.schema
+ }
+
+ fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {
+ let stream = futures::stream::iter([Ok(self.batch.clone())])
+ .chain(futures::stream::pending());
+ Box::pin(RecordBatchStreamAdapter::new(
+ Arc::clone(&self.schema),
+ stream,
+ ))
+ }
+ }
+
#[test]
fn strength_reduced_u64_remainder_matches_modulo() {
let divisors = [
@@ -3007,6 +3032,37 @@ mod tests {
Ok(())
}
+ #[tokio::test]
+ async fn unbounded_input_emits_before_batch_size() -> Result<()> {
+ let schema = test_schema(false);
+ let batch = create_batch();
+ let source = Arc::new(StreamingTableExec::try_new(
+ Arc::clone(&schema),
+ vec![Arc::new(UnboundedTestPartition {
+ schema: Arc::clone(&schema),
+ batch: batch.clone(),
+ })],
+ None,
+ vec![],
+ true,
+ None,
+ )?);
+ let exec = RepartitionExec::try_new(source, Partitioning::RoundRobinBatch(1))?;
+ let session_config = SessionConfig::new().with_batch_size(batch.num_rows() * 2);
+ let task_ctx =
+ Arc::new(TaskContext::default().with_session_config(session_config));
+
+ let mut stream = exec.execute(0, task_ctx)?;
+ let output =
+ tokio::time::timeout(std::time::Duration::from_secs(5), stream.next())
+ .await
+ .expect("unbounded repartition withheld a partial batch")
+ .expect("unbounded input ended unexpectedly")?;
+
+ assert_eq!(batch, output);
+ Ok(())
+ }
+
fn test_schema(nullable: bool) -> Arc<Schema> {
Arc::new(Schema::new(vec![Field::new(
"c0",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org