You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/09/10 09:13:37 UTC
(datafusion) branch gh-readonly-queue/main/pr-24853-b3cb365dd6e5e0441f3f58817fee55dae1814fdb created (now b8c41877ba)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-24853-b3cb365dd6e5e0441f3f58817fee55dae1814fdb
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at b8c41877ba test: add regression coverage for terminal hash aggregate drain (#24853)
This branch includes the following new commits:
new b8c41877ba test: add regression coverage for terminal hash aggregate drain (#24853)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: test: add regression coverage for terminal hash aggregate drain (#24853)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-24853-b3cb365dd6e5e0441f3f58817fee55dae1814fdb
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit b8c41877ba377c9c165b96b57296d9396d917807
Author: Huang Qiwei <qi...@qq.com>
AuthorDate: Thu Sep 10 09:13:14 2026 +0000
test: add regression coverage for terminal hash aggregate drain (#24853)
## Which issue does this PR close?
- Closes #24849.
## Rationale for this change
#23182 fixed a performance regression in terminal hash aggregate output
by avoiding repeated destructive `EmitTo::First` calls. However, the
remaining `MaterializedAggregateOutput` unit test only exercises the
slicing helper and does not verify the behavior through `AggregateExec`.
A result-only integration test would pass with both the regressed and
fixed implementations, while a wall-clock assertion would be unreliable
in CI. This PR therefore uses a test `GroupsAccumulator` that returns an
error if terminal output falls back to destructive `EmitTo::First`.
## What changes are included in this PR?
- Adds `single_grouped_aggregate_avoids_destructive_terminal_drain`,
which runs unordered Single hash aggregation with more groups than the
configured `batch_size`.
- Verifies that `AggregateExec` selects `SingleHash`, emits multiple
bounded batches, and returns the correct aggregate results for repeated
group keys.
- Keeps the test independent of `MaterializedAggregateOutput`; Single
and Final hash aggregation exercise the same terminal drain helper.
## What is the testing strategy for this PR?
- Verified that temporarily restoring destructive
`EmitTo::First(batch_size)` causes the new test to fail with the
expected accumulator error, while the current implementation passes.
- Applied an equivalent FinalHash accumulator oracle to the historical
regression: it fails on `d58e0c6d4^` and passes on `d58e0c6d4`. The
migrated SingleHash stream did not exist at that historical revision.
- Ran `cargo fmt --all`.
- Ran `cargo clippy --all-targets --all-features -- -D warnings`.
- Ran the extended workspace test suite with
`avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption`;
all tests passed after initializing the required Arrow and Parquet test
fixtures.
## Are there any user-facing changes?
No. This PR only adds regression test coverage.
---
datafusion/physical-plan/src/aggregates/mod.rs | 68 +++++++++++++++++++++++++-
1 file changed, 67 insertions(+), 1 deletion(-)
diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs
index fc6a676e95..e6ad514556 100644
--- a/datafusion/physical-plan/src/aggregates/mod.rs
+++ b/datafusion/physical-plan/src/aggregates/mod.rs
@@ -4352,6 +4352,72 @@ mod tests {
Ok(())
}
+ /// Single and Final hash aggregation share the same terminal drain helper. Single
+ /// exercises it directly from raw input without coupling this test to partial state.
+ #[tokio::test]
+ async fn single_grouped_aggregate_avoids_destructive_terminal_drain() -> Result<()> {
+ const BATCH_SIZE: usize = 2;
+ const NUM_GROUPS: usize = 3;
+
+ let schema = Arc::new(Schema::new(vec![
+ Field::new("key", DataType::Int32, false),
+ Field::new("value", DataType::Int32, false),
+ ]));
+ let input_batches = vec![RecordBatch::try_new(
+ Arc::clone(&schema),
+ vec![
+ Arc::new(Int32Array::from(vec![1, 2, 3, 1, 2, 3])),
+ Arc::new(Int32Array::from(vec![10, 20, 30, 40, 50, 60])),
+ ],
+ )?];
+ let input =
+ TestMemoryExec::try_new_exec(&[input_batches], Arc::clone(&schema), None)?;
+ let group_by =
+ PhysicalGroupBy::new_single(vec![(col("key", &schema)?, "key".to_string())]);
+ let udaf = Arc::new(AggregateUDF::from(NoFirstEmitUdaf::new()));
+ let aggregates: Vec<Arc<AggregateFunctionExpr>> = vec![Arc::new(
+ AggregateExprBuilder::new(udaf, vec![col("value", &schema)?])
+ .schema(Arc::clone(&schema))
+ .alias("no_first_emit(value)")
+ .build()?,
+ )];
+ let aggregate = Arc::new(AggregateExec::try_new(
+ AggregateMode::Single,
+ group_by,
+ aggregates,
+ vec![None],
+ input,
+ Arc::clone(&schema),
+ )?);
+ let task_ctx = new_migrated_hash_ctx(BATCH_SIZE);
+
+ let stream = aggregate.execute_typed(0, &task_ctx)?;
+ assert!(matches!(stream, StreamType::SingleHash(_)));
+
+ let stream: SendableRecordBatchStream = stream.into();
+ let batches = collect(stream).await?;
+ assert!(batches.len() > 1, "expected terminal output to be chunked");
+ assert!(
+ batches.iter().all(|batch| batch.num_rows() <= BATCH_SIZE),
+ "terminal output exceeded the configured batch size"
+ );
+ assert_eq!(
+ batches.iter().map(RecordBatch::num_rows).sum::<usize>(),
+ NUM_GROUPS
+ );
+ assert_snapshot!(batches_to_sort_string(&batches), @r"
+ +-----+----------------------+
+ | key | no_first_emit(value) |
+ +-----+----------------------+
+ | 1 | 2 |
+ | 2 | 2 |
+ | 3 | 2 |
+ +-----+----------------------+
+ ");
+
+ Ok(())
+ }
+
#[tokio::test]
async fn limited_distinct_aggregate_uses_migrated_hash_streams() -> Result<()> {
let schema =
@@ -8278,7 +8344,7 @@ mod tests {
Ok(Arc::new(Int64Array::from(counts)))
}
EmitTo::First(_) => internal_err!(
- "partial grouped aggregate output must materialize with EmitTo::All before slicing"
+ "grouped aggregate terminal output must not use EmitTo::First"
),
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org