You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/27 17:34:44 UTC
(datafusion) branch main updated: perf: implement convert_to_state for SparkAvg (#21548)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new ca1d39dfca perf: implement convert_to_state for SparkAvg (#21548)
ca1d39dfca is described below
commit ca1d39dfca531620b69a3a17018a899bb478a361
Author: Alex Zhang <87...@users.noreply.github.com>
AuthorDate: Mon Apr 27 10:10:03 2026 -0700
perf: implement convert_to_state for SparkAvg (#21548)
## Which issue does this PR close?
- Part of #17964.
## Rationale for this change
SparkAvg's AvgGroupsAccumulator doesn't implement
supports_convert_to_state (defaults to false), which prevents the
skip-partial-aggregation optimization from kicking in for queries that
use Spark's avg().
I ran into this while benchmarking a Spark Connect engine built on
DataFusion. On TPC-H q17 at SF10, the partial aggregate for
avg(l_quantity) grouped by l_partkey (~2M groups out of 60M rows) was
not triggering skip-aggregation:
| Metric | Without convert_to_state | With convert_to_state |
|--------|-------------------------|-----------------------|
| Partial aggregate memory | 923 MB | 40 MB |
| Partial aggregate elapsed | 4.75s | 109ms |
The skip-aggregation probe (#11627) detects when a partial aggregate
isn't reducing cardinality and falls back to passing rows through as
state directly. This needs convert_to_state so the accumulator can
produce [sum, count] state arrays from raw input. The built-in Avg
already has this (#11734), but it wasn't carried over when SparkAvg was
migrated from Comet in #17871.
## What changes are included in this PR?
Adds convert_to_state() and supports_convert_to_state() to
AvgGroupsAccumulator in datafusion-spark.
Follows the same approach as the built-in Avg, adapted for SparkAvg's
differences:
- State order is [sum, count] (vs [count, sum] in the built-in)
- Count type is Int64 (vs UInt64 in the built-in)
- Null handling uses NullBuffer::union directly instead of pulling in
datafusion-functions-aggregate-common as a dep
Also cleaned up the fully-qualified arrow::array::BooleanArray
references in update_batch / merge_batch since adding BooleanArray to
the import block triggered the unused_qualifications lint.
## Are these changes tested?
Yes, unit tests covering basic conversion, null propagation, filter
handling, and a roundtrip through merge_batch to verify the converted
state produces correct results end-to-end.
## Are there any user-facing changes?
No. Queries using avg() through the Spark function registry will
automatically benefit from skip-partial-aggregation on high-cardinality
groupings.
---------
Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
Cargo.lock | 1 +
datafusion/spark/Cargo.toml | 1 +
datafusion/spark/src/function/aggregate/avg.rs | 172 +++++++++++++++++++++++--
3 files changed, 161 insertions(+), 13 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ed3c4825bd..9d60540d9c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2632,6 +2632,7 @@ dependencies = [
"datafusion-expr",
"datafusion-functions",
"datafusion-functions-aggregate",
+ "datafusion-functions-aggregate-common",
"datafusion-functions-nested",
"log",
"num-traits",
diff --git a/datafusion/spark/Cargo.toml b/datafusion/spark/Cargo.toml
index 0651fe5651..72bca58b7a 100644
--- a/datafusion/spark/Cargo.toml
+++ b/datafusion/spark/Cargo.toml
@@ -55,6 +55,7 @@ datafusion-execution = { workspace = true }
datafusion-expr = { workspace = true }
datafusion-functions = { workspace = true, features = ["crypto_expressions"] }
datafusion-functions-aggregate = { workspace = true }
+datafusion-functions-aggregate-common = { workspace = true }
datafusion-functions-nested = { workspace = true }
log = { workspace = true }
num-traits = { workspace = true }
diff --git a/datafusion/spark/src/function/aggregate/avg.rs b/datafusion/spark/src/function/aggregate/avg.rs
index c504f5387f..5f4d2c253a 100644
--- a/datafusion/spark/src/function/aggregate/avg.rs
+++ b/datafusion/spark/src/function/aggregate/avg.rs
@@ -16,7 +16,8 @@
// under the License.
use arrow::array::{
- Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, Int64Array, PrimitiveArray,
+ Array, ArrayRef, ArrowNativeTypeOp, ArrowNumericType, BooleanArray, Int64Array,
+ PrimitiveArray,
builder::PrimitiveBuilder,
cast::AsArray,
types::{Float64Type, Int64Type},
@@ -31,6 +32,9 @@ use datafusion_expr::{
Accumulator, AggregateUDFImpl, Coercion, EmitTo, GroupsAccumulator, ReversedUDAF,
Signature, TypeSignatureClass, Volatility,
};
+use datafusion_functions_aggregate_common::aggregate::groups_accumulator::nulls::{
+ filtered_null_mask, set_nulls,
+};
use std::sync::Arc;
/// AVG aggregate expression
@@ -248,7 +252,7 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- _opt_filter: Option<&arrow::array::BooleanArray>,
+ _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 1, "single argument to update_batch");
@@ -285,26 +289,26 @@ where
&mut self,
values: &[ArrayRef],
group_indices: &[usize],
- _opt_filter: Option<&arrow::array::BooleanArray>,
+ _opt_filter: Option<&BooleanArray>,
total_num_groups: usize,
) -> Result<()> {
assert_eq!(values.len(), 2, "two arguments to merge_batch");
// first batch is partial sums, second is counts
let partial_sums = values[0].as_primitive::<T>();
let partial_counts = values[1].as_primitive::<Int64Type>();
- // update counts with partial counts
- self.counts.resize(total_num_groups, 0);
- let iter1 = group_indices.iter().zip(partial_counts.values().iter());
- for (&group_index, &partial_count) in iter1 {
- self.counts[group_index] += partial_count;
- }
- // update sums
+ self.counts.resize(total_num_groups, 0);
self.sums.resize(total_num_groups, T::default_value());
- let iter2 = group_indices.iter().zip(partial_sums.values().iter());
- for (&group_index, &new_value) in iter2 {
+
+ for (idx, &group_index) in group_indices.iter().enumerate() {
+ // Skip null state entries emitted by convert_to_state for
+ // filtered / null input rows.
+ if partial_counts.is_null(idx) || partial_sums.is_null(idx) {
+ continue;
+ }
+ self.counts[group_index] += partial_counts.value(idx);
let sum = &mut self.sums[group_index];
- *sum = sum.add_wrapping(new_value);
+ *sum = sum.add_wrapping(partial_sums.value(idx));
}
Ok(())
@@ -343,7 +347,149 @@ where
])
}
+ fn convert_to_state(
+ &self,
+ values: &[ArrayRef],
+ opt_filter: Option<&BooleanArray>,
+ ) -> Result<Vec<ArrayRef>> {
+ let sums = values[0]
+ .as_primitive::<T>()
+ .clone()
+ .with_data_type(self.return_data_type.clone());
+ let counts = Int64Array::from_value(1, sums.len());
+
+ let nulls = filtered_null_mask(opt_filter, &sums);
+ let counts = set_nulls(counts, nulls.clone());
+ let sums = set_nulls(sums, nulls);
+
+ // [sum, count] - must match state() and merge_batch()
+ Ok(vec![
+ Arc::new(sums) as ArrayRef,
+ Arc::new(counts) as ArrayRef,
+ ])
+ }
+
+ fn supports_convert_to_state(&self) -> bool {
+ true
+ }
+
fn size(&self) -> usize {
self.counts.capacity() * size_of::<i64>() + self.sums.capacity() * size_of::<T>()
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use arrow::array::Float64Array;
+
+ fn make_acc() -> AvgGroupsAccumulator<Float64Type, impl Fn(f64, i64) -> Result<f64>> {
+ AvgGroupsAccumulator::<Float64Type, _>::new(&DataType::Float64, |sum, count| {
+ Ok(sum / count as f64)
+ })
+ }
+
+ #[test]
+ fn supports_convert_to_state() {
+ assert!(make_acc().supports_convert_to_state());
+ }
+
+ #[test]
+ fn convert_to_state_basic() {
+ let acc = make_acc();
+ let values: Vec<ArrayRef> =
+ vec![Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))];
+ let state = acc.convert_to_state(&values, None).unwrap();
+
+ assert_eq!(state.len(), 2);
+ let sums = state[0].as_primitive::<Float64Type>();
+ let counts = state[1].as_primitive::<Int64Type>();
+
+ assert_eq!(sums.values().as_ref(), &[1.0, 2.0, 3.0]);
+ assert_eq!(counts.values().as_ref(), &[1, 1, 1]);
+ assert_eq!(sums.null_count(), 0);
+ assert_eq!(counts.null_count(), 0);
+ }
+
+ #[test]
+ fn convert_to_state_with_nulls() {
+ let acc = make_acc();
+ let values: Vec<ArrayRef> = vec![Arc::new(Float64Array::from(vec![
+ Some(1.0),
+ None,
+ Some(3.0),
+ ]))];
+ let state = acc.convert_to_state(&values, None).unwrap();
+
+ let sums = state[0].as_primitive::<Float64Type>();
+ let counts = state[1].as_primitive::<Int64Type>();
+
+ assert!(!sums.is_null(0));
+ assert!(sums.is_null(1));
+ assert!(!sums.is_null(2));
+
+ assert_eq!(counts.value(0), 1);
+ assert!(counts.is_null(1));
+ assert_eq!(counts.value(2), 1);
+ }
+
+ #[test]
+ fn convert_to_state_with_filter() {
+ let acc = make_acc();
+ let values: Vec<ArrayRef> =
+ vec![Arc::new(Float64Array::from(vec![1.0, 2.0, 3.0]))];
+ let filter = BooleanArray::from(vec![true, false, true]);
+ let state = acc.convert_to_state(&values, Some(&filter)).unwrap();
+
+ let sums = state[0].as_primitive::<Float64Type>();
+ let counts = state[1].as_primitive::<Int64Type>();
+
+ assert!(!sums.is_null(0));
+ assert!(sums.is_null(1));
+ assert!(!sums.is_null(2));
+
+ assert_eq!(counts.value(0), 1);
+ assert!(counts.is_null(1));
+ assert_eq!(counts.value(2), 1);
+ }
+
+ #[test]
+ fn convert_to_state_roundtrips_through_merge() {
+ let mut acc = make_acc();
+ let input: Vec<ArrayRef> =
+ vec![Arc::new(Float64Array::from(vec![10.0, 20.0, 30.0]))];
+ let state = acc.convert_to_state(&input, None).unwrap();
+
+ // feed the converted state back through merge_batch
+ acc.merge_batch(
+ &state,
+ &[0, 0, 0],
+ None,
+ 1, // single group
+ )
+ .unwrap();
+
+ let result = acc.evaluate(EmitTo::All).unwrap();
+ let result = result.as_primitive::<Float64Type>();
+ assert_eq!(result.value(0), 20.0); // (10+20+30)/3
+ }
+
+ #[test]
+ fn convert_to_state_null_merge_matches_direct() {
+ // avg([1.0, NULL, 3.0]) must be 2.0 after a convert_to_state → merge_batch
+ // round-trip. Before the merge-path null fix this leaked the backing
+ // buffer value at the null slot and produced the wrong average.
+ let mut acc = make_acc();
+ let input: Vec<ArrayRef> = vec![Arc::new(Float64Array::from(vec![
+ Some(1.0),
+ None,
+ Some(3.0),
+ ]))];
+ let state = acc.convert_to_state(&input, None).unwrap();
+ acc.merge_batch(&state, &[0, 0, 0], None, 1).unwrap();
+
+ let result = acc.evaluate(EmitTo::All).unwrap();
+ let result = result.as_primitive::<Float64Type>();
+ assert_eq!(result.value(0), 2.0);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org