You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/01 14:01:36 UTC

(datafusion-comet) branch gh-readonly-queue/main/pr-6076-618b4cac6e874d9c394d4a6a8cbae014c64cffea created (now d2712e3b3e)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6076-618b4cac6e874d9c394d4a6a8cbae014c64cffea
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


      at d2712e3b3e fix: match Spark statistical aggregate updates and merges (#6076)

This branch includes the following new commits:

     new d2712e3b3e fix: match Spark statistical aggregate updates and merges (#6076)

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org


(datafusion-comet) 01/01: fix: match Spark statistical aggregate updates and merges (#6076)

Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6076-618b4cac6e874d9c394d4a6a8cbae014c64cffea
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git

commit d2712e3b3ee40663f195e8d9b732f63ff4d18f69
Author: KUAN-HAO HUANG <10...@users.noreply.github.com>
AuthorDate: Thu Oct 1 14:01:10 2026 +0000

    fix: match Spark statistical aggregate updates and merges (#6076)
    
    * fix: match Spark variance updates for large nearby values
    
    * fix: match Spark statistical aggregate merges
    
    * fix: preserve Spark semantics when merging empty partials
    
    * test: cover regression aggregates merging fractional constants
    
    * fix: align scalar correlation evaluation with Spark
    
    * fix: preserve correlation underflow and NaN semantics
---
 native/spark-expr/src/agg_funcs/correlation.rs     | 163 +++++++++++++++---
 native/spark-expr/src/agg_funcs/covariance.rs      |  92 +++++++++-
 native/spark-expr/src/agg_funcs/regr.rs            |  90 +++++++++-
 native/spark-expr/src/agg_funcs/stddev.rs          |  39 +++++
 native/spark-expr/src/agg_funcs/variance.rs        | 180 +++++++++++++++++++-
 native/spark-expr/src/agg_funcs/welford.rs         |  58 +++++--
 .../apache/comet/exec/CometAggregateSuite.scala    | 187 +++++++++++++++++++++
 7 files changed, 762 insertions(+), 47 deletions(-)

diff --git a/native/spark-expr/src/agg_funcs/correlation.rs b/native/spark-expr/src/agg_funcs/correlation.rs
index d69ac61def..c75d61ea1e 100644
--- a/native/spark-expr/src/agg_funcs/correlation.rs
+++ b/native/spark-expr/src/agg_funcs/correlation.rs
@@ -144,8 +144,10 @@ impl CorrelationAccumulator {
     pub fn try_new(null_on_divide_by_zero: bool) -> Result<Self> {
         Ok(Self {
             covar: CovarianceAccumulator::try_new(StatsType::Population, null_on_divide_by_zero)?,
-            stddev1: StddevAccumulator::try_new(StatsType::Population, null_on_divide_by_zero)?,
-            stddev2: StddevAccumulator::try_new(StatsType::Population, null_on_divide_by_zero)?,
+            stddev1: StddevAccumulator::try_new(StatsType::Population, null_on_divide_by_zero)?
+                .with_pearson_update(),
+            stddev2: StddevAccumulator::try_new(StatsType::Population, null_on_divide_by_zero)?
+                .with_pearson_update(),
             null_on_divide_by_zero,
         })
     }
@@ -227,10 +229,6 @@ impl Accumulator for CorrelationAccumulator {
     }
 
     fn evaluate(&mut self) -> Result<ScalarValue> {
-        let covar = self.covar.evaluate()?;
-        let stddev1 = self.stddev1.evaluate()?;
-        let stddev2 = self.stddev2.evaluate()?;
-
         if self.covar.get_count() == 0.0 {
             return Ok(ScalarValue::Float64(None));
         } else if self.covar.get_count() == 1.0 {
@@ -240,14 +238,17 @@ impl Accumulator for CorrelationAccumulator {
                 return Ok(ScalarValue::Float64(Some(f64::NAN)));
             }
         }
-        match (covar, stddev1, stddev2) {
-            (
-                ScalarValue::Float64(Some(c)),
-                ScalarValue::Float64(Some(s1)),
-                ScalarValue::Float64(Some(s2)),
-            ) if s1 != 0.0 && s2 != 0.0 => Ok(ScalarValue::Float64(Some(c / (s1 * s2)))),
-            _ => Ok(ScalarValue::Float64(None)),
+        let m2_product = self.stddev1.get_m2() * self.stddev2.get_m2();
+        // The product can underflow even when both moments are nonzero.
+        // A zero moment paired with infinity produces NaN, not a zero denominator.
+        if m2_product == 0.0 {
+            return Ok(ScalarValue::Float64(None));
         }
+        // Match Spark and the grouped path's raw-moment evaluation. Normalizing
+        // first changes rounding and can avoid overflow in m2_1 * m2_2.
+        Ok(ScalarValue::Float64(Some(
+            self.covar.get_algo_const() / m2_product.sqrt(),
+        )))
     }
 
     fn size(&self) -> usize {
@@ -280,8 +281,10 @@ impl CorrelationGroupsAccumulator {
         // that intent explicit.
         Self {
             covar: CovarianceGroupsAccumulator::new(StatsType::Population, false),
-            var1: VarianceGroupsAccumulator::new(StatsType::Population, false),
-            var2: VarianceGroupsAccumulator::new(StatsType::Population, false),
+            var1: VarianceGroupsAccumulator::new(StatsType::Population, false)
+                .with_pearson_update(),
+            var2: VarianceGroupsAccumulator::new(StatsType::Population, false)
+                .with_pearson_update(),
             null_on_divide_by_zero,
         }
     }
@@ -403,16 +406,15 @@ impl GroupsAccumulator for CorrelationGroupsAccumulator {
                 }
                 continue;
             }
-            // Population stats: divide m2 / count, c / count. The 1/count
-            // factors cancel in c / (s1 * s2), so we work with raw moments.
-            let s1_sq = m2_1s[i];
-            let s2_sq = m2_2s[i];
-            if s1_sq == 0.0 || s2_sq == 0.0 {
+            // Match Spark's raw-moment product, including overflow, underflow
+            // and NaN from a zero moment paired with infinity.
+            let m2_product = m2_1s[i] * m2_2s[i];
+            if m2_product == 0.0 {
                 values.push(0.0);
                 validity.push(false);
                 continue;
             }
-            values.push(algo_consts[i] / (s1_sq * s2_sq).sqrt());
+            values.push(algo_consts[i] / m2_product.sqrt());
             validity.push(true);
         }
 
@@ -473,6 +475,125 @@ mod groups_tests {
             .collect()
     }
 
+    #[test]
+    fn correlation_evaluates_raw_moments_exactly() {
+        // Spark divides ck by sqrt(m2_1 * m2_2). Normalizing the moments
+        // first changes rounding, and avoids overflow that Spark preserves.
+        for (values, expected) in [
+            ([1e16, 1e16 + 2.0], Some(1.0)),
+            ([1e16 + 2.0, 1e16], None),
+            ([-1e16, -1e16 - 2.0], Some(1.0)),
+            ([1e100, 2e100], Some(0.0)),
+            ([1e-100, 2e-100], None),
+        ] {
+            for sign in [-1.0, 1.0] {
+                for null_on_divide_by_zero in [false, true] {
+                    let input: Vec<ArrayRef> = vec![
+                        Arc::new(Float64Array::from(vec![
+                            Some(values[0]),
+                            None,
+                            Some(values[1]),
+                        ])),
+                        Arc::new(Float64Array::from(vec![
+                            Some(sign * values[0]),
+                            Some(0.0),
+                            Some(sign * values[1]),
+                        ])),
+                    ];
+                    let mut scalar =
+                        CorrelationAccumulator::try_new(null_on_divide_by_zero).unwrap();
+                    let mut grouped = CorrelationGroupsAccumulator::new(null_on_divide_by_zero);
+                    scalar.update_batch(&input).unwrap();
+                    grouped.update_batch(&input, &[0, 0, 0], None, 1).unwrap();
+                    let state = scalar
+                        .state()
+                        .unwrap()
+                        .iter()
+                        .map(|s| s.to_array_of_size(1).unwrap())
+                        .collect::<Vec<_>>();
+                    let mut merged_scalar =
+                        CorrelationAccumulator::try_new(null_on_divide_by_zero).unwrap();
+                    let mut merged_grouped =
+                        CorrelationGroupsAccumulator::new(null_on_divide_by_zero);
+                    merged_scalar.merge_batch(&state).unwrap();
+                    merged_grouped.merge_batch(&state, &[0], 1).unwrap();
+                    for result in [
+                        scalar.evaluate().unwrap(),
+                        merged_scalar.evaluate().unwrap(),
+                    ] {
+                        assert_eq!(result, ScalarValue::Float64(expected.map(|v| sign * v)));
+                    }
+                    for result in [evaluate(&mut grouped)[0], evaluate(&mut merged_grouped)[0]] {
+                        assert_eq!(result, expected.map(|v| sign * v));
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn correlation_zero_and_infinite_moments_yield_nan() {
+        let input: Vec<ArrayRef> = vec![
+            Arc::new(Float64Array::from(vec![1e200, -1e200])),
+            Arc::new(Float64Array::from(vec![0.1, 0.1])),
+        ];
+        let mut scalar = CorrelationAccumulator::try_new(true).unwrap();
+        let mut grouped = CorrelationGroupsAccumulator::new(true);
+        scalar.update_batch(&input).unwrap();
+        grouped.update_batch(&input, &[0, 0], None, 1).unwrap();
+        let state = scalar
+            .state()
+            .unwrap()
+            .iter()
+            .map(|s| s.to_array_of_size(1).unwrap())
+            .collect::<Vec<_>>();
+        let mut merged_scalar = CorrelationAccumulator::try_new(true).unwrap();
+        let mut merged_grouped = CorrelationGroupsAccumulator::new(true);
+        merged_scalar.merge_batch(&state).unwrap();
+        merged_grouped.merge_batch(&state, &[0], 1).unwrap();
+        for result in [
+            scalar.evaluate().unwrap(),
+            merged_scalar.evaluate().unwrap(),
+        ] {
+            assert!(matches!(result, ScalarValue::Float64(Some(v)) if v.is_nan()));
+        }
+        for result in [evaluate(&mut grouped)[0], evaluate(&mut merged_grouped)[0]] {
+            assert!(result.unwrap().is_nan());
+        }
+    }
+
+    #[test]
+    fn correlation_merge_empty_partials() {
+        for empty_first in [false, true] {
+            let mut scalar = CorrelationAccumulator::try_new(true).unwrap();
+            let mut grouped = CorrelationGroupsAccumulator::new(true);
+            let counts = if empty_first {
+                [0.0, 100.0]
+            } else {
+                [100.0, 0.0]
+            };
+            for count in counts {
+                let mean = if count == 0.0 { 0.0 } else { 1e155 };
+                let state: Vec<ArrayRef> = [count, mean, mean, 0.0, 0.0, 0.0]
+                    .into_iter()
+                    .map(|v| Arc::new(Float64Array::from(vec![v])) as ArrayRef)
+                    .collect();
+                scalar.merge_batch(&state).unwrap();
+                grouped.merge_batch(&state, &[0], 1).unwrap();
+            }
+            let ScalarValue::Float64(scalar) = scalar.evaluate().unwrap() else {
+                panic!("expected a double correlation");
+            };
+            for result in [scalar, evaluate(&mut grouped)[0]] {
+                if empty_first {
+                    assert_eq!(result, None);
+                } else {
+                    assert!(result.unwrap().is_nan());
+                }
+            }
+        }
+    }
+
     #[test]
     fn perfectly_correlated_single_group() {
         let mut a = acc(true);
diff --git a/native/spark-expr/src/agg_funcs/covariance.rs b/native/spark-expr/src/agg_funcs/covariance.rs
index 548c118720..28e09574a8 100644
--- a/native/spark-expr/src/agg_funcs/covariance.rs
+++ b/native/spark-expr/src/agg_funcs/covariance.rs
@@ -283,9 +283,7 @@ impl Accumulator for CovarianceAccumulator {
 
         for i in 0..counts.len() {
             let c = counts.value(i);
-            if c == 0.0 {
-                continue;
-            }
+            // Even empty partials affect Spark's NaN propagation during merge.
             let (new_count, new_mean1, new_mean2, new_c) = super::welford::covariance_merge(
                 self.count,
                 self.mean1,
@@ -428,9 +426,7 @@ impl GroupsAccumulator for CovarianceGroupsAccumulator {
 
         for (i, &group_index) in group_indices.iter().enumerate() {
             let partial_count = counts.value(i);
-            if partial_count == 0.0 {
-                continue;
-            }
+            // Even empty partials affect Spark's NaN propagation during merge.
             let (new_count, new_m1, new_m2, new_c) = super::welford::covariance_merge(
                 self.counts[group_index],
                 self.mean1s[group_index],
@@ -513,6 +509,90 @@ mod groups_tests {
         assert!((evaluate(&mut acc)[0].unwrap() - 4.0).abs() < 1e-12);
     }
 
+    #[test]
+    fn large_offset_covariance_merge() {
+        let mut scalar = CovarianceAccumulator::try_new(StatsType::Population, true).unwrap();
+        let mut grouped = pop();
+        for (value, count) in [(0.0, 0), (1e17 - 32.0, 3), (1e17 - 16.0, 2), (0.0, 0)] {
+            let mut xs = vec![Some(value); count];
+            xs.push(None);
+            let ys = xs.iter().map(|v| v.map(|v| -v)).collect::<Vec<_>>();
+            let values: Vec<ArrayRef> = vec![
+                Arc::new(Float64Array::from(xs)),
+                Arc::new(Float64Array::from(ys)),
+            ];
+            let mut partial = pop();
+            partial
+                .update_batch(&values, &vec![0; count + 1], None, 2)
+                .unwrap();
+            let state = partial.state(EmitTo::All).unwrap();
+            scalar.merge_batch(&state).unwrap();
+            grouped.merge_batch(&state, &[0, 1], 2).unwrap();
+        }
+        let expected = -61.44000000000001;
+        assert_eq!(
+            scalar.evaluate().unwrap(),
+            ScalarValue::Float64(Some(expected))
+        );
+        assert_eq!(evaluate(&mut grouped), vec![Some(expected), None]);
+    }
+
+    #[test]
+    fn covariance_merge_empty_partials() {
+        for value in [1e154, -1e154, 1e155, -1e155] {
+            for empty_first in [false, true] {
+                for stats in [StatsType::Population, StatsType::Sample] {
+                    let mut scalar = CovarianceAccumulator::try_new(stats, true).unwrap();
+                    let mut grouped = CovarianceGroupsAccumulator::new(stats, true);
+                    let counts = if empty_first { [0, 100] } else { [100, 0] };
+                    for count in counts {
+                        let xs: ArrayRef = Arc::new(Float64Array::from(
+                            (0..101)
+                                .map(|i| (i < count).then_some(value))
+                                .collect::<Vec<_>>(),
+                        ));
+                        let ys: ArrayRef = Arc::new(Float64Array::from(vec![Some(-value); 101]));
+                        // A null in just one input still produces a zero-count partial.
+                        let values = [xs, ys];
+                        let mut partial = CovarianceAccumulator::try_new(stats, true).unwrap();
+                        partial.update_batch(&values).unwrap();
+                        let state = partial
+                            .state()
+                            .unwrap()
+                            .iter()
+                            .map(|v| v.to_array_of_size(1).unwrap())
+                            .collect::<Vec<_>>();
+                        scalar.merge_batch(&state).unwrap();
+
+                        let mut partial = CovarianceGroupsAccumulator::new(stats, true);
+                        let mut group_indices = vec![0; 101];
+                        // The second group has no valid input pairs in either partial.
+                        group_indices[100] = 1;
+                        partial
+                            .update_batch(&values, &group_indices, None, 2)
+                            .unwrap();
+                        grouped
+                            .merge_batch(&partial.state(EmitTo::All).unwrap(), &[0, 1], 2)
+                            .unwrap();
+                    }
+                    let expected_nan = value.abs() == 1e155 && !empty_first;
+                    let ScalarValue::Float64(Some(actual)) = scalar.evaluate().unwrap() else {
+                        panic!("expected a non-null covariance");
+                    };
+                    let grouped = evaluate(&mut grouped);
+                    for result in [actual, grouped[0].unwrap()] {
+                        if expected_nan {
+                            assert!(result.is_nan(), "value={value}, empty_first={empty_first}");
+                        } else {
+                            assert_eq!(result, 0.0);
+                        }
+                    }
+                    assert_eq!(grouped[1], None);
+                }
+            }
+        }
+    }
+
     #[test]
     fn null_in_either_column_skipped() {
         let mut acc = pop();
diff --git a/native/spark-expr/src/agg_funcs/regr.rs b/native/spark-expr/src/agg_funcs/regr.rs
index 72511fc987..0b8838c985 100644
--- a/native/spark-expr/src/agg_funcs/regr.rs
+++ b/native/spark-expr/src/agg_funcs/regr.rs
@@ -306,8 +306,10 @@ impl RegrR2Accumulator {
     fn try_new(constant_dependent_is_perfect_fit: bool) -> Result<Self> {
         Ok(Self {
             covar: CovarianceAccumulator::try_new(StatsType::Population, false)?,
-            var_y: VarianceAccumulator::try_new(StatsType::Population, false)?,
-            var_x: VarianceAccumulator::try_new(StatsType::Population, false)?,
+            var_y: VarianceAccumulator::try_new(StatsType::Population, false)?
+                .with_pearson_update(),
+            var_x: VarianceAccumulator::try_new(StatsType::Population, false)?
+                .with_pearson_update(),
             constant_dependent_is_perfect_fit,
         })
     }
@@ -533,6 +535,45 @@ mod tests {
         );
     }
 
+    #[test]
+    fn regr_merge_empty_partials() {
+        for kind in [
+            RegrType::SXX,
+            RegrType::SYY,
+            RegrType::SXY,
+            RegrType::R2,
+            RegrType::Slope,
+            RegrType::Intercept,
+        ] {
+            for empty_first in [false, true] {
+                let mut merged = acc(kind);
+                let counts = if empty_first { [0, 100] } else { [100, 0] };
+                for count in counts {
+                    let mut partial = acc(kind);
+                    let values = vec![Some(1e155); count];
+                    partial.update_batch(&cols(values.clone(), values)).unwrap();
+                    let state = partial
+                        .state()
+                        .unwrap()
+                        .iter()
+                        .map(|v| v.to_array_of_size(1).unwrap())
+                        .collect::<Vec<_>>();
+                    merged.merge_batch(&state).unwrap();
+                }
+                let ScalarValue::Float64(result) = merged.evaluate().unwrap() else {
+                    panic!("expected a double regression result");
+                };
+                if !empty_first {
+                    assert!(result.unwrap().is_nan(), "{kind:?}");
+                } else if matches!(kind, RegrType::SXX | RegrType::SYY | RegrType::SXY) {
+                    assert_eq!(result, Some(0.0));
+                } else {
+                    assert_eq!(result, None);
+                }
+            }
+        }
+    }
+
     fn perfect_line() -> (Vec<Option<f64>>, Vec<Option<f64>>) {
         // y = 2x + 1
         let x = vec![1.0, 2.0, 3.0, 4.0, 5.0];
@@ -680,6 +721,51 @@ mod tests {
         }
     }
 
+    #[test]
+    fn merge_fractional_constant_partials() {
+        // #6423: the first merge into an empty final buffer used to round the
+        // constant mean away from 0.1. Later merges then produced nonzero M2.
+        for (kind, constant_dependent, expected) in [
+            (RegrType::Slope, false, None),
+            (RegrType::Intercept, false, None),
+            (RegrType::R2, false, None),
+            (RegrType::R2, true, Some(1.0)),
+            (RegrType::SXX, false, Some(0.0)),
+            (RegrType::SYY, true, Some(0.0)),
+            (RegrType::SXY, false, Some(0.0)),
+        ] {
+            for reverse in [false, true] {
+                let mut merged = acc(kind);
+                let starts = if reverse { [3, 0] } else { [0, 3] };
+                for start in starts {
+                    let varying = (start..start + 3).map(|v| Some(v as f64)).collect();
+                    let constant = vec![Some(0.1); 3];
+                    let values = match kind {
+                        // Spark's RegrReplacement duplicates the selected column.
+                        RegrType::SXX | RegrType::SYY => cols(constant.clone(), constant),
+                        _ if constant_dependent => cols(constant, varying),
+                        _ => cols(varying, constant),
+                    };
+                    let mut partial = acc(kind);
+                    partial.update_batch(&values).unwrap();
+                    let state = partial
+                        .state()
+                        .unwrap()
+                        .iter()
+                        .map(|v| v.to_array_of_size(1).unwrap())
+                        .collect::<Vec<_>>();
+                    merged.merge_batch(&state).unwrap();
+                }
+                // A tolerance would hide the tiny nonzero moments behind the bug.
+                assert_eq!(
+                    merged.evaluate().unwrap(),
+                    ScalarValue::Float64(expected),
+                    "{kind:?}, constant_dependent={constant_dependent}, reverse={reverse}"
+                );
+            }
+        }
+    }
+
     #[test]
     fn merge_matches_single_batch() {
         let (y, x) = perfect_line();
diff --git a/native/spark-expr/src/agg_funcs/stddev.rs b/native/spark-expr/src/agg_funcs/stddev.rs
index bbceaa72dc..8ef9411119 100644
--- a/native/spark-expr/src/agg_funcs/stddev.rs
+++ b/native/spark-expr/src/agg_funcs/stddev.rs
@@ -163,6 +163,11 @@ impl StddevAccumulator {
     pub fn get_m2(&self) -> f64 {
         self.variance.get_m2()
     }
+
+    pub(super) fn with_pearson_update(mut self) -> Self {
+        self.variance = self.variance.with_pearson_update();
+        self
+    }
 }
 
 impl Accumulator for StddevAccumulator {
@@ -268,6 +273,40 @@ mod groups_tests {
     use arrow::array::AsArray;
     use arrow::datatypes::Float64Type;
 
+    #[test]
+    fn stddev_merge_empty_partials() {
+        for empty_first in [false, true] {
+            for stats in [StatsType::Population, StatsType::Sample] {
+                let mut scalar = StddevAccumulator::try_new(stats, true).unwrap();
+                let mut grouped = StddevGroupsAccumulator::new(stats, true);
+                let counts = if empty_first {
+                    [0.0, 100.0]
+                } else {
+                    [100.0, 0.0]
+                };
+                for count in counts {
+                    let state: Vec<ArrayRef> = [count, if count == 0.0 { 0.0 } else { 1e155 }, 0.0]
+                        .into_iter()
+                        .map(|v| Arc::new(Float64Array::from(vec![v])) as ArrayRef)
+                        .collect();
+                    scalar.merge_batch(&state).unwrap();
+                    grouped.merge_batch(&state, &[0], 1).unwrap();
+                }
+                let ScalarValue::Float64(Some(scalar)) = scalar.evaluate().unwrap() else {
+                    panic!("expected a non-null standard deviation");
+                };
+                let grouped = grouped.evaluate(EmitTo::All).unwrap();
+                for result in [scalar, grouped.as_primitive::<Float64Type>().value(0)] {
+                    if empty_first {
+                        assert_eq!(result, 0.0);
+                    } else {
+                        assert!(result.is_nan());
+                    }
+                }
+            }
+        }
+    }
+
     #[test]
     fn pop_stddev_single_group() {
         let mut acc = StddevGroupsAccumulator::new(StatsType::Population, false);
diff --git a/native/spark-expr/src/agg_funcs/variance.rs b/native/spark-expr/src/agg_funcs/variance.rs
index 57a8f6da50..4d89bff569 100644
--- a/native/spark-expr/src/agg_funcs/variance.rs
+++ b/native/spark-expr/src/agg_funcs/variance.rs
@@ -29,6 +29,8 @@ use datafusion::physical_expr::expressions::StatsType;
 use std::mem::size_of;
 use std::sync::Arc;
 
+use super::welford::VarianceUpdate;
+
 /// VAR_SAMP and VAR_POP aggregate expression
 /// The implementation mostly is the same as the DataFusion's implementation. The reason
 /// we have our own implementation is that DataFusion has UInt64 for state_field `count`,
@@ -144,6 +146,7 @@ pub struct VarianceAccumulator {
     count: f64,
     stats_type: StatsType,
     null_on_divide_by_zero: bool,
+    update: VarianceUpdate,
 }
 
 impl VarianceAccumulator {
@@ -155,9 +158,15 @@ impl VarianceAccumulator {
             count: 0_f64,
             stats_type: s_type,
             null_on_divide_by_zero,
+            update: VarianceUpdate::CentralMoment,
         })
     }
 
+    pub(super) fn with_pearson_update(mut self) -> Self {
+        self.update = VarianceUpdate::Pearson;
+        self
+    }
+
     pub fn get_count(&self) -> f64 {
         self.count
     }
@@ -184,7 +193,8 @@ impl Accumulator for VarianceAccumulator {
         let arr = downcast_value!(&values[0], Float64Array).iter().flatten();
 
         for value in arr {
-            let (c, m, m2) = super::welford::variance_update(self.count, self.mean, self.m2, value);
+            let (c, m, m2) =
+                super::welford::variance_update(self.count, self.mean, self.m2, value, self.update);
             self.count = c;
             self.mean = m;
             self.m2 = m2;
@@ -214,9 +224,7 @@ impl Accumulator for VarianceAccumulator {
 
         for i in 0..counts.len() {
             let c = counts.value(i);
-            if c == 0_f64 {
-                continue;
-            }
+            // Even empty partials affect Spark's NaN propagation during merge.
             let (new_count, new_mean, new_m2) = super::welford::variance_merge(
                 self.count,
                 self.mean,
@@ -271,6 +279,7 @@ pub(crate) struct VarianceGroupsAccumulator {
     pub(super) m2s: Vec<f64>,
     stats_type: StatsType,
     null_on_divide_by_zero: bool,
+    update: VarianceUpdate,
 }
 
 impl VarianceGroupsAccumulator {
@@ -281,9 +290,15 @@ impl VarianceGroupsAccumulator {
             m2s: Vec::new(),
             stats_type,
             null_on_divide_by_zero,
+            update: VarianceUpdate::CentralMoment,
         }
     }
 
+    pub(super) fn with_pearson_update(mut self) -> Self {
+        self.update = VarianceUpdate::Pearson;
+        self
+    }
+
     fn resize(&mut self, total_num_groups: usize) {
         self.counts.resize(total_num_groups, 0.0);
         self.means.resize(total_num_groups, 0.0);
@@ -327,6 +342,7 @@ impl GroupsAccumulator for VarianceGroupsAccumulator {
                 self.means[group_index],
                 self.m2s[group_index],
                 value,
+                self.update,
             );
             self.counts[group_index] = c;
             self.means[group_index] = m;
@@ -351,9 +367,7 @@ impl GroupsAccumulator for VarianceGroupsAccumulator {
 
         for (i, &group_index) in group_indices.iter().enumerate() {
             let partial_count = partial_counts.value(i);
-            if partial_count == 0.0 {
-                continue;
-            }
+            // Even empty partials affect Spark's NaN propagation during merge.
             let (new_count, new_mean, new_m2) = super::welford::variance_merge(
                 self.counts[group_index],
                 self.means[group_index],
@@ -421,6 +435,158 @@ mod groups_tests {
             .collect()
     }
 
+    #[test]
+    fn large_offset_variance() {
+        for pair in [[1e16, 1e16 + 2.0], [1e16 + 2.0, 1e16], [-1e16, -1e16 - 2.0]] {
+            for (stats, expected) in [(StatsType::Population, 1.0), (StatsType::Sample, 2.0)] {
+                let values: ArrayRef =
+                    Arc::new(Float64Array::from(vec![Some(pair[0]), None, Some(pair[1])]));
+                for batch_size in [1, 3] {
+                    let mut scalar = VarianceAccumulator::try_new(stats, true).unwrap();
+                    let mut grouped = VarianceGroupsAccumulator::new(stats, true);
+                    for offset in (0..3).step_by(batch_size) {
+                        let batch = [values.slice(offset, batch_size)];
+                        scalar.update_batch(&batch).unwrap();
+                        grouped
+                            .update_batch(&batch, &vec![0; batch_size], None, 2)
+                            .unwrap();
+                    }
+                    assert_eq!(
+                        scalar.evaluate().unwrap(),
+                        ScalarValue::Float64(Some(expected))
+                    );
+                    let states = grouped.state(EmitTo::All).unwrap();
+                    let mut merged = VarianceGroupsAccumulator::new(stats, true);
+                    merged.merge_batch(&states, &[0, 1], 2).unwrap();
+                    assert_eq!(evaluate(&mut merged), vec![Some(expected), None]);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn large_offset_variance_merge() {
+        for (partitions, population, sample) in [
+            (
+                [(1e17 - 32.0, 3), (1e17 - 16.0, 2)],
+                61.44000000000001,
+                76.80000000000001,
+            ),
+            ([(1e17 - 96.0, 3), (1e17 - 32.0, 3)], 1024.0, 1228.8),
+        ] {
+            for sign in [1.0, -1.0] {
+                for reverse in [false, true] {
+                    let mut partitions = partitions;
+                    if reverse {
+                        partitions.reverse();
+                    }
+                    for (stats, expected) in [
+                        (StatsType::Population, population),
+                        (StatsType::Sample, sample),
+                    ] {
+                        let mut scalar = VarianceAccumulator::try_new(stats, true).unwrap();
+                        let mut grouped = VarianceGroupsAccumulator::new(stats, true);
+                        // Include empty partials before and after the nonempty states.
+                        for (value, count) in
+                            [(0.0, 0)].into_iter().chain(partitions).chain([(0.0, 0)])
+                        {
+                            let mut values = vec![Some(sign * value); count];
+                            values.push(None);
+                            let values: ArrayRef = Arc::new(Float64Array::from(values));
+                            let mut partial = VarianceAccumulator::try_new(stats, true).unwrap();
+                            partial.update_batch(&[Arc::clone(&values)]).unwrap();
+                            let state = partial
+                                .state()
+                                .unwrap()
+                                .iter()
+                                .map(|v| v.to_array_of_size(1).unwrap())
+                                .collect::<Vec<_>>();
+                            scalar.merge_batch(&state).unwrap();
+
+                            let mut partial = VarianceGroupsAccumulator::new(stats, true);
+                            partial
+                                .update_batch(&[values], &vec![0; count + 1], None, 2)
+                                .unwrap();
+                            grouped
+                                .merge_batch(&partial.state(EmitTo::All).unwrap(), &[0, 1], 2)
+                                .unwrap();
+                        }
+                        assert_eq!(
+                            scalar.evaluate().unwrap(),
+                            ScalarValue::Float64(Some(expected))
+                        );
+                        assert_eq!(evaluate(&mut grouped), vec![Some(expected), None]);
+                    }
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn variance_merge_empty_partials() {
+        for value in [1e154, -1e154, 1e155, -1e155] {
+            for empty_first in [false, true] {
+                for stats in [StatsType::Population, StatsType::Sample] {
+                    for update in [VarianceUpdate::CentralMoment, VarianceUpdate::Pearson] {
+                        let mut scalar = VarianceAccumulator::try_new(stats, true).unwrap();
+                        let mut grouped = VarianceGroupsAccumulator::new(stats, true);
+                        scalar.update = update;
+                        grouped.update = update;
+                        let counts = if empty_first { [0, 100] } else { [100, 0] };
+                        for count in counts {
+                            let values: ArrayRef = Arc::new(Float64Array::from(
+                                (0..101)
+                                    .map(|i| (i < count).then_some(value))
+                                    .collect::<Vec<_>>(),
+                            ));
+                            let mut partial = VarianceAccumulator::try_new(stats, true).unwrap();
+                            partial.update = update;
+                            partial.update_batch(&[Arc::clone(&values)]).unwrap();
+                            let state = partial
+                                .state()
+                                .unwrap()
+                                .iter()
+                                .map(|v| v.to_array_of_size(1).unwrap())
+                                .collect::<Vec<_>>();
+                            scalar.merge_batch(&state).unwrap();
+
+                            let mut partial = VarianceGroupsAccumulator::new(stats, true);
+                            partial.update = update;
+                            let mut group_indices = vec![0; 101];
+                            // Keep a separate all-null group alongside the non-empty group.
+                            group_indices[100] = 1;
+                            partial
+                                .update_batch(&[values], &group_indices, None, 2)
+                                .unwrap();
+                            grouped
+                                .merge_batch(&partial.state(EmitTo::All).unwrap(), &[0, 1], 2)
+                                .unwrap();
+                        }
+                        // Spark evaluates delta * deltaN * n1 * n2 even when n2 is zero.
+                        // For 1e155, the third multiplication overflows before multiplying by
+                        // zero. Empty-first and the smaller-magnitude control remain zero.
+                        let expected_nan = value.abs() == 1e155 && !empty_first;
+                        let ScalarValue::Float64(Some(actual)) = scalar.evaluate().unwrap() else {
+                            panic!("expected a non-null variance");
+                        };
+                        let grouped = evaluate(&mut grouped);
+                        for result in [actual, grouped[0].unwrap()] {
+                            if expected_nan {
+                                assert!(
+                                    result.is_nan(),
+                                    "value={value}, empty_first={empty_first}"
+                                );
+                            } else {
+                                assert_eq!(result, 0.0);
+                            }
+                        }
+                        assert_eq!(grouped[1], None);
+                    }
+                }
+            }
+        }
+    }
+
     #[test]
     fn pop_variance_single_group() {
         let mut acc = pop_acc();
diff --git a/native/spark-expr/src/agg_funcs/welford.rs b/native/spark-expr/src/agg_funcs/welford.rs
index bcc44b2988..dc3f8551e3 100644
--- a/native/spark-expr/src/agg_funcs/welford.rs
+++ b/native/spark-expr/src/agg_funcs/welford.rs
@@ -23,12 +23,30 @@
 use arrow::buffer::NullBuffer;
 use datafusion::physical_expr::expressions::StatsType;
 
+#[derive(Debug, Clone, Copy)]
+pub(super) enum VarianceUpdate {
+    CentralMoment,
+    Pearson,
+}
+
 #[inline]
-pub(crate) fn variance_update(count: f64, mean: f64, m2: f64, value: f64) -> (f64, f64, f64) {
+pub(super) fn variance_update(
+    count: f64,
+    mean: f64,
+    m2: f64,
+    value: f64,
+    update: VarianceUpdate,
+) -> (f64, f64, f64) {
     let new_count = count + 1.0;
     let delta1 = value - mean;
-    let new_mean = delta1 / new_count + mean;
-    let delta2 = value - new_mean;
+    let delta_n = delta1 / new_count;
+    let new_mean = mean + delta_n;
+    // Match Spark's CentralMomentAgg without subtracting the rounded new mean.
+    // PearsonCorrelation (also used by regr_r2) deliberately uses that subtraction.
+    let delta2 = match update {
+        VarianceUpdate::CentralMoment => delta1 - delta_n,
+        VarianceUpdate::Pearson => value - new_mean,
+    };
     let new_m2 = m2 + delta1 * delta2;
     (new_count, new_mean, new_m2)
 }
@@ -53,9 +71,16 @@ pub(crate) fn variance_merge(
     m2_b: f64,
 ) -> (f64, f64, f64) {
     let new_count = count_a + count_b;
-    let new_mean = mean_a * count_a / new_count + mean_b * count_b / new_count;
-    let delta = mean_a - mean_b;
-    let new_m2 = m2_a + m2_b + delta * delta * count_a * count_b / new_count;
+    // CentralMomentAgg and PearsonCorrelation use the same merge expressions.
+    // Preserve Spark's operation order to avoid rounding large means differently.
+    let delta = mean_b - mean_a;
+    let delta_n = if new_count == 0.0 {
+        0.0
+    } else {
+        delta / new_count
+    };
+    let new_mean = mean_a + delta_n * count_b;
+    let new_m2 = m2_a + m2_b + delta * delta_n * count_a * count_b;
     (new_count, new_mean, new_m2)
 }
 
@@ -149,10 +174,21 @@ pub(crate) fn covariance_merge(
     c_b: f64,
 ) -> (f64, f64, f64, f64) {
     let new_count = count_a + count_b;
-    let new_mean1 = mean1_a * count_a / new_count + mean1_b * count_b / new_count;
-    let new_mean2 = mean2_a * count_a / new_count + mean2_b * count_b / new_count;
-    let delta1 = mean1_a - mean1_b;
-    let delta2 = mean2_a - mean2_b;
-    let new_c = c_a + c_b + delta1 * delta2 * count_a * count_b / new_count;
+    // Keep covariance aligned with variance and Spark's Covariance/PearsonCorrelation.
+    let delta1 = mean1_b - mean1_a;
+    let delta2 = mean2_b - mean2_a;
+    let delta1_n = if new_count == 0.0 {
+        0.0
+    } else {
+        delta1 / new_count
+    };
+    let delta2_n = if new_count == 0.0 {
+        0.0
+    } else {
+        delta2 / new_count
+    };
+    let new_mean1 = mean1_a + delta1_n * count_b;
+    let new_mean2 = mean2_a + delta2_n * count_b;
+    let new_c = c_a + c_b + delta1 * delta2_n * count_a * count_b;
     (new_count, new_mean1, new_mean2, new_c)
 }
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala
index ed16cbb1d4..62417c314d 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala
@@ -2759,6 +2759,193 @@ class CometAggregateSuite extends CometTestBase with AdaptiveSparkPlanHelper {
     }
   }
 
+  test("statistical aggregates with large nearby values") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+      "spark.sql.files.minPartitionNum" -> "1",
+      CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
+      CometConf.COMET_SHUFFLE_MODE.key -> "native") {
+      val cases = Seq(
+        (Seq(1e16, 1e16 + 2), Some(1.0)),
+        (Seq(1e16 + 2, 1e16), None),
+        (Seq(-1e16, -1e16 - 2), Some(1.0)))
+      for ((values, expectedCorr) <- cases) {
+        // One ordered file keeps both values in the same partial accumulator. Splitting
+        // them across files would only exercise merging two single-row states.
+        withTempPath { path =>
+          (Seq(Some(values.head), None, Some(values.last)))
+            .map(v => (0, v))
+            .toDF("g", "v")
+            .coalesce(1)
+            .write
+            .parquet(path.getCanonicalPath)
+          withParquetTable(path.getCanonicalPath, "large_moments") {
+            for (groupBy <- Seq("", " GROUP BY g")) {
+              val query = "SELECT var_pop(v), var_samp(v), stddev_pop(v), stddev_samp(v) " +
+                "FROM large_moments" + groupBy
+              val (_, cometPlan) = checkSparkAnswerAndOperator(query)
+              val aggregates = cometPlan.collect { case a: CometHashAggregateExec => a }
+              assert(aggregates.exists(_.modes.contains(Partial)))
+              assert(aggregates.exists(_.modes.contains(Final)))
+              checkAnswer(sql(query), Seq(Row(1.0, 2.0, 1.0, math.sqrt(2.0))))
+
+              // CORR and REGR_R2 use PearsonCorrelation's update, while REGR_SXX/SYY
+              // and the variance used by slope/intercept follow CentralMomentAgg.
+              val corrQuery = "SELECT corr(v, v) FROM large_moments" + groupBy
+              checkSparkAnswerAndNumOfAggregates(corrQuery, 2)
+              // Reversing the positive pair rounds the Pearson mean to the second
+              // value, leaving zero M2 and a NULL correlation.
+              checkAnswer(sql(corrQuery), Seq(Row(expectedCorr.map(Double.box).orNull)))
+              checkSparkAnswerWithTolAndNumOfAggregates(
+                "SELECT regr_r2(v, v), regr_sxx(v, v), regr_syy(v, v), " +
+                  "regr_slope(v, v), regr_intercept(v, v) FROM large_moments" + groupBy,
+                2)
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("statistical aggregates correlation uses raw moments at extreme magnitudes") {
+    withSQLConf(
+      SQLConf.ANSI_ENABLED.key -> "false",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+      "spark.sql.files.minPartitionNum" -> "1",
+      CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
+      CometConf.COMET_SHUFFLE_MODE.key -> "native") {
+      val cases = Seq(
+        (Seq(1e100, 2e100), Some(0.0), None),
+        (Seq(1e-100, 2e-100), None, None),
+        (Seq(1e200, -1e200), Some(Double.NaN), Some(Double.NaN)))
+      for ((values, expectedCorr, expectedConstantCorr) <- cases) {
+        withTempPath { path =>
+          Seq(Some(values.head), None, Some(values.last))
+            .map(v => (0, v, 0.1))
+            .toDF("g", "v", "x")
+            .coalesce(1)
+            .write
+            .parquet(path.getCanonicalPath)
+          withParquetTable(path.getCanonicalPath, "correlation_extremes") {
+            assert(spark.table("correlation_extremes").rdd.getNumPartitions == 1)
+            for (groupBy <- Seq("", " GROUP BY g")) {
+              val query = "SELECT corr(v, v), corr(v, -v), corr(v, x) " +
+                "FROM correlation_extremes" + groupBy
+              checkSparkAnswerAndNumOfAggregates(query, 2)
+              // The raw-moment product can overflow, underflow, or become NaN (0 * Inf).
+              // ANSI-off division by a zero denominator returns NULL.
+              checkAnswer(
+                sql(query),
+                Seq(
+                  Row(
+                    expectedCorr.map(Double.box).orNull,
+                    expectedCorr.map(v => Double.box(-v)).orNull,
+                    expectedConstantCorr.map(Double.box).orNull)))
+            }
+          }
+        }
+      }
+    }
+  }
+
+  test("statistical aggregates merge large nearby values across partitions") {
+    withSQLConf(
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+      SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1048576",
+      SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "1048576",
+      CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
+      CometConf.COMET_SHUFFLE_MODE.key -> "native") {
+      withTempPath { path =>
+        // Two constant-valued files produce separate partials with zero M2. The
+        // old merge returns 576 instead of 1024, regardless of which partial arrives first.
+        for (value <- Seq(1e17 - 96, 1e17 - 32)) {
+          (Seq.fill(3)((0, Option(value))) ++ Seq((0, None), (1, None)))
+            .toDF("g", "v")
+            .coalesce(1)
+            .write
+            .mode("append")
+            .parquet(path.getCanonicalPath)
+        }
+        withParquetTable(path.getCanonicalPath, "merged_moments") {
+          assert(spark.table("merged_moments").rdd.getNumPartitions == 2)
+          for (groupBy <- Seq("", " GROUP BY g")) {
+            val query = "SELECT var_pop(v), var_samp(v), stddev_pop(v), stddev_samp(v) " +
+              "FROM merged_moments" + groupBy
+            val (_, cometPlan) = checkSparkAnswerAndOperator(query)
+            val aggregates = cometPlan.collect { case a: CometHashAggregateExec => a }
+            assert(aggregates.exists(_.modes.contains(Partial)))
+            assert(aggregates.exists(_.modes.contains(Final)))
+            val expected = Seq(Row(1024.0, 1228.8, 32.0, math.sqrt(1228.8))) ++
+              (if (groupBy.isEmpty) Seq.empty else Seq(Row(null, null, null, null)))
+            checkAnswer(sql(query), expected)
+            checkSparkAnswerWithTolAndNumOfAggregates(
+              "SELECT covar_pop(v, -v), covar_samp(v, -v), corr(v, -v), regr_r2(v, -v), " +
+                "regr_sxx(v, -v), regr_syy(v, -v), regr_sxy(v, -v), " +
+                "regr_slope(v, -v), regr_intercept(v, -v) FROM merged_moments" + groupBy,
+              2)
+          }
+        }
+      }
+    }
+  }
+
+  test("statistical aggregates merge fractional constants across partitions") {
+    // https://github.com/apache/datafusion-comet/issues/6423
+    // https://github.com/apache/datafusion-comet/issues/6481
+    withSQLConf(
+      SQLConf.ANSI_ENABLED.key -> "false",
+      SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false",
+      SQLConf.SHUFFLE_PARTITIONS.key -> "1",
+      SQLConf.FILES_MAX_PARTITION_BYTES.key -> "1048576",
+      SQLConf.FILES_OPEN_COST_IN_BYTES.key -> "1048576",
+      CometConf.COMET_SHUFFLE_ENABLED.key -> "true",
+      CometConf.COMET_SHUFFLE_MODE.key -> "native") {
+      val swapped = RegrSparkVersions.r2DegenerateCasesSwapped(spark.version)
+      val constantX: java.lang.Double = if (swapped) null else 1.0
+      val constantY: java.lang.Double = if (swapped) 1.0 else null
+      for (reverse <- Seq(false, true)) {
+        withTempPath { path =>
+          val starts = if (reverse) Seq(3, 0) else Seq(0, 3)
+          for (start <- starts) {
+            (start until start + 3)
+              .map(y => (0, y.toDouble, 0.1))
+              .toDF("g", "y", "x")
+              .coalesce(1)
+              .write
+              .mode("append")
+              .parquet(path.getCanonicalPath)
+          }
+          withParquetTable(path.getCanonicalPath, "fractional_constants") {
+            assert(spark.table("fractional_constants").rdd.getNumPartitions == 2)
+            for (groupBy <- Seq("", " GROUP BY g")) {
+              val query = "SELECT regr_slope(y, x), regr_intercept(y, x), regr_r2(y, x), " +
+                "regr_sxx(y, x), regr_sxy(y, x), regr_r2(x, y), regr_syy(x, y) " +
+                "FROM fractional_constants" + groupBy
+              val (_, cometPlan) = checkSparkAnswerAndOperator(query)
+              val aggregates = cometPlan.collect { case a: CometHashAggregateExec => a }
+              assert(aggregates.exists(_.modes.contains(Partial)))
+              assert(aggregates.exists(_.modes.contains(Final)))
+              // Keep the degenerate-case results exact. A tolerance can hide nonzero M2.
+              checkAnswer(sql(query), Seq(Row(null, null, constantX, 0.0, 0.0, constantY, 0.0)))
+
+              val statsQuery = "SELECT corr(y, x), covar_pop(y, x), covar_samp(y, x), " +
+                "var_pop(x), var_samp(x), stddev_pop(x), stddev_samp(x) " +
+                "FROM fractional_constants" + groupBy
+              val (_, statsPlan) = checkSparkAnswerAndOperator(statsQuery)
+              val statsAggregates = statsPlan.collect { case a: CometHashAggregateExec => a }
+              assert(statsAggregates.exists(_.modes.contains(Partial)))
+              assert(statsAggregates.exists(_.modes.contains(Final)))
+              checkAnswer(sql(statsQuery), Seq(Row(null, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0)))
+            }
+          }
+        }
+      }
+    }
+  }
+
   test("var_pop and var_samp") {
     withSQLConf(CometConf.COMET_SHUFFLE_ENABLED.key -> "true") {
       Seq("native", "jvm").foreach { cometShuffleMode =>


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org