You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/06/13 01:35:08 UTC
(datafusion) 01/01: fix: Enable sliding window execution for covar_pop, covar_samp, and corr (#22764)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22764-b8998c762bb864a9f3607a518384b03dcf40eb61
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 37768b8aadd0e9de543029778814d585588ff64f
Author: pchintar <89...@users.noreply.github.com>
AuthorDate: Sat Jun 13 07:04:43 2026 +0530
fix: Enable sliding window execution for covar_pop, covar_samp, and corr (#22764)
## Which issue does this PR close?
- Closes #22763 .
## Rationale for this change
Bounded sliding window queries using `covar_pop`, `covar_samp`, and
`corr` currently fail with a `retract_batch is not implemented` error,
preventing these aggregates from being used with sliding window frames.
## What changes are included in this PR?
* Included `supports_retract_batch()` for the covariance and correlation
accumulators.
* Added SQL logic tests covering bounded sliding window execution for
covariance and correlation aggregates.
## Are these changes tested?
Yes.
Added SQL logic tests covering:
* Single-row bounded sliding frames
* Multi-row bounded sliding frames
for `covar_pop`, `covar_samp`, and `corr`.
## Are there any user-facing changes?
Yes.
`covar_pop`, `covar_samp`, and `corr` can now be used with bounded
sliding window frames that previously failed. Also, no changes were made
to any public APIs.
---
datafusion/functions-aggregate/src/correlation.rs | 4 +
datafusion/functions-aggregate/src/covariance.rs | 12 ++
datafusion/functions-aggregate/src/variance.rs | 7 ++
datafusion/sqllogictest/test_files/window.slt | 136 ++++++++++++++++++++++
4 files changed, 159 insertions(+)
diff --git a/datafusion/functions-aggregate/src/correlation.rs b/datafusion/functions-aggregate/src/correlation.rs
index 2621fcf0bf..5a95cfe832 100644
--- a/datafusion/functions-aggregate/src/correlation.rs
+++ b/datafusion/functions-aggregate/src/correlation.rs
@@ -281,6 +281,10 @@ impl Accumulator for CorrelationAccumulator {
self.stddev2.retract_batch(&values[1..2])?;
Ok(())
}
+
+ fn supports_retract_batch(&self) -> bool {
+ true
+ }
}
#[derive(Default)]
diff --git a/datafusion/functions-aggregate/src/covariance.rs b/datafusion/functions-aggregate/src/covariance.rs
index 18d602ab33..bd7c8a0390 100644
--- a/datafusion/functions-aggregate/src/covariance.rs
+++ b/datafusion/functions-aggregate/src/covariance.rs
@@ -305,6 +305,14 @@ impl Accumulator for CovarianceAccumulator {
_ => continue,
};
+ if self.count <= 1 {
+ self.count = 0;
+ self.mean1 = 0.0;
+ self.mean2 = 0.0;
+ self.algo_const = 0.0;
+ continue;
+ }
+
let new_count = self.count - 1;
let delta1 = self.mean1 - value1;
let new_mean1 = delta1 / new_count as f64 + self.mean1;
@@ -373,4 +381,8 @@ impl Accumulator for CovarianceAccumulator {
fn size(&self) -> usize {
size_of_val(self)
}
+
+ fn supports_retract_batch(&self) -> bool {
+ true
+ }
}
diff --git a/datafusion/functions-aggregate/src/variance.rs b/datafusion/functions-aggregate/src/variance.rs
index ce3e00b9ff..d5fddf01f2 100644
--- a/datafusion/functions-aggregate/src/variance.rs
+++ b/datafusion/functions-aggregate/src/variance.rs
@@ -348,6 +348,13 @@ impl Accumulator for VarianceAccumulator {
fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> {
let arr = as_float64_array(&values[0])?;
for value in arr.iter().flatten() {
+ if self.count <= 1 {
+ self.count = 0;
+ self.mean = 0.0;
+ self.m2 = 0.0;
+ continue;
+ }
+
let new_count = self.count - 1;
let delta1 = self.mean - value;
let new_mean = delta1 / new_count as f64 + self.mean;
diff --git a/datafusion/sqllogictest/test_files/window.slt b/datafusion/sqllogictest/test_files/window.slt
index 1b51950a70..59b43d6476 100644
--- a/datafusion/sqllogictest/test_files/window.slt
+++ b/datafusion/sqllogictest/test_files/window.slt
@@ -6626,6 +6626,142 @@ ORDER BY i;
3 1
4 NULL
+# Covariance/correlation sliding-window regression test. Verifies correct
+# results across row removals and a NULL-gap empty-frame transition.
+query IRRR
+SELECT
+ column1,
+ covar_pop(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ ),
+ covar_samp(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ ),
+ corr(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ )
+FROM (
+ VALUES
+ (1, 10.0, 5.0),
+ (2, NULL, NULL),
+ (3, NULL, NULL),
+ (4, 30.0, 10.0),
+ (5, 40.0, 20.0),
+ (6, 50.0, 10.0)
+);
+----
+1 0 NULL NULL
+2 0 NULL NULL
+3 NULL NULL NULL
+4 0 NULL NULL
+5 25 50 1
+6 -25 -50 -1
+
+# Multi-row covariance/correlation sliding-window regression test. Verifies
+# correct accumulation when valid rows enter the frame after a reset.
+query IRRR
+SELECT
+ column1,
+ covar_pop(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
+ ),
+ covar_samp(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
+ ),
+ corr(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 2 PRECEDING AND CURRENT ROW
+ )
+FROM (
+ VALUES
+ (1, 10.0, 5.0),
+ (2, NULL, NULL),
+ (3, NULL, NULL),
+ (4, 30.0, 10.0),
+ (5, 40.0, 20.0),
+ (6, 50.0, 10.0)
+);
+----
+1 0 NULL NULL
+2 0 NULL NULL
+3 0 NULL NULL
+4 0 NULL NULL
+5 25 50 1
+6 0 0 0
+
+# Covariance/correlation sliding-window regression test. Rows with NULL in
+# either input column must not contribute to the aggregate state.
+query IRRR
+SELECT
+ column1,
+ covar_pop(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 3 PRECEDING AND CURRENT ROW
+ ),
+ covar_samp(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 3 PRECEDING AND CURRENT ROW
+ ),
+ corr(column2, column3) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 3 PRECEDING AND CURRENT ROW
+ )
+FROM (
+ VALUES
+ (1, 10.0, 5.0),
+ (2, 20.0, NULL),
+ (3, NULL, 15.0),
+ (4, 30.0, 10.0),
+ (5, 40.0, 20.0)
+);
+----
+1 0 NULL NULL
+2 0 NULL NULL
+3 0 NULL NULL
+4 25 50 1
+5 25 50 1
+
+# Variance/stddev sliding-window regression test. Verifies that retracting
+# the last valid row resets the aggregate state.
+query IRRRR
+SELECT
+ column1,
+ var_pop(column2) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ ),
+ var_samp(column2) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ ),
+ stddev_pop(column2) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ ),
+ stddev_samp(column2) OVER (
+ ORDER BY column1
+ ROWS BETWEEN 1 PRECEDING AND CURRENT ROW
+ )
+FROM (
+ VALUES
+ (1, 10.0),
+ (2, NULL),
+ (3, NULL),
+ (4, 30.0),
+ (5, 40.0)
+);
+----
+1 0 NULL 0 NULL
+2 0 NULL 0 NULL
+3 NULL NULL NULL NULL
+4 0 NULL 0 NULL
+5 25 50 5 7.071067811865
+
# Decimal variant — the integer-division path would otherwise panic on an
# empty frame.
query IR
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org