You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/03 13:06:53 UTC
(datafusion) branch gh-readonly-queue/main/pr-25951-22a0fdb64614f4d751ed0009dbe1ac850ec79282 created (now a93c9b8b5e)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-25951-22a0fdb64614f4d751ed0009dbe1ac850ec79282
in repository https://gitbox.apache.org/repos/asf/datafusion.git
at a93c9b8b5e feat: Support RunEndEncoded type in approx_distinct (#25951)
This branch includes the following new commits:
new a93c9b8b5e feat: Support RunEndEncoded type in approx_distinct (#25951)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion) 01/01: feat: Support RunEndEncoded type in approx_distinct (#25951)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-25951-22a0fdb64614f4d751ed0009dbe1ac850ec79282
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit a93c9b8b5e52d6edf19bc6d929860a4ec4ec5df2
Author: Michael Kleen <mk...@gmail.com>
AuthorDate: Sat Oct 3 13:06:33 2026 +0000
feat: Support RunEndEncoded type in approx_distinct (#25951)
## Which issue does this PR close?
- Closes #22989
## Rationale for this change
- Support the Arrow type `RunEndEncoded` type for `approx_distinct`
## What changes are included in this PR?
- Enable `HLLAccumulator` and `HllGroupsAccumulator` to support
`RunEndEncoded`
- Tests for the non-grouped and grouped path as part of `aggregate.slt`
## Are these changes tested?
Yes.
## Are there any user-facing changes?
Yes, `approx_distinct` supports now `RunEndEncoded` but no breaking
changes.
---
.../functions-aggregate/src/approx_distinct.rs | 22 ++++++----
datafusion/sqllogictest/test_files/aggregate.slt | 49 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 7 deletions(-)
diff --git a/datafusion/functions-aggregate/src/approx_distinct.rs b/datafusion/functions-aggregate/src/approx_distinct.rs
index a54a382d89..1a37249dfb 100644
--- a/datafusion/functions-aggregate/src/approx_distinct.rs
+++ b/datafusion/functions-aggregate/src/approx_distinct.rs
@@ -835,7 +835,9 @@ impl AggregateUDFImpl for ApproxDistinct {
| DataType::Struct(_)
| DataType::Union(_, _)
| DataType::LargeBinary => Box::new(HLLAccumulator::new()),
- DataType::Dictionary(_, _) if is_supported_type(data_type) => {
+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
+ if is_supported_type(data_type) =>
+ {
Box::new(HLLAccumulator::new())
}
DataType::Null => {
@@ -894,25 +896,31 @@ fn is_fixed_domain_type(data_type: &DataType) -> bool {
}
fn is_supported_type(data_type: &DataType) -> bool {
- let value_type = dictionary_value_type(data_type);
+ let value_type = encoded_value_type(data_type);
matches!(value_type, DataType::Null)
|| is_fixed_domain_type(value_type)
|| is_hll_groups_type(value_type)
}
-fn dictionary_value_type(data_type: &DataType) -> &DataType {
+fn encoded_value_type(data_type: &DataType) -> &DataType {
let mut value_type = data_type;
- while let DataType::Dictionary(_, inner) = value_type {
- value_type = inner;
+ loop {
+ match value_type {
+ DataType::Dictionary(_, inner) => value_type = inner,
+ DataType::RunEndEncoded(_, values) => value_type = values.data_type(),
+ _ => return value_type,
+ }
}
- value_type
}
/// Returns true for the data types backed by the HyperLogLog
/// [`HllGroupsAccumulator`]. The fixed-domain types (booleans / small ints) and
/// `Null` fall back to the per-group [`Accumulator`] path.
fn is_hll_groups_type(data_type: &DataType) -> bool {
- if matches!(data_type, DataType::Dictionary(_, _)) {
+ if matches!(
+ data_type,
+ DataType::Dictionary(_, _) | DataType::RunEndEncoded(_, _)
+ ) {
return is_supported_type(data_type);
}
diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt
index a53de58134..d72372dee4 100644
--- a/datafusion/sqllogictest/test_files/aggregate.slt
+++ b/datafusion/sqllogictest/test_files/aggregate.slt
@@ -1862,6 +1862,55 @@ SELECT approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_
statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type Dictionary\(Int32, Float64\) is not implemented
SELECT g, approx_distinct(arrow_cast(i, 'Dictionary(Int32, Float64)')) FROM approx_distinct_group_test GROUP BY g;
+# RunEndEncoded: run-end encoded values must hash identically to the plain
+# values, so the counts below match the Utf8 / Int32 cases above.
+query I
+SELECT approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test WHERE g = 2;
+----
+2
+
+query II
+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Utf8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+query II
+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int64, "values": Int32)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# RunEndEncoded over a fixed-domain value type (Int8) goes through the
+# HyperLogLog accumulators and must match the Int32 counts above.
+query II
+SELECT g, approx_distinct(arrow_cast(arrow_cast(i, 'Int8'), 'RunEndEncoded("run_ends": non-null Int32, "values": Int8)')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# RunEndEncoded over a dictionary
+query II
+SELECT g, approx_distinct(arrow_cast(s, 'RunEndEncoded("run_ends": non-null Int32, "values": Dictionary(Int32, Utf8))')) FROM approx_distinct_group_test GROUP BY g ORDER BY g;
+----
+1 2
+2 2
+3 0
+4 1
+
+# Unsupported value types are rejected, as for a bare Float64
+statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
+SELECT approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test;
+
+statement error DataFusion error: This feature is not implemented: Support for 'approx_distinct' for data type RunEndEncoded
+SELECT g, approx_distinct(arrow_cast(i, 'RunEndEncoded("run_ends": non-null Int32, "values": Float64)')) FROM approx_distinct_group_test GROUP BY g;
+
# List
statement ok
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org