You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/04/15 14:02:16 UTC
(datafusion) 01/01: feat: support `array_compact` builtin function (#21522)
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch gh-readonly-queue/main/pr-21522-244f89120b254f824b6fed0f8a88c23a74bf09d7
in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 26c6121f037bf347cad6b71fa81c67de81a9f1c5
Author: Oleks V <co...@users.noreply.github.com>
AuthorDate: Wed Apr 15 07:01:49 2026 -0700
feat: support `array_compact` builtin function (#21522)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes #.
## Rationale for this change
Create `array_compact` function which removes NULLs from input array.
There is no direct counterparty in DuckDB however the function used in
Spark, SnowFlake
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
## What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
## Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---
datafusion/functions-nested/src/array_compact.rs | 191 +++++++++++++++++++++++
datafusion/functions-nested/src/lib.rs | 3 +
datafusion/sqllogictest/test_files/array.slt | 93 +++++++++++
docs/source/user-guide/sql/scalar_functions.md | 33 ++++
4 files changed, 320 insertions(+)
diff --git a/datafusion/functions-nested/src/array_compact.rs b/datafusion/functions-nested/src/array_compact.rs
new file mode 100644
index 0000000000..11be494b5b
--- /dev/null
+++ b/datafusion/functions-nested/src/array_compact.rs
@@ -0,0 +1,191 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! [`ScalarUDFImpl`] definitions for array_compact function.
+
+use crate::utils::make_scalar_function;
+use arrow::array::{
+ Array, ArrayRef, Capacities, GenericListArray, MutableArrayData, OffsetSizeTrait,
+ make_array,
+};
+use arrow::buffer::OffsetBuffer;
+use arrow::datatypes::DataType;
+use arrow::datatypes::DataType::{LargeList, List, Null};
+use datafusion_common::cast::{as_large_list_array, as_list_array};
+use datafusion_common::{Result, exec_err, utils::take_function_args};
+use datafusion_expr::{
+ ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+ Volatility,
+};
+use datafusion_macros::user_doc;
+use std::sync::Arc;
+
+make_udf_expr_and_func!(
+ ArrayCompact,
+ array_compact,
+ array,
+ "removes null values from the array.",
+ array_compact_udf
+);
+
+#[user_doc(
+ doc_section(label = "Array Functions"),
+ description = "Removes null values from the array.",
+ syntax_example = "array_compact(array)",
+ sql_example = r#"```sql
+> select array_compact([1, NULL, 2, NULL, 3]) arr;
++-----------+
+| arr |
++-----------+
+| [1, 2, 3] |
++-----------+
+```"#,
+ argument(
+ name = "array",
+ description = "Array expression. Can be a constant, column, or function, and any combination of array operators."
+ )
+)]
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct ArrayCompact {
+ signature: Signature,
+ aliases: Vec<String>,
+}
+
+impl Default for ArrayCompact {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl ArrayCompact {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::array(Volatility::Immutable),
+ aliases: vec!["list_compact".to_string()],
+ }
+ }
+}
+
+impl ScalarUDFImpl for ArrayCompact {
+ fn name(&self) -> &str {
+ "array_compact"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+ Ok(arg_types[0].clone())
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+ make_scalar_function(array_compact_inner)(&args.args)
+ }
+
+ fn aliases(&self) -> &[String] {
+ &self.aliases
+ }
+
+ fn documentation(&self) -> Option<&Documentation> {
+ self.doc()
+ }
+}
+
+/// array_compact SQL function
+fn array_compact_inner(arg: &[ArrayRef]) -> Result<ArrayRef> {
+ let [input_array] = take_function_args("array_compact", arg)?;
+
+ match &input_array.data_type() {
+ List(field) => {
+ let array = as_list_array(input_array)?;
+ compact_list::<i32>(array, field)
+ }
+ LargeList(field) => {
+ let array = as_large_list_array(input_array)?;
+ compact_list::<i64>(array, field)
+ }
+ Null => Ok(Arc::clone(input_array)),
+ array_type => exec_err!("array_compact does not support type '{array_type}'."),
+ }
+}
+
+/// Remove null elements from each row of a list array.
+fn compact_list<O: OffsetSizeTrait>(
+ list_array: &GenericListArray<O>,
+ field: &Arc<arrow::datatypes::Field>,
+) -> Result<ArrayRef> {
+ let values = list_array.values();
+
+ // Fast path: no nulls in values, return input unchanged
+ if values.null_count() == 0 {
+ return Ok(Arc::new(list_array.clone()));
+ }
+
+ let original_data = values.to_data();
+ let capacity = original_data.len() - values.null_count();
+ let mut offsets = Vec::<O>::with_capacity(list_array.len() + 1);
+ offsets.push(O::zero());
+ let mut mutable = MutableArrayData::with_capacities(
+ vec![&original_data],
+ false,
+ Capacities::Array(capacity),
+ );
+
+ for row_index in 0..list_array.len() {
+ if list_array.nulls().is_some_and(|n| n.is_null(row_index)) {
+ offsets.push(offsets[row_index]);
+ continue;
+ }
+
+ let start = list_array.offsets()[row_index].as_usize();
+ let end = list_array.offsets()[row_index + 1].as_usize();
+ let mut copied = 0usize;
+
+ // Batch consecutive non-null elements into single extend() calls
+ // to reduce per-element overhead. For [1, 2, NULL, 3, 4] this
+ // produces 2 extend calls (0..2, 3..5) instead of 4 individual ones.
+ let mut batch_start: Option<usize> = None;
+ for i in start..end {
+ if values.is_null(i) {
+ // Null breaks the current batch — flush it
+ if let Some(bs) = batch_start {
+ mutable.extend(0, bs, i);
+ copied += i - bs;
+ batch_start = None;
+ }
+ } else if batch_start.is_none() {
+ batch_start = Some(i);
+ }
+ }
+ // Flush any remaining batch after the loop
+ if let Some(bs) = batch_start {
+ mutable.extend(0, bs, end);
+ copied += end - bs;
+ }
+
+ offsets.push(offsets[row_index] + O::usize_as(copied));
+ }
+
+ let new_values = make_array(mutable.freeze());
+ Ok(Arc::new(GenericListArray::<O>::try_new(
+ Arc::clone(field),
+ OffsetBuffer::new(offsets.into()),
+ new_values,
+ list_array.nulls().cloned(),
+ )?))
+}
diff --git a/datafusion/functions-nested/src/lib.rs b/datafusion/functions-nested/src/lib.rs
index 99b25ec964..99c33fb640 100644
--- a/datafusion/functions-nested/src/lib.rs
+++ b/datafusion/functions-nested/src/lib.rs
@@ -37,6 +37,7 @@
#[macro_use]
pub mod macros;
+pub mod array_compact;
pub mod array_has;
pub mod arrays_zip;
pub mod cardinality;
@@ -77,6 +78,7 @@ use std::sync::Arc;
/// Fluent-style API for creating `Expr`s
pub mod expr_fn {
+ pub use super::array_compact::array_compact;
pub use super::array_has::array_has;
pub use super::array_has::array_has_all;
pub use super::array_has::array_has_any;
@@ -128,6 +130,7 @@ pub mod expr_fn {
/// Return all default nested type functions
pub fn all_default_nested_functions() -> Vec<Arc<ScalarUDF>> {
vec![
+ array_compact::array_compact_udf(),
string::array_to_string_udf(),
string::string_to_array_udf(),
range::range_udf(),
diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt
index 81d5c8f91a..7b6779abe4 100644
--- a/datafusion/sqllogictest/test_files/array.slt
+++ b/datafusion/sqllogictest/test_files/array.slt
@@ -7685,6 +7685,99 @@ from array_distinct_table_2D_fixed;
[[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
[[5, 6], NULL]
+## array_compact (aliases: `list_compact`)
+
+# basic: remove nulls from integer array
+query ?
+select array_compact([1, NULL, 2, NULL, 3]);
+----
+[1, 2, 3]
+
+# no nulls present
+query ?
+select array_compact([1, 2, 3]);
+----
+[1, 2, 3]
+
+# all nulls
+query ?
+select array_compact(arrow_cast([NULL, NULL, NULL], 'List(Int64)'));
+----
+[]
+
+# empty array
+query ?
+select array_compact([]);
+----
+[]
+
+# NULL input returns NULL
+query ?
+select array_compact(NULL::INT[]);
+----
+NULL
+
+# string array
+query ?
+select array_compact(['a', NULL, 'b', NULL, 'c']);
+----
+[a, b, c]
+
+# float array
+query ?
+select array_compact([1.0, NULL, 2.0, NULL]);
+----
+[1.0, 2.0]
+
+# nested array (2D)
+query ?
+select array_compact([make_array(1, 2), NULL, make_array(3, 4)]);
+----
+[[1, 2], [3, 4]]
+
+# LargeList
+query ?
+select array_compact(arrow_cast([1, NULL, 2, NULL, 3], 'LargeList(Int64)'));
+----
+[1, 2, 3]
+
+# alias list_compact
+query ?
+select list_compact([1, NULL, 2]);
+----
+[1, 2]
+
+# table-based test
+statement ok
+CREATE TABLE array_compact_table AS VALUES
+ (make_array(1, NULL, 2, NULL, 3)),
+ (make_array(NULL, NULL, NULL)),
+ (make_array(4, 5, 6)),
+ (NULL::INT[])
+;
+
+query ?
+select array_compact(column1) from array_compact_table;
+----
+[1, 2, 3]
+[]
+[4, 5, 6]
+NULL
+
+statement ok
+DROP TABLE array_compact_table;
+
+# FixedSizeList (coerced to List)
+query ?
+select array_compact(arrow_cast(make_array(1, NULL, 2, NULL, 3), 'FixedSizeList(5, Int64)'));
+----
+[1, 2, 3]
+
+query ?
+select array_compact(arrow_cast(make_array(NULL, NULL, NULL), 'FixedSizeList(3, Int64)'));
+----
+[]
+
## arrays_zip (aliases: `list_zip`)
# Spark example: arrays_zip(array(1, 2, 3), array(2, 3, 4))
diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md
index ca8707270f..9f5814447b 100644
--- a/docs/source/user-guide/sql/scalar_functions.md
+++ b/docs/source/user-guide/sql/scalar_functions.md
@@ -3237,6 +3237,7 @@ _Alias of [current_date](#current_date)._
- [array_any_value](#array_any_value)
- [array_append](#array_append)
- [array_cat](#array_cat)
+- [array_compact](#array_compact)
- [array_concat](#array_concat)
- [array_contains](#array_contains)
- [array_dims](#array_dims)
@@ -3285,6 +3286,7 @@ _Alias of [current_date](#current_date)._
- [list_any_value](#list_any_value)
- [list_append](#list_append)
- [list_cat](#list_cat)
+- [list_compact](#list_compact)
- [list_concat](#list_concat)
- [list_contains](#list_contains)
- [list_dims](#list_dims)
@@ -3391,6 +3393,33 @@ array_append(array, element)
_Alias of [array_concat](#array_concat)._
+### `array_compact`
+
+Removes null values from the array.
+
+```sql
+array_compact(array)
+```
+
+#### Arguments
+
+- **array**: Array expression. Can be a constant, column, or function, and any combination of array operators.
+
+#### Example
+
+```sql
+> select array_compact([1, NULL, 2, NULL, 3]) arr;
++-----------+
+| arr |
++-----------+
+| [1, 2, 3] |
++-----------+
+```
+
+#### Aliases
+
+- list_compact
+
### `array_concat`
Concatenates arrays.
@@ -4535,6 +4564,10 @@ _Alias of [array_append](#array_append)._
_Alias of [array_concat](#array_concat)._
+### `list_compact`
+
+_Alias of [array_compact](#array_compact)._
+
### `list_concat`
_Alias of [array_concat](#array_concat)._
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org