You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/04/08 18:15:51 UTC
(datafusion-comet) branch main updated: feat: Mark array_compact as Compatible and improve test coverage (#3889)
This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 8d3ad57b1 feat: Mark array_compact as Compatible and improve test coverage (#3889)
8d3ad57b1 is described below
commit 8d3ad57b1090b5bbbbcce87df81fcc49147b4712
Author: Andy Grove <ag...@apache.org>
AuthorDate: Wed Apr 8 12:15:46 2026 -0600
feat: Mark array_compact as Compatible and improve test coverage (#3889)
---
.../main/scala/org/apache/comet/serde/arrays.scala | 2 +-
.../org/apache/comet/shims/CometExprShim.scala | 6 ++++++
.../sql-tests/expressions/array/array_compact.sql | 24 +++++++++++++++++++---
3 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/arrays.scala b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
index 648c58bde..439325e36 100644
--- a/spark/src/main/scala/org/apache/comet/serde/arrays.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/arrays.scala
@@ -314,7 +314,7 @@ object CometArrayRepeat extends CometExpressionSerde[ArrayRepeat] {
object CometArrayCompact extends CometExpressionSerde[Expression] {
- override def getSupportLevel(expr: Expression): SupportLevel = Incompatible(None)
+ override def getSupportLevel(expr: Expression): SupportLevel = Compatible()
override def convert(
expr: Expression,
diff --git a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
index 2c5cebd16..bf8406c42 100644
--- a/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
+++ b/spark/src/main/spark-4.0/org/apache/comet/shims/CometExprShim.scala
@@ -113,6 +113,12 @@ trait CometExprShim extends CommonStringExprs {
// val optExpr = scalarFunctionExprToProto("width_bucket", childExprs: _*)
// optExprWithInfo(optExpr, wb, wb.children: _*)
+ // KnownNotContainsNull is a TaggingExpression added in Spark 4.0 that only
+ // changes schema metadata (containsNull = false). It has no runtime effect,
+ // so we pass through to the child expression.
+ case k: KnownNotContainsNull =>
+ exprToProtoInternal(k.child, inputs, binding)
+
case _ => None
}
}
diff --git a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
index 9b834a4db..83cd73097 100644
--- a/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
+++ b/spark/src/test/resources/sql-tests/expressions/array/array_compact.sql
@@ -15,7 +15,6 @@
-- specific language governing permissions and limitations
-- under the License.
--- ConfigMatrix: parquet.enable.dictionary=false,true
statement
CREATE TABLE test_array_compact(arr array<int>) USING parquet
@@ -23,9 +22,28 @@ CREATE TABLE test_array_compact(arr array<int>) USING parquet
statement
INSERT INTO test_array_compact VALUES (array(1, NULL, 2, NULL, 3)), (array()), (NULL), (array(NULL, NULL)), (array(1, 2, 3))
-query spark_answer_only
+-- column argument
+query
SELECT array_compact(arr) FROM test_array_compact
-- literal arguments
-query spark_answer_only
+query
SELECT array_compact(array(1, NULL, 2, NULL, 3))
+
+-- string element type
+statement
+CREATE TABLE test_array_compact_str(arr array<string>) USING parquet
+
+statement
+INSERT INTO test_array_compact_str VALUES (array('a', NULL, 'b', NULL, 'c')), (array()), (NULL), (array(NULL, NULL)), (array('', NULL, '', NULL))
+
+query
+SELECT array_compact(arr) FROM test_array_compact_str
+
+-- double element type
+query
+SELECT array_compact(array(1.0, NULL, 2.0, NULL, 3.0))
+
+-- nested array type (removes null arrays from outer, preserves null elements in inner)
+query
+SELECT array_compact(array(array(1, NULL, 3), NULL, array(NULL, 2, 3)))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org