You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/02 12:55:55 UTC
(datafusion-comet) branch main updated: fix: run reverse of a binary value through the codegen dispatcher (#6461)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 351facd26c fix: run reverse of a binary value through the codegen dispatcher (#6461)
351facd26c is described below
commit 351facd26ca0febbfe0a4925549e3b89d22747d5
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri Oct 2 11:48:17 2026 +0000
fix: run reverse of a binary value through the codegen dispatcher (#6461)
* fix: run reverse of a binary value through the codegen dispatcher
Spark 4.2 accepts binary in reverse and reverses its bytes. CometReverse
sent every argument that is not an array to the native string reverse,
which fails the query on bytes that are not valid UTF-8. Report binary
input as Unsupported, so it runs Spark's own code through the codegen
dispatcher whether or not incompatible expressions are allowed.
Closes #6460.
* test: assert that binary reverse runs through the codegen dispatcher
---
docs/source/user-guide/latest/expressions.md | 2 +-
.../apache/comet/serde/collectionOperations.scala | 10 ++++++-
.../expressions/string/reverse_binary.sql | 32 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/docs/source/user-guide/latest/expressions.md b/docs/source/user-guide/latest/expressions.md
index 0cb4d96cf6..a809adedc7 100644
--- a/docs/source/user-guide/latest/expressions.md
+++ b/docs/source/user-guide/latest/expressions.md
@@ -206,7 +206,7 @@ The tables below list every Spark built-in expression with its current status.
| `array_size` | ✅ | — | |
| `cardinality` | ✅ | Native | |
| `concat` | ✅ | Hybrid | Binary/array children and non-UTF8_BINARY collations route through the JVM codegen dispatcher |
-| `reverse` | ✅ | Hybrid | Arrays with binary, struct, or map elements, and collated strings, route through the JVM codegen dispatcher ([details](compatibility/expressions/array.md)) |
+| `reverse` | ✅ | Hybrid | Binary input (Spark 4.2), arrays with binary, struct, or map elements, and collated strings, route through the JVM codegen dispatcher ([details](compatibility/expressions/array.md)) |
| `size` | ✅ | Native | |
---
diff --git a/spark/src/main/scala/org/apache/comet/serde/collectionOperations.scala b/spark/src/main/scala/org/apache/comet/serde/collectionOperations.scala
index 58ef333936..326c981ca9 100644
--- a/spark/src/main/scala/org/apache/comet/serde/collectionOperations.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/collectionOperations.scala
@@ -20,7 +20,7 @@
package org.apache.comet.serde
import org.apache.spark.sql.catalyst.expressions.{Attribute, Reverse, Shuffle}
-import org.apache.spark.sql.types.ArrayType
+import org.apache.spark.sql.types.{ArrayType, BinaryType}
import org.apache.comet.serde.ExprOuterClass.Expr
import org.apache.comet.serde.QueryPlanSerde.exprToProtoInternal
@@ -38,12 +38,20 @@ object CometReverse
"reverse does not support non-UTF8_BINARY collations " +
"(https://github.com/apache/datafusion-comet/issues/2190)"
+ // Spark 4.2 also accepts binary and reverses its bytes. The native reverse UDF only takes
+ // strings, so there is no native path for it.
+ private val binaryReason = "reverse has no native implementation for binary input"
+
override def getIncompatibleReasons(): Seq[String] =
CometArrayReverse.getIncompatibleReasons() :+ collationReason
+ override def getUnsupportedReasons(): Seq[String] = Seq(binaryReason)
+
override def getSupportLevel(expr: Reverse): SupportLevel = {
if (expr.child.dataType.isInstanceOf[ArrayType]) {
CometArrayReverse.getSupportLevel(expr)
+ } else if (expr.child.dataType == BinaryType) {
+ Unsupported(Some(binaryReason))
} else if (hasNonDefaultStringCollation(expr.child.dataType)) {
Incompatible(Some(collationReason))
} else {
diff --git a/spark/src/test/resources/sql-tests/expressions/string/reverse_binary.sql b/spark/src/test/resources/sql-tests/expressions/string/reverse_binary.sql
new file mode 100644
index 0000000000..6835728bf2
--- /dev/null
+++ b/spark/src/test/resources/sql-tests/expressions/string/reverse_binary.sql
@@ -0,0 +1,32 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- Spark 4.2 reverses the bytes of a binary argument and returns binary. Earlier versions cast it
+-- to a string first, which reverse.sql covers. Comet has no native reverse for binary, so it runs
+-- Spark's code through the codegen dispatcher, whether or not incompatible expressions are allowed.
+-- MinSparkVersion: 4.2
+-- ConfigMatrix: spark.comet.expression.Reverse.allowIncompatible=false,true
+
+statement
+CREATE TABLE test_reverse_binary(b binary) USING parquet
+
+-- X'636166C3A9' is 'café' in UTF-8: reversed by byte, not by character, it is not valid UTF-8.
+statement
+INSERT INTO test_reverse_binary VALUES (X'CAFE'), (X''), (NULL), (X'01'), (X'636166C3A9')
+
+query expect_dispatch(reverse)
+SELECT reverse(b) FROM test_reverse_binary
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org