You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by dh...@apache.org on 2023/05/19 15:21:56 UTC
[arrow-datafusion] branch support-is-distinct-from-for-binaryarray created (now 44abd6a0d5)
This is an automated email from the ASF dual-hosted git repository.
dheres pushed a change to branch support-is-distinct-from-for-binaryarray
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
at 44abd6a0d5 Support is [not] distinct from for binaryarray types
This branch includes the following new commits:
new 44abd6a0d5 Support is [not] distinct from for binaryarray types
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[arrow-datafusion] 01/01: Support is [not] distinct from for binaryarray types
Posted by dh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dheres pushed a commit to branch support-is-distinct-from-for-binaryarray
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
commit 44abd6a0d590167335a07956c328968f9463af07
Author: Daniël Heres <da...@coralogix.com>
AuthorDate: Fri May 19 17:21:03 2023 +0200
Support is [not] distinct from for binaryarray types
---
.../core/tests/sqllogictests/test_files/select.slt | 15 ++++++++++++++
datafusion/physical-expr/src/expressions/binary.rs | 23 ++++++++++++++++++++--
.../src/expressions/binary/kernels_arrow.rs | 22 +++++++++++++++++++++
testing | 2 +-
4 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/datafusion/core/tests/sqllogictests/test_files/select.slt b/datafusion/core/tests/sqllogictests/test_files/select.slt
index 03f96bad95..8ab7cd1b13 100644
--- a/datafusion/core/tests/sqllogictests/test_files/select.slt
+++ b/datafusion/core/tests/sqllogictests/test_files/select.slt
@@ -298,6 +298,21 @@ select column1 is not distinct from column2 from t;
false
+# Binary Expression for Binary
+# issue: https://github.com/apache/arrow-datafusion/issues/5893
+statement ok
+CREATE TABLE t as select arrow_cast('Bar', 'Binary') as column1, 'B%' as column2;
+
+query B
+select column1 is distinct from column2 from t;
+----
+true
+
+query B
+select column1 is not distinct from column2 from t;
+----
+false
+
# select all
# these two queries should return the same result
query R
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index 7bdbba88a8..a93c377112 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -71,10 +71,10 @@ use kernels_arrow::{
add_decimal_dyn_scalar, add_dyn_decimal, add_dyn_temporal, add_dyn_temporal_scalar,
divide_decimal_dyn_scalar, divide_dyn_opt_decimal, is_distinct_from,
is_distinct_from_bool, is_distinct_from_decimal, is_distinct_from_f32,
- is_distinct_from_f64, is_distinct_from_null, is_distinct_from_utf8,
+ is_distinct_from_f64, is_distinct_from_null, is_distinct_from_utf8, is_distinct_from_binary,
is_not_distinct_from, is_not_distinct_from_bool, is_not_distinct_from_decimal,
is_not_distinct_from_f32, is_not_distinct_from_f64, is_not_distinct_from_null,
- is_not_distinct_from_utf8, modulus_decimal_dyn_scalar, modulus_dyn_decimal,
+ is_not_distinct_from_utf8, is_not_distinct_from_binary, modulus_decimal_dyn_scalar, modulus_dyn_decimal,
multiply_decimal_dyn_scalar, multiply_dyn_decimal, subtract_decimal_dyn_scalar,
subtract_dyn_decimal, subtract_dyn_temporal, subtract_dyn_temporal_scalar,
};
@@ -245,6 +245,22 @@ macro_rules! compute_utf8_op {
}};
}
+/// Invoke a compute kernel on a pair of binary data arrays
+macro_rules! compute_binary_op {
+ ($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident) => {{
+ let ll = $LEFT
+ .as_any()
+ .downcast_ref::<$DT>()
+ .expect("compute_op failed to downcast left side array");
+ let rr = $RIGHT
+ .as_any()
+ .downcast_ref::<$DT>()
+ .expect("compute_op failed to downcast right side array");
+ Ok(Arc::new(paste::expr! {[<$OP _binary>]}(&ll, &rr)?))
+ }};
+}
+
+
/// Invoke a compute kernel on a data array and a scalar value
macro_rules! compute_utf8_op_scalar {
($LEFT:expr, $RIGHT:expr, $OP:ident, $DT:ident, $OP_TYPE:expr) => {{
@@ -510,7 +526,10 @@ macro_rules! binary_array_op {
DataType::Float32 => compute_f32_op!($LEFT, $RIGHT, $OP, Float32Array),
DataType::Float64 => compute_f64_op!($LEFT, $RIGHT, $OP, Float64Array),
DataType::Utf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, StringArray),
+ DataType::Binary => compute_binary_op!($LEFT, $RIGHT, $OP, BinaryArray),
+ DataType::LargeBinary => compute_binary_op!($LEFT, $RIGHT, $OP, LargeBinaryArray),
DataType::LargeUtf8 => compute_utf8_op!($LEFT, $RIGHT, $OP, LargeStringArray),
+
DataType::Timestamp(TimeUnit::Nanosecond, _) => {
compute_op!($LEFT, $RIGHT, $OP, TimestampNanosecondArray)
}
diff --git a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
index 90fca17157..50a9f86c06 100644
--- a/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
+++ b/datafusion/physical-expr/src/expressions/binary/kernels_arrow.rs
@@ -210,6 +210,17 @@ pub(crate) fn is_distinct_from_utf8<OffsetSize: OffsetSizeTrait>(
.collect())
}
+pub(crate) fn is_distinct_from_binary<OffsetSize: OffsetSizeTrait>(
+ left: &GenericBinaryArray<OffsetSize>,
+ right: &GenericBinaryArray<OffsetSize>,
+) -> Result<BooleanArray> {
+ Ok(left
+ .iter()
+ .zip(right.iter())
+ .map(|(x, y)| Some(x != y))
+ .collect())
+}
+
pub(crate) fn is_distinct_from_null(
left: &NullArray,
_right: &NullArray,
@@ -241,6 +252,17 @@ pub(crate) fn is_not_distinct_from_utf8<OffsetSize: OffsetSizeTrait>(
.collect())
}
+pub(crate) fn is_not_distinct_from_binary<OffsetSize: OffsetSizeTrait>(
+ left: &GenericBinaryArray<OffsetSize>,
+ right: &GenericBinaryArray<OffsetSize>,
+) -> Result<BooleanArray> {
+ Ok(left
+ .iter()
+ .zip(right.iter())
+ .map(|(x, y)| Some(x == y))
+ .collect())
+}
+
pub(crate) fn is_distinct_from_decimal(
left: &Decimal128Array,
right: &Decimal128Array,
diff --git a/testing b/testing
index e81d0c6de3..5bab2f264a 160000
--- a/testing
+++ b/testing
@@ -1 +1 @@
-Subproject commit e81d0c6de35948b3be7984af8e00413b314cde6e
+Subproject commit 5bab2f264a23f5af68f69ea93d24ef1e8e77fc88