You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by vi...@apache.org on 2022/08/29 20:42:29 UTC
[arrow-rs] branch master updated: Compare dictionary array with string array (#2549)
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 171f80b31 Compare dictionary array with string array (#2549)
171f80b31 is described below
commit 171f80b31dbdea9b34ea516f392b018272dcaebf
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Mon Aug 29 13:42:25 2022 -0700
Compare dictionary array with string array (#2549)
* Compare dictionary array to string array
* Trigger Build
---
arrow/src/compute/kernels/comparison.rs | 179 ++++++++++++++++++++++++++++++++
1 file changed, 179 insertions(+)
diff --git a/arrow/src/compute/kernels/comparison.rs b/arrow/src/compute/kernels/comparison.rs
index 1c346a311..2fedd8940 100644
--- a/arrow/src/compute/kernels/comparison.rs
+++ b/arrow/src/compute/kernels/comparison.rs
@@ -2133,6 +2133,50 @@ macro_rules! typed_dict_non_dict_cmp {
}};
}
+#[cfg(feature = "dyn_cmp_dict")]
+macro_rules! typed_dict_string_array_cmp {
+ ($LEFT: expr, $RIGHT: expr, $LEFT_KEY_TYPE: expr, $RIGHT_TYPE: tt, $OP: expr) => {{
+ match $LEFT_KEY_TYPE {
+ DataType::Int8 => {
+ let left = as_dictionary_array::<Int8Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::Int16 => {
+ let left = as_dictionary_array::<Int16Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::Int32 => {
+ let left = as_dictionary_array::<Int32Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::Int64 => {
+ let left = as_dictionary_array::<Int64Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::UInt8 => {
+ let left = as_dictionary_array::<UInt8Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::UInt16 => {
+ let left = as_dictionary_array::<UInt16Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::UInt32 => {
+ let left = as_dictionary_array::<UInt32Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ DataType::UInt64 => {
+ let left = as_dictionary_array::<UInt64Type>($LEFT);
+ cmp_dict_string_array::<_, $RIGHT_TYPE, _>(left, $RIGHT, $OP)
+ }
+ t => Err(ArrowError::NotYetImplemented(format!(
+ "Cannot compare dictionary array of key type {}",
+ t
+ ))),
+ }
+ }};
+}
+
#[cfg(feature = "dyn_cmp_dict")]
macro_rules! typed_cmp_dict_non_dict {
($LEFT: expr, $RIGHT: expr, $OP_BOOL: expr, $OP: expr, $OP_FLOAT: expr) => {{
@@ -2169,6 +2213,12 @@ macro_rules! typed_cmp_dict_non_dict {
(DataType::Float64, DataType::Float64) => {
typed_dict_non_dict_cmp!($LEFT, $RIGHT, left_key_type.as_ref(), Float64Type, $OP_BOOL, $OP_FLOAT)
}
+ (DataType::Utf8, DataType::Utf8) => {
+ typed_dict_string_array_cmp!($LEFT, $RIGHT, left_key_type.as_ref(), i32, $OP)
+ }
+ (DataType::LargeUtf8, DataType::LargeUtf8) => {
+ typed_dict_string_array_cmp!($LEFT, $RIGHT, left_key_type.as_ref(), i64, $OP)
+ }
(t1, t2) if t1 == t2 => Err(ArrowError::NotYetImplemented(format!(
"Comparing dictionary array of type {} with array of type {} is not yet implemented",
t1, t2
@@ -2538,6 +2588,29 @@ where
)
}
+/// Perform given operation on `DictionaryArray` and `GenericStringArray`. The value
+/// type of `DictionaryArray` is same as `GenericStringArray`'s type.
+#[cfg(feature = "dyn_cmp_dict")]
+fn cmp_dict_string_array<K, OffsetSize: OffsetSizeTrait, F>(
+ left: &DictionaryArray<K>,
+ right: &dyn Array,
+ op: F,
+) -> Result<BooleanArray>
+where
+ K: ArrowNumericType,
+ F: Fn(&str, &str) -> bool,
+{
+ compare_op(
+ left.downcast_dict::<GenericStringArray<OffsetSize>>()
+ .unwrap(),
+ right
+ .as_any()
+ .downcast_ref::<GenericStringArray<OffsetSize>>()
+ .unwrap(),
+ op,
+ )
+}
+
/// Perform given operation on two `DictionaryArray`s which value type is
/// primitive type. Returns an error if the two arrays have different value
/// type
@@ -6480,6 +6553,112 @@ mod tests {
);
}
+ #[test]
+ #[cfg(feature = "dyn_cmp_dict")]
+ fn test_eq_dyn_neq_dyn_dictionary_to_utf8_array() {
+ let test1 = vec!["a", "a", "b", "c"];
+ let test2 = vec!["a", "b", "b", "d"];
+
+ let dict_array: DictionaryArray<Int8Type> = test1
+ .iter()
+ .map(|&x| if x == "b" { None } else { Some(x) })
+ .collect();
+
+ let array: StringArray = test2
+ .iter()
+ .map(|&x| if x == "b" { None } else { Some(x) })
+ .collect();
+
+ let result = eq_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(false)])
+ );
+
+ let result = eq_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(false)])
+ );
+
+ let result = neq_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(true)])
+ );
+
+ let result = neq_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(true)])
+ );
+ }
+
+ #[test]
+ #[cfg(feature = "dyn_cmp_dict")]
+ fn test_lt_dyn_lt_eq_dyn_gt_dyn_gt_eq_dyn_dictionary_to_utf8_array() {
+ let test1 = vec!["abc", "abc", "b", "cde"];
+ let test2 = vec!["abc", "b", "b", "def"];
+
+ let dict_array: DictionaryArray<Int8Type> = test1
+ .iter()
+ .map(|&x| if x == "b" { None } else { Some(x) })
+ .collect();
+
+ let array: StringArray = test2
+ .iter()
+ .map(|&x| if x == "b" { None } else { Some(x) })
+ .collect();
+
+ let result = lt_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(true)])
+ );
+
+ let result = lt_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(false)])
+ );
+
+ let result = lt_eq_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(true)])
+ );
+
+ let result = lt_eq_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(false)])
+ );
+
+ let result = gt_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(false)])
+ );
+
+ let result = gt_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(false), None, None, Some(true)])
+ );
+
+ let result = gt_eq_dyn(&dict_array, &array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(false)])
+ );
+
+ let result = gt_eq_dyn(&array, &dict_array);
+ assert_eq!(
+ result.unwrap(),
+ BooleanArray::from(vec![Some(true), None, None, Some(true)])
+ );
+ }
+
#[test]
fn test_dict_nlike_kernels() {
let data =