You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2022/09/27 17:20:20 UTC

[arrow-rs] branch master updated: Move unary kernels to arrow-array (#2787) (#2789)

This is an automated email from the ASF dual-hosted git repository.

tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git


The following commit(s) were added to refs/heads/master by this push:
     new 2ba130761 Move unary kernels to arrow-array (#2787) (#2789)
2ba130761 is described below

commit 2ba130761ce5ba90e177b6e4e3496d83a9d4b912
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Tue Sep 27 18:20:14 2022 +0100

    Move unary kernels to arrow-array (#2787) (#2789)
    
    * Move primitive arity kernels (#2787)
    
    * Fix doc
---
 arrow-array/src/array/primitive_array.rs | 119 +++++++++++++++++++++++++++----
 arrow/src/compute/kernels/arity.rs       |  76 ++------------------
 2 files changed, 108 insertions(+), 87 deletions(-)

diff --git a/arrow-array/src/array/primitive_array.rs b/arrow-array/src/array/primitive_array.rs
index f9e4e7675..4ac191ac9 100644
--- a/arrow-array/src/array/primitive_array.rs
+++ b/arrow-array/src/array/primitive_array.rs
@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::builder::{BooleanBufferBuilder, PrimitiveBuilder};
+use crate::builder::{BooleanBufferBuilder, BufferBuilder, PrimitiveBuilder};
 use crate::iterator::PrimitiveIter;
 use crate::raw_pointer::RawPtrBox;
 use crate::temporal_conversions::{as_date, as_datetime, as_duration, as_time};
@@ -23,6 +23,7 @@ use crate::trusted_len::trusted_len_unzip;
 use crate::types::*;
 use crate::{print_long_array, Array, ArrayAccessor};
 use arrow_buffer::{bit_util, ArrowNativeType, Buffer, MutableBuffer};
+use arrow_data::bit_iterator::try_for_each_valid_idx;
 use arrow_data::ArrayData;
 use arrow_schema::DataType;
 use chrono::{Duration, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime};
@@ -298,20 +299,10 @@ impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
 
     /// Creates a PrimitiveArray based on a constant value with `count` elements
     pub fn from_value(value: T::Native, count: usize) -> Self {
-        // # Safety: iterator (0..count) correctly reports its length
-        let val_buf = unsafe { Buffer::from_trusted_len_iter((0..count).map(|_| value)) };
-        let data = unsafe {
-            ArrayData::new_unchecked(
-                T::DATA_TYPE,
-                val_buf.len() / std::mem::size_of::<<T as ArrowPrimitiveType>::Native>(),
-                None,
-                None,
-                0,
-                vec![val_buf],
-                vec![],
-            )
-        };
-        PrimitiveArray::from(data)
+        unsafe {
+            let val_buf = Buffer::from_trusted_len_iter((0..count).map(|_| value));
+            build_primitive_array(count, val_buf, 0, None)
+        }
     }
 
     /// Returns an iterator that returns the values of `array.value(i)` for an iterator with each element `i`
@@ -332,6 +323,104 @@ impl<T: ArrowPrimitiveType> PrimitiveArray<T> {
     ) -> impl Iterator<Item = Option<T::Native>> + 'a {
         indexes.map(|opt_index| opt_index.map(|index| self.value_unchecked(index)))
     }
+
+    /// Applies an unary and infallible function to a primitive array.
+    /// This is the fastest way to perform an operation on a primitive array when
+    /// the benefits of a vectorized operation outweigh the cost of branching nulls and non-nulls.
+    ///
+    /// # Implementation
+    ///
+    /// This will apply the function for all values, including those on null slots.
+    /// This implies that the operation must be infallible for any value of the corresponding type
+    /// or this function may panic.
+    /// # Example
+    /// ```rust
+    /// # use arrow_array::{Int32Array, types::Int32Type};
+    /// # fn main() {
+    /// let array = Int32Array::from(vec![Some(5), Some(7), None]);
+    /// let c = array.unary(|x| x * 2 + 1);
+    /// assert_eq!(c, Int32Array::from(vec![Some(11), Some(15), None]));
+    /// # }
+    /// ```
+    pub fn unary<F, O>(&self, op: F) -> PrimitiveArray<O>
+    where
+        O: ArrowPrimitiveType,
+        F: Fn(T::Native) -> O::Native,
+    {
+        let data = self.data();
+        let len = self.len();
+        let null_count = self.null_count();
+
+        let null_buffer = data.null_buffer().map(|b| b.bit_slice(data.offset(), len));
+        let values = self.values().iter().map(|v| op(*v));
+        // JUSTIFICATION
+        //  Benefit
+        //      ~60% speedup
+        //  Soundness
+        //      `values` is an iterator with a known size because arrays are sized.
+        let buffer = unsafe { Buffer::from_trusted_len_iter(values) };
+        unsafe { build_primitive_array(len, buffer, null_count, null_buffer) }
+    }
+
+    /// Applies a unary and fallible function to all valid values in a primitive array
+    ///
+    /// This is unlike [`Self::unary`] which will apply an infallible function to all rows
+    /// regardless of validity, in many cases this will be significantly faster and should
+    /// be preferred if `op` is infallible.
+    ///
+    /// Note: LLVM is currently unable to effectively vectorize fallible operations
+    pub fn try_unary<F, O, E>(&self, op: F) -> Result<PrimitiveArray<O>, E>
+    where
+        O: ArrowPrimitiveType,
+        F: Fn(T::Native) -> Result<O::Native, E>,
+    {
+        let data = self.data();
+        let len = self.len();
+        let null_count = self.null_count();
+
+        if null_count == 0 {
+            let values = self.values().iter().map(|v| op(*v));
+            // JUSTIFICATION
+            //  Benefit
+            //      ~60% speedup
+            //  Soundness
+            //      `values` is an iterator with a known size because arrays are sized.
+            let buffer = unsafe { Buffer::try_from_trusted_len_iter(values)? };
+            return Ok(unsafe { build_primitive_array(len, buffer, 0, None) });
+        }
+
+        let null_buffer = data.null_buffer().map(|b| b.bit_slice(data.offset(), len));
+        let mut buffer = BufferBuilder::<O::Native>::new(len);
+        buffer.append_n_zeroed(len);
+        let slice = buffer.as_slice_mut();
+
+        try_for_each_valid_idx(len, 0, null_count, null_buffer.as_deref(), |idx| {
+            unsafe { *slice.get_unchecked_mut(idx) = op(self.value_unchecked(idx))? };
+            Ok::<_, E>(())
+        })?;
+
+        Ok(unsafe {
+            build_primitive_array(len, buffer.finish(), null_count, null_buffer)
+        })
+    }
+}
+
+#[inline]
+unsafe fn build_primitive_array<O: ArrowPrimitiveType>(
+    len: usize,
+    buffer: Buffer,
+    null_count: usize,
+    null_buffer: Option<Buffer>,
+) -> PrimitiveArray<O> {
+    PrimitiveArray::from(ArrayData::new_unchecked(
+        O::DATA_TYPE,
+        len,
+        Some(null_count),
+        null_buffer,
+        0,
+        vec![buffer],
+        vec![],
+    ))
 }
 
 impl<T: ArrowPrimitiveType> From<PrimitiveArray<T>> for ArrayData {
diff --git a/arrow/src/compute/kernels/arity.rs b/arrow/src/compute/kernels/arity.rs
index 5f875e6dd..cb5184c0e 100644
--- a/arrow/src/compute/kernels/arity.rs
+++ b/arrow/src/compute/kernels/arity.rs
@@ -48,92 +48,24 @@ unsafe fn build_primitive_array<O: ArrowPrimitiveType>(
     ))
 }
 
-/// Applies an unary and infallible function to a primitive array.
-/// This is the fastest way to perform an operation on a primitive array when
-/// the benefits of a vectorized operation outweigh the cost of branching nulls and non-nulls.
-///
-/// # Implementation
-///
-/// This will apply the function for all values, including those on null slots.
-/// This implies that the operation must be infallible for any value of the corresponding type
-/// or this function may panic.
-/// # Example
-/// ```rust
-/// # use arrow::array::Int32Array;
-/// # use arrow::datatypes::Int32Type;
-/// # use arrow::compute::kernels::arity::unary;
-/// # fn main() {
-/// let array = Int32Array::from(vec![Some(5), Some(7), None]);
-/// let c = unary::<_, _, Int32Type>(&array, |x| x * 2 + 1);
-/// assert_eq!(c, Int32Array::from(vec![Some(11), Some(15), None]));
-/// # }
-/// ```
+/// See [`PrimitiveArray::unary`]
 pub fn unary<I, F, O>(array: &PrimitiveArray<I>, op: F) -> PrimitiveArray<O>
 where
     I: ArrowPrimitiveType,
     O: ArrowPrimitiveType,
     F: Fn(I::Native) -> O::Native,
 {
-    let data = array.data();
-    let len = data.len();
-    let null_count = data.null_count();
-
-    let null_buffer = data
-        .null_buffer()
-        .map(|b| b.bit_slice(data.offset(), data.len()));
-
-    let values = array.values().iter().map(|v| op(*v));
-    // JUSTIFICATION
-    //  Benefit
-    //      ~60% speedup
-    //  Soundness
-    //      `values` is an iterator with a known size because arrays are sized.
-    let buffer = unsafe { Buffer::from_trusted_len_iter(values) };
-    unsafe { build_primitive_array(len, buffer, null_count, null_buffer) }
+    array.unary(op)
 }
 
-/// Applies a unary and fallible function to all valid values in a primitive array
-///
-/// This is unlike [`unary`] which will apply an infallible function to all rows regardless
-/// of validity, in many cases this will be significantly faster and should be preferred
-/// if `op` is infallible.
-///
-/// Note: LLVM is currently unable to effectively vectorize fallible operations
+/// See [`PrimitiveArray::try_unary`]
 pub fn try_unary<I, F, O>(array: &PrimitiveArray<I>, op: F) -> Result<PrimitiveArray<O>>
 where
     I: ArrowPrimitiveType,
     O: ArrowPrimitiveType,
     F: Fn(I::Native) -> Result<O::Native>,
 {
-    let len = array.len();
-    let null_count = array.null_count();
-
-    if null_count == 0 {
-        let values = array.values().iter().map(|v| op(*v));
-        // JUSTIFICATION
-        //  Benefit
-        //      ~60% speedup
-        //  Soundness
-        //      `values` is an iterator with a known size because arrays are sized.
-        let buffer = unsafe { Buffer::try_from_trusted_len_iter(values)? };
-        return Ok(unsafe { build_primitive_array(len, buffer, 0, None) });
-    }
-
-    let null_buffer = array
-        .data_ref()
-        .null_buffer()
-        .map(|b| b.bit_slice(array.offset(), array.len()));
-
-    let mut buffer = BufferBuilder::<O::Native>::new(len);
-    buffer.append_n_zeroed(array.len());
-    let slice = buffer.as_slice_mut();
-
-    try_for_each_valid_idx(array.len(), 0, null_count, null_buffer.as_deref(), |idx| {
-        unsafe { *slice.get_unchecked_mut(idx) = op(array.value_unchecked(idx))? };
-        Ok::<_, ArrowError>(())
-    })?;
-
-    Ok(unsafe { build_primitive_array(len, buffer.finish(), null_count, null_buffer) })
+    array.try_unary(op)
 }
 
 /// A helper function that applies an infallible unary function to a dictionary array with primitive value type.