You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ne...@apache.org on 2020/02/21 06:25:28 UTC
[arrow] branch master updated: ARROW-7887: [Rust] Add
date/time/duration/timestamp types to filter kernel
This is an automated email from the ASF dual-hosted git repository.
nevime pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 7fb03c0 ARROW-7887: [Rust] Add date/time/duration/timestamp types to filter kernel
7fb03c0 is described below
commit 7fb03c0be067933040ea73aeecc803c81a9274ef
Author: Max Burke <ma...@urbanlogiq.com>
AuthorDate: Fri Feb 21 08:24:56 2020 +0200
ARROW-7887: [Rust] Add date/time/duration/timestamp types to filter kernel
The filter kernel was missing implementations for the temporal types (ie: date / time / duration / timestamp).
Closes #6455 from maxburke/filter_temporal_types and squashes the following commits:
688edcb76 <Max Burke> Remove implementations of TimestampType::from as requested by @nevi-me
556df2d2b <Max Burke> Cargo fmt
ac6b886b9 <Max Burke> Add tests for temporal type filtering, as well as missing array-from-timestamp-vector creation functionality.
2fa0623da <Max Burke> Reformat
b51dd25d4 <Max Burke> Add date/time/duration/timestamp types to filter kernel
Authored-by: Max Burke <ma...@urbanlogiq.com>
Signed-off-by: Neville Dipale <ne...@gmail.com>
---
rust/arrow/src/array/array.rs | 31 ++++++++
rust/arrow/src/compute/kernels/filter.rs | 126 ++++++++++++++++++++++++++++++-
2 files changed, 156 insertions(+), 1 deletion(-)
diff --git a/rust/arrow/src/array/array.rs b/rust/arrow/src/array/array.rs
index b4de40f..3af9e40 100644
--- a/rust/arrow/src/array/array.rs
+++ b/rust/arrow/src/array/array.rs
@@ -1813,6 +1813,37 @@ mod tests {
}
#[test]
+ fn test_timestamp_array_from_vec() {
+ let arr = TimestampSecondArray::from_vec(vec![1, -5], None);
+ assert_eq!(2, arr.len());
+ assert_eq!(0, arr.offset());
+ assert_eq!(0, arr.null_count());
+ assert_eq!(1, arr.value(0));
+ assert_eq!(-5, arr.value(1));
+
+ let arr = TimestampMillisecondArray::from_vec(vec![1, -5], None);
+ assert_eq!(2, arr.len());
+ assert_eq!(0, arr.offset());
+ assert_eq!(0, arr.null_count());
+ assert_eq!(1, arr.value(0));
+ assert_eq!(-5, arr.value(1));
+
+ let arr = TimestampMicrosecondArray::from_vec(vec![1, -5], None);
+ assert_eq!(2, arr.len());
+ assert_eq!(0, arr.offset());
+ assert_eq!(0, arr.null_count());
+ assert_eq!(1, arr.value(0));
+ assert_eq!(-5, arr.value(1));
+
+ let arr = TimestampNanosecondArray::from_vec(vec![1, -5], None);
+ assert_eq!(2, arr.len());
+ assert_eq!(0, arr.offset());
+ assert_eq!(0, arr.null_count());
+ assert_eq!(1, arr.value(0));
+ assert_eq!(-5, arr.value(1));
+ }
+
+ #[test]
fn test_primitive_array_slice() {
let arr = Int32Array::from(vec![
Some(0),
diff --git a/rust/arrow/src/compute/kernels/filter.rs b/rust/arrow/src/compute/kernels/filter.rs
index 959f7e1..52e12cf 100644
--- a/rust/arrow/src/compute/kernels/filter.rs
+++ b/rust/arrow/src/compute/kernels/filter.rs
@@ -20,7 +20,7 @@
use std::sync::Arc;
use crate::array::*;
-use crate::datatypes::{ArrowNumericType, DataType};
+use crate::datatypes::{ArrowNumericType, DataType, TimeUnit};
use crate::error::{ArrowError, Result};
/// Helper function to perform boolean lambda function on values from two arrays.
@@ -87,6 +87,44 @@ pub fn filter(array: &Array, filter: &BooleanArray) -> Result<ArrayRef> {
DataType::Float32 => filter_array!(array, filter, Float32Array),
DataType::Float64 => filter_array!(array, filter, Float64Array),
DataType::Boolean => filter_array!(array, filter, BooleanArray),
+ DataType::Date32(_) => filter_array!(array, filter, Date32Array),
+ DataType::Date64(_) => filter_array!(array, filter, Date64Array),
+ DataType::Time32(TimeUnit::Second) => {
+ filter_array!(array, filter, Time32SecondArray)
+ }
+ DataType::Time32(TimeUnit::Millisecond) => {
+ filter_array!(array, filter, Time32MillisecondArray)
+ }
+ DataType::Time64(TimeUnit::Microsecond) => {
+ filter_array!(array, filter, Time64MicrosecondArray)
+ }
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ filter_array!(array, filter, Time64NanosecondArray)
+ }
+ DataType::Duration(TimeUnit::Second) => {
+ filter_array!(array, filter, DurationSecondArray)
+ }
+ DataType::Duration(TimeUnit::Millisecond) => {
+ filter_array!(array, filter, DurationMillisecondArray)
+ }
+ DataType::Duration(TimeUnit::Microsecond) => {
+ filter_array!(array, filter, DurationMicrosecondArray)
+ }
+ DataType::Duration(TimeUnit::Nanosecond) => {
+ filter_array!(array, filter, DurationNanosecondArray)
+ }
+ DataType::Timestamp(TimeUnit::Second, _) => {
+ filter_array!(array, filter, TimestampSecondArray)
+ }
+ DataType::Timestamp(TimeUnit::Millisecond, _) => {
+ filter_array!(array, filter, TimestampMillisecondArray)
+ }
+ DataType::Timestamp(TimeUnit::Microsecond, _) => {
+ filter_array!(array, filter, TimestampMicrosecondArray)
+ }
+ DataType::Timestamp(TimeUnit::Nanosecond, _) => {
+ filter_array!(array, filter, TimestampNanosecondArray)
+ }
DataType::Binary => {
let b = array.as_any().downcast_ref::<BinaryArray>().unwrap();
let mut values: Vec<&[u8]> = Vec::with_capacity(b.len());
@@ -118,6 +156,92 @@ pub fn filter(array: &Array, filter: &BooleanArray) -> Result<ArrayRef> {
mod tests {
use super::*;
+ macro_rules! def_temporal_test {
+ ($test:ident, $array_type: ident, $data: expr) => {
+ #[test]
+ fn $test() {
+ let a = $data;
+ let b = BooleanArray::from(vec![true, false, true, false]);
+ let c = filter(&a, &b).unwrap();
+ let d = c.as_ref().as_any().downcast_ref::<$array_type>().unwrap();
+ assert_eq!(2, d.len());
+ assert_eq!(1, d.value(0));
+ assert_eq!(3, d.value(1));
+ }
+ };
+ }
+
+ def_temporal_test!(
+ test_filter_date32,
+ Date32Array,
+ Date32Array::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_date64,
+ Date64Array,
+ Date64Array::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_time32_second,
+ Time32SecondArray,
+ Time32SecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_time32_millisecond,
+ Time32MillisecondArray,
+ Time32MillisecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_time64_microsecond,
+ Time64MicrosecondArray,
+ Time64MicrosecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_time64_nanosecond,
+ Time64NanosecondArray,
+ Time64NanosecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_duration_second,
+ DurationSecondArray,
+ DurationSecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_duration_millisecond,
+ DurationMillisecondArray,
+ DurationMillisecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_duration_microsecond,
+ DurationMicrosecondArray,
+ DurationMicrosecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_duration_nanosecond,
+ DurationNanosecondArray,
+ DurationNanosecondArray::from(vec![1, 2, 3, 4])
+ );
+ def_temporal_test!(
+ test_filter_timestamp_second,
+ TimestampSecondArray,
+ TimestampSecondArray::from_vec(vec![1, 2, 3, 4], None)
+ );
+ def_temporal_test!(
+ test_filter_timestamp_millisecond,
+ TimestampMillisecondArray,
+ TimestampMillisecondArray::from_vec(vec![1, 2, 3, 4], None)
+ );
+ def_temporal_test!(
+ test_filter_timestamp_microsecond,
+ TimestampMicrosecondArray,
+ TimestampMicrosecondArray::from_vec(vec![1, 2, 3, 4], None)
+ );
+ def_temporal_test!(
+ test_filter_timestamp_nanosecond,
+ TimestampNanosecondArray,
+ TimestampNanosecondArray::from_vec(vec![1, 2, 3, 4], None)
+ );
+
#[test]
fn test_filter_array() {
let a = Int32Array::from(vec![5, 6, 7, 8, 9]);