You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by vi...@apache.org on 2022/08/17 16:55:26 UTC
[arrow-rs] branch master updated: Revise FromIterator for Decimal128Array to use Into instead of Borrow (#2442)
This is an automated email from the ASF dual-hosted git repository.
viirya pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 3557428ef Revise FromIterator for Decimal128Array to use Into instead of Borrow (#2442)
3557428ef is described below
commit 3557428efb17c0992caf2a467b713f3d10519ab1
Author: Liang-Chi Hsieh <vi...@gmail.com>
AuthorDate: Wed Aug 17 09:55:21 2022 -0700
Revise FromIterator for Decimal128Array to use Into instead of Borrow (#2442)
* Change FromIterator for Decimal128Array
* fix clippy
* Fix doc test
---
arrow/src/array/array_decimal.rs | 37 ++++++++---
arrow/src/array/equal/mod.rs | 28 +++++----
arrow/src/array/ord.rs | 4 +-
arrow/src/array/transform/mod.rs | 16 ++---
arrow/src/compute/kernels/cast.rs | 21 ++++---
arrow/src/compute/kernels/sort.rs | 10 +--
parquet/src/arrow/array_reader/primitive_array.rs | 76 +++++++++++++++--------
7 files changed, 120 insertions(+), 72 deletions(-)
diff --git a/arrow/src/array/array_decimal.rs b/arrow/src/array/array_decimal.rs
index 98fc9d0c7..4af3d1dba 100644
--- a/arrow/src/array/array_decimal.rs
+++ b/arrow/src/array/array_decimal.rs
@@ -16,8 +16,6 @@
// under the License.
use crate::array::ArrayAccessor;
-
-use std::borrow::Borrow;
use std::convert::From;
use std::fmt;
use std::{any::Any, iter::FromIterator};
@@ -45,9 +43,9 @@ use crate::util::decimal::{BasicDecimal, Decimal256};
///
/// // Create a DecimalArray with the default precision and scale
/// let decimal_array: Decimal128Array = vec![
-/// Some(8_887_000_000),
+/// Some(8_887_000_000_i128),
/// None,
-/// Some(-8_887_000_000),
+/// Some(-8_887_000_000_i128),
/// ]
/// .into_iter().collect();
///
@@ -452,8 +450,8 @@ impl<Ptr: Into<Decimal256>> FromIterator<Option<Ptr>> for Decimal256Array {
}
}
-impl<Ptr: Borrow<Option<i128>>> FromIterator<Ptr> for Decimal128Array {
- fn from_iter<I: IntoIterator<Item = Ptr>>(iter: I) -> Self {
+impl<Ptr: Into<i128>> FromIterator<Option<Ptr>> for Decimal128Array {
+ fn from_iter<I: IntoIterator<Item = Option<Ptr>>>(iter: I) -> Self {
let iter = iter.into_iter();
let (lower, upper) = iter.size_hint();
let size_hint = upper.unwrap_or(lower);
@@ -462,9 +460,9 @@ impl<Ptr: Borrow<Option<i128>>> FromIterator<Ptr> for Decimal128Array {
let buffer: Buffer = iter
.map(|item| {
- if let Some(a) = item.borrow() {
+ if let Some(a) = item {
null_buf.append(true);
- *a
+ a.into()
} else {
null_buf.append(false);
// arbitrary value for NULL
@@ -547,6 +545,7 @@ impl<'a, const BYTE_WIDTH: usize> BasicDecimalArray<BYTE_WIDTH> {
mod tests {
use crate::array::Decimal256Builder;
use crate::datatypes::{DECIMAL256_MAX_PRECISION, DECIMAL_DEFAULT_SCALE};
+ use crate::util::decimal::Decimal128;
use crate::{array::Decimal128Builder, datatypes::Field};
use num::{BigInt, Num};
@@ -749,8 +748,8 @@ mod tests {
#[test]
fn test_decimal_array_fmt_debug() {
- let arr = [Some(8887000000), Some(-8887000000), None]
- .iter()
+ let arr = [Some(8887000000_i128), Some(-8887000000_i128), None]
+ .into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(23, 6)
.unwrap();
@@ -937,4 +936,22 @@ mod tests {
);
assert!(!array.is_null(2));
}
+
+ #[test]
+ fn test_from_iter_decimal128array() {
+ let array: Decimal128Array = vec![
+ Some(Decimal128::new_from_i128(38, 10, -100)),
+ None,
+ Some(Decimal128::new_from_i128(38, 10, 101)),
+ ]
+ .into_iter()
+ .collect();
+ assert_eq!(array.len(), 3);
+ assert_eq!(array.data_type(), &DataType::Decimal128(38, 10));
+ assert_eq!(-100_i128, array.value(0).into());
+ assert!(!array.is_null(0));
+ assert!(array.is_null(1));
+ assert_eq!(101_i128, array.value(2).into());
+ assert!(!array.is_null(2));
+ }
}
diff --git a/arrow/src/array/equal/mod.rs b/arrow/src/array/equal/mod.rs
index 6fdc06f83..975fbd2b3 100644
--- a/arrow/src/array/equal/mod.rs
+++ b/arrow/src/array/equal/mod.rs
@@ -841,8 +841,8 @@ mod tests {
test_equal(&a_slice, &b_slice, false);
}
- fn create_decimal_array(data: &[Option<i128>]) -> ArrayData {
- data.iter()
+ fn create_decimal_array(data: Vec<Option<i128>>) -> ArrayData {
+ data.into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(23, 6)
.unwrap()
@@ -851,32 +851,36 @@ mod tests {
#[test]
fn test_decimal_equal() {
- let a = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000)]);
- let b = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000)]);
+ let a = create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000)]);
+ let b = create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000)]);
test_equal(&a, &b, true);
- let b = create_decimal_array(&[Some(15_887_000_000), Some(-8_887_000_000)]);
+ let b = create_decimal_array(vec![Some(15_887_000_000), Some(-8_887_000_000)]);
test_equal(&a, &b, false);
}
// Test the case where null_count > 0
#[test]
fn test_decimal_null() {
- let a = create_decimal_array(&[Some(8_887_000_000), None, Some(-8_887_000_000)]);
- let b = create_decimal_array(&[Some(8_887_000_000), None, Some(-8_887_000_000)]);
+ let a =
+ create_decimal_array(vec![Some(8_887_000_000), None, Some(-8_887_000_000)]);
+ let b =
+ create_decimal_array(vec![Some(8_887_000_000), None, Some(-8_887_000_000)]);
test_equal(&a, &b, true);
- let b = create_decimal_array(&[Some(8_887_000_000), Some(-8_887_000_000), None]);
+ let b =
+ create_decimal_array(vec![Some(8_887_000_000), Some(-8_887_000_000), None]);
test_equal(&a, &b, false);
- let b = create_decimal_array(&[Some(15_887_000_000), None, Some(-8_887_000_000)]);
+ let b =
+ create_decimal_array(vec![Some(15_887_000_000), None, Some(-8_887_000_000)]);
test_equal(&a, &b, false);
}
#[test]
fn test_decimal_offsets() {
// Test the case where offset != 0
- let a = create_decimal_array(&[
+ let a = create_decimal_array(vec![
Some(8_887_000_000),
None,
None,
@@ -884,7 +888,7 @@ mod tests {
None,
None,
]);
- let b = create_decimal_array(&[
+ let b = create_decimal_array(vec![
None,
Some(8_887_000_000),
None,
@@ -914,7 +918,7 @@ mod tests {
let b_slice = b.slice(2, 3);
test_equal(&a_slice, &b_slice, false);
- let b = create_decimal_array(&[
+ let b = create_decimal_array(vec![
None,
None,
None,
diff --git a/arrow/src/array/ord.rs b/arrow/src/array/ord.rs
index 47173aa7d..dd6539589 100644
--- a/arrow/src/array/ord.rs
+++ b/arrow/src/array/ord.rs
@@ -300,8 +300,8 @@ pub mod tests {
#[test]
fn test_decimal() -> Result<()> {
- let array = vec![Some(5), Some(2), Some(3)]
- .iter()
+ let array = vec![Some(5_i128), Some(2_i128), Some(3_i128)]
+ .into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(23, 6)
.unwrap();
diff --git a/arrow/src/array/transform/mod.rs b/arrow/src/array/transform/mod.rs
index f0fccef14..dafbc05c6 100644
--- a/arrow/src/array/transform/mod.rs
+++ b/arrow/src/array/transform/mod.rs
@@ -687,12 +687,12 @@ mod tests {
};
fn create_decimal_array(
- array: &[Option<i128>],
+ array: Vec<Option<i128>>,
precision: usize,
scale: usize,
) -> Decimal128Array {
array
- .iter()
+ .into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(precision, scale)
.unwrap()
@@ -702,28 +702,28 @@ mod tests {
#[cfg(not(feature = "force_validate"))]
fn test_decimal() {
let decimal_array =
- create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);
+ create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3);
let arrays = vec![Array::data(&decimal_array)];
let mut a = MutableArrayData::new(arrays, true, 3);
a.extend(0, 0, 3);
a.extend(0, 2, 3);
let result = a.freeze();
let array = Decimal128Array::from(result);
- let expected = create_decimal_array(&[Some(1), Some(2), None, None], 10, 3);
+ let expected = create_decimal_array(vec![Some(1), Some(2), None, None], 10, 3);
assert_eq!(array, expected);
}
#[test]
#[cfg(not(feature = "force_validate"))]
fn test_decimal_offset() {
let decimal_array =
- create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);
+ create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3);
let decimal_array = decimal_array.slice(1, 3); // 2, null, 3
let arrays = vec![decimal_array.data()];
let mut a = MutableArrayData::new(arrays, true, 2);
a.extend(0, 0, 2); // 2, null
let result = a.freeze();
let array = Decimal128Array::from(result);
- let expected = create_decimal_array(&[Some(2), None], 10, 3);
+ let expected = create_decimal_array(vec![Some(2), None], 10, 3);
assert_eq!(array, expected);
}
@@ -731,7 +731,7 @@ mod tests {
#[cfg(not(feature = "force_validate"))]
fn test_decimal_null_offset_nulls() {
let decimal_array =
- create_decimal_array(&[Some(1), Some(2), None, Some(3)], 10, 3);
+ create_decimal_array(vec![Some(1), Some(2), None, Some(3)], 10, 3);
let decimal_array = decimal_array.slice(1, 3); // 2, null, 3
let arrays = vec![decimal_array.data()];
let mut a = MutableArrayData::new(arrays, true, 2);
@@ -741,7 +741,7 @@ mod tests {
let result = a.freeze();
let array = Decimal128Array::from(result);
let expected = create_decimal_array(
- &[Some(2), None, None, None, None, None, Some(3)],
+ vec![Some(2), None, None, None, None, None, Some(3)],
10,
3,
);
diff --git a/arrow/src/compute/kernels/cast.rs b/arrow/src/compute/kernels/cast.rs
index ebbf4a65b..4f59c00fc 100644
--- a/arrow/src/compute/kernels/cast.rs
+++ b/arrow/src/compute/kernels/cast.rs
@@ -2591,12 +2591,12 @@ mod tests {
}
fn create_decimal_array(
- array: &[Option<i128>],
+ array: Vec<Option<i128>>,
precision: usize,
scale: usize,
) -> Result<Decimal128Array> {
array
- .iter()
+ .into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(precision, scale)
}
@@ -2618,7 +2618,7 @@ mod tests {
let output_type = DataType::Decimal128(20, 4);
assert!(can_cast_types(&input_type, &output_type));
let array = vec![Some(1123456), Some(2123456), Some(3123456), None];
- let input_decimal_array = create_decimal_array(&array, 20, 3).unwrap();
+ let input_decimal_array = create_decimal_array(array, 20, 3).unwrap();
let array = Arc::new(input_decimal_array) as ArrayRef;
generate_cast_test_case!(
&array,
@@ -2633,7 +2633,7 @@ mod tests {
);
// negative test
let array = vec![Some(123456), None];
- let input_decimal_array = create_decimal_array(&array, 10, 0).unwrap();
+ let input_decimal_array = create_decimal_array(array, 10, 0).unwrap();
let array = Arc::new(input_decimal_array) as ArrayRef;
let result = cast(&array, &DataType::Decimal128(2, 2));
assert!(result.is_err());
@@ -2647,7 +2647,7 @@ mod tests {
let output_type = DataType::Decimal256(20, 4);
assert!(can_cast_types(&input_type, &output_type));
let array = vec![Some(1123456), Some(2123456), Some(3123456), None];
- let input_decimal_array = create_decimal_array(&array, 20, 3).unwrap();
+ let input_decimal_array = create_decimal_array(array, 20, 3).unwrap();
let array = Arc::new(input_decimal_array) as ArrayRef;
generate_cast_test_case!(
&array,
@@ -2739,7 +2739,7 @@ mod tests {
assert!(!can_cast_types(&decimal_type, &DataType::UInt8));
let value_array: Vec<Option<i128>> =
vec![Some(125), Some(225), Some(325), None, Some(525)];
- let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap();
+ let decimal_array = create_decimal_array(value_array, 38, 2).unwrap();
let array = Arc::new(decimal_array) as ArrayRef;
// i8
generate_cast_test_case!(
@@ -2786,7 +2786,7 @@ mod tests {
// overflow test: out of range of max i8
let value_array: Vec<Option<i128>> = vec![Some(24400)];
- let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap();
+ let decimal_array = create_decimal_array(value_array, 38, 2).unwrap();
let array = Arc::new(decimal_array) as ArrayRef;
let casted_array = cast(&array, &DataType::Int8);
assert_eq!(
@@ -2806,7 +2806,7 @@ mod tests {
Some(112345678),
Some(112345679),
];
- let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap();
+ let decimal_array = create_decimal_array(value_array, 38, 2).unwrap();
let array = Arc::new(decimal_array) as ArrayRef;
generate_cast_test_case!(
&array,
@@ -2834,7 +2834,7 @@ mod tests {
Some(112345678901234568),
Some(112345678901234560),
];
- let decimal_array = create_decimal_array(&value_array, 38, 2).unwrap();
+ let decimal_array = create_decimal_array(value_array, 38, 2).unwrap();
let array = Arc::new(decimal_array) as ArrayRef;
generate_cast_test_case!(
&array,
@@ -5280,7 +5280,8 @@ mod tests {
Arc::new(DurationMicrosecondArray::from(vec![1000, 2000])),
Arc::new(DurationNanosecondArray::from(vec![1000, 2000])),
Arc::new(
- create_decimal_array(&[Some(1), Some(2), Some(3), None], 38, 0).unwrap(),
+ create_decimal_array(vec![Some(1), Some(2), Some(3), None], 38, 0)
+ .unwrap(),
),
]
}
diff --git a/arrow/src/compute/kernels/sort.rs b/arrow/src/compute/kernels/sort.rs
index dca09a66a..0e2273e92 100644
--- a/arrow/src/compute/kernels/sort.rs
+++ b/arrow/src/compute/kernels/sort.rs
@@ -1062,8 +1062,8 @@ mod tests {
use std::convert::TryFrom;
use std::sync::Arc;
- fn create_decimal_array(data: &[Option<i128>]) -> Decimal128Array {
- data.iter()
+ fn create_decimal_array(data: Vec<Option<i128>>) -> Decimal128Array {
+ data.into_iter()
.collect::<Decimal128Array>()
.with_precision_and_scale(23, 6)
.unwrap()
@@ -1075,7 +1075,7 @@ mod tests {
limit: Option<usize>,
expected_data: Vec<u32>,
) {
- let output = create_decimal_array(&data);
+ let output = create_decimal_array(data);
let expected = UInt32Array::from(expected_data);
let output =
sort_to_indices(&(Arc::new(output) as ArrayRef), options, limit).unwrap();
@@ -1088,8 +1088,8 @@ mod tests {
limit: Option<usize>,
expected_data: Vec<Option<i128>>,
) {
- let output = create_decimal_array(&data);
- let expected = Arc::new(create_decimal_array(&expected_data)) as ArrayRef;
+ let output = create_decimal_array(data);
+ let expected = Arc::new(create_decimal_array(expected_data)) as ArrayRef;
let output = match limit {
Some(_) => {
sort_limit(&(Arc::new(output) as ArrayRef), options, limit).unwrap()
diff --git a/parquet/src/arrow/array_reader/primitive_array.rs b/parquet/src/arrow/array_reader/primitive_array.rs
index abe99d4d2..d3f71dbab 100644
--- a/parquet/src/arrow/array_reader/primitive_array.rs
+++ b/parquet/src/arrow/array_reader/primitive_array.rs
@@ -24,7 +24,10 @@ use crate::column::page::PageIterator;
use crate::data_type::DataType;
use crate::errors::{ParquetError, Result};
use crate::schema::types::ColumnDescPtr;
-use arrow::array::{Array, ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, Decimal128Array, Float32Array, Float64Array, Int32Array, Int64Array};
+use arrow::array::{
+ ArrayDataBuilder, ArrayRef, BooleanArray, BooleanBufferBuilder, Decimal128Array,
+ Float32Array, Float64Array, Int32Array, Int64Array,
+};
use arrow::buffer::Buffer;
use arrow::datatypes::DataType as ArrowType;
use std::any::Any;
@@ -33,9 +36,9 @@ use std::sync::Arc;
/// Primitive array readers are leaves of array reader tree. They accept page iterator
/// and read them into primitive arrays.
pub struct PrimitiveArrayReader<T>
- where
- T: DataType,
- T::T: ScalarValue,
+where
+ T: DataType,
+ T::T: ScalarValue,
{
data_type: ArrowType,
pages: Box<dyn PageIterator>,
@@ -45,9 +48,9 @@ pub struct PrimitiveArrayReader<T>
}
impl<T> PrimitiveArrayReader<T>
- where
- T: DataType,
- T::T: ScalarValue,
+where
+ T: DataType,
+ T::T: ScalarValue,
{
/// Construct primitive array reader.
pub fn new(
@@ -77,9 +80,9 @@ impl<T> PrimitiveArrayReader<T>
/// Implementation of primitive array reader.
impl<T> ArrayReader for PrimitiveArrayReader<T>
- where
- T: DataType,
- T::T: ScalarValue,
+where
+ T: DataType,
+ T::T: ScalarValue,
{
fn as_any(&self) -> &dyn Any {
self
@@ -187,7 +190,7 @@ impl<T> ArrayReader for PrimitiveArrayReader<T>
.downcast_ref::<Int32Array>()
.unwrap()
.iter()
- .map(|v| v.map(|v| v.into()))
+ .map(|v| v.map(|v| v as i128))
.collect::<Decimal128Array>(),
ArrowType::Int64 => array
@@ -195,7 +198,7 @@ impl<T> ArrayReader for PrimitiveArrayReader<T>
.downcast_ref::<Int64Array>()
.unwrap()
.iter()
- .map(|v| v.map(|v| v.into()))
+ .map(|v| v.map(|v| v as i128))
.collect::<Decimal128Array>(),
_ => {
return Err(arrow_err!(
@@ -204,7 +207,7 @@ impl<T> ArrayReader for PrimitiveArrayReader<T>
));
}
}
- .with_precision_and_scale(p, s)?;
+ .with_precision_and_scale(p, s)?;
Arc::new(array) as ArrayRef
}
@@ -243,11 +246,11 @@ mod tests {
use crate::util::test_common::rand_gen::make_pages;
use crate::util::InMemoryPageIterator;
use arrow::array::{Array, PrimitiveArray};
- use arrow::datatypes::{ArrowPrimitiveType};
+ use arrow::datatypes::ArrowPrimitiveType;
+ use arrow::datatypes::DataType::Decimal128;
use rand::distributions::uniform::SampleUniform;
use std::collections::VecDeque;
- use arrow::datatypes::DataType::Decimal128;
#[allow(clippy::too_many_arguments)]
fn make_column_chunks<T: DataType>(
@@ -313,7 +316,7 @@ mod tests {
column_desc,
None,
)
- .unwrap();
+ .unwrap();
// expect no values to be read
let array = array_reader.next_batch(50).unwrap();
@@ -360,7 +363,7 @@ mod tests {
column_desc,
None,
)
- .unwrap();
+ .unwrap();
// Read first 50 values, which are all from the first column chunk
let array = array_reader.next_batch(50).unwrap();
@@ -560,7 +563,7 @@ mod tests {
column_desc,
None,
)
- .unwrap();
+ .unwrap();
let mut accu_len: usize = 0;
@@ -602,7 +605,6 @@ mod tests {
}
}
-
#[test]
fn test_primitive_array_reader_decimal_types() {
// parquet `INT32` to decimal
@@ -641,18 +643,30 @@ mod tests {
column_desc,
None,
)
- .unwrap();
+ .unwrap();
// read data from the reader
// the data type is decimal(8,2)
let array = array_reader.next_batch(50).unwrap();
assert_eq!(array.data_type(), &Decimal128(8, 2));
let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
- let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::<Decimal128Array>().with_precision_and_scale(8, 2).unwrap();
+ let data_decimal_array = data[0..50]
+ .iter()
+ .copied()
+ .map(|v| Some(v as i128))
+ .collect::<Decimal128Array>()
+ .with_precision_and_scale(8, 2)
+ .unwrap();
assert_eq!(array, &data_decimal_array);
// not equal with different data type(precision and scale)
- let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::<Decimal128Array>().with_precision_and_scale(9, 0).unwrap();
+ let data_decimal_array = data[0..50]
+ .iter()
+ .copied()
+ .map(|v| Some(v as i128))
+ .collect::<Decimal128Array>()
+ .with_precision_and_scale(9, 0)
+ .unwrap();
assert_ne!(array, &data_decimal_array)
}
@@ -692,18 +706,30 @@ mod tests {
column_desc,
None,
)
- .unwrap();
+ .unwrap();
// read data from the reader
// the data type is decimal(18,4)
let array = array_reader.next_batch(50).unwrap();
assert_eq!(array.data_type(), &Decimal128(18, 4));
let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
- let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::<Decimal128Array>().with_precision_and_scale(18, 4).unwrap();
+ let data_decimal_array = data[0..50]
+ .iter()
+ .copied()
+ .map(|v| Some(v as i128))
+ .collect::<Decimal128Array>()
+ .with_precision_and_scale(18, 4)
+ .unwrap();
assert_eq!(array, &data_decimal_array);
// not equal with different data type(precision and scale)
- let data_decimal_array = data[0..50].iter().copied().map(|v| Some(v as i128)).collect::<Decimal128Array>().with_precision_and_scale(34, 0).unwrap();
+ let data_decimal_array = data[0..50]
+ .iter()
+ .copied()
+ .map(|v| Some(v as i128))
+ .collect::<Decimal128Array>()
+ .with_precision_and_scale(34, 0)
+ .unwrap();
assert_ne!(array, &data_decimal_array)
}
}