You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/07/26 17:27:05 UTC
[arrow-rs] branch master updated: Fix interval to duration casting (#4553) (#4562)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 96886303df Fix interval to duration casting (#4553) (#4562)
96886303df is described below
commit 96886303dfa681fa0ae0910c651d50b2cf15015b
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Wed Jul 26 13:27:00 2023 -0400
Fix interval to duration casting (#4553) (#4562)
* Fix interval to duration casting (#4553)
* Clippy
* Review feedback
---
arrow-cast/src/cast.rs | 231 +++++++++++++++++--------------------------------
1 file changed, 78 insertions(+), 153 deletions(-)
diff --git a/arrow-cast/src/cast.rs b/arrow-cast/src/cast.rs
index 3a5c27fb60..2ee8c51b0a 100644
--- a/arrow-cast/src/cast.rs
+++ b/arrow-cast/src/cast.rs
@@ -447,20 +447,11 @@ fn cast_interval_day_time_to_interval_month_day_nano(
}
/// Cast the array from interval to duration
-fn cast_interval_to_duration<D: ArrowTemporalType<Native = i64>>(
+fn cast_month_day_nano_to_duration<D: ArrowTemporalType<Native = i64>>(
array: &dyn Array,
cast_options: &CastOptions,
) -> Result<ArrayRef, ArrowError> {
- let array = array
- .as_any()
- .downcast_ref::<IntervalMonthDayNanoArray>()
- .ok_or_else(|| {
- ArrowError::ComputeError(
- "Internal Error: Cannot cast interval to IntervalArray of expected type"
- .to_string(),
- )
- })?;
-
+ let array = array.as_primitive::<IntervalMonthDayNanoType>();
let scale = match D::DATA_TYPE {
DataType::Duration(TimeUnit::Second) => 1_000_000_000,
DataType::Duration(TimeUnit::Millisecond) => 1_000_000,
@@ -470,16 +461,9 @@ fn cast_interval_to_duration<D: ArrowTemporalType<Native = i64>>(
};
if cast_options.safe {
- let iter = array.iter().map(|v| {
- v.and_then(|v| {
- let v = v / scale;
- if v > i64::MAX as i128 {
- None
- } else {
- Some(v as i64)
- }
- })
- });
+ let iter = array
+ .iter()
+ .map(|v| v.and_then(|v| (v >> 64 == 0).then_some((v as i64) / scale)));
Ok(Arc::new(unsafe {
PrimitiveArray::<D>::from_trusted_len_iter(iter)
}))
@@ -487,17 +471,9 @@ fn cast_interval_to_duration<D: ArrowTemporalType<Native = i64>>(
let vec = array
.iter()
.map(|v| {
- v.map(|v| {
- let v = v / scale;
- if v > i64::MAX as i128 {
- Err(ArrowError::ComputeError(format!(
- "Cannot cast to {:?}. Overflowing on {:?}",
- D::DATA_TYPE,
- v
- )))
- } else {
- Ok(v as i64)
- }
+ v.map(|v| match v >> 64 {
+ 0 => Ok((v as i64) / scale),
+ _ => Err(ArrowError::ComputeError("Cannot convert interval containing non-zero months or days to duration".to_string()))
})
.transpose()
})
@@ -2174,16 +2150,16 @@ pub fn cast_with_options(
cast_duration_to_interval::<DurationNanosecondType>(array, cast_options)
}
(Interval(IntervalUnit::MonthDayNano), Duration(TimeUnit::Second)) => {
- cast_interval_to_duration::<DurationSecondType>(array, cast_options)
+ cast_month_day_nano_to_duration::<DurationSecondType>(array, cast_options)
}
(Interval(IntervalUnit::MonthDayNano), Duration(TimeUnit::Millisecond)) => {
- cast_interval_to_duration::<DurationMillisecondType>(array, cast_options)
+ cast_month_day_nano_to_duration::<DurationMillisecondType>(array, cast_options)
}
(Interval(IntervalUnit::MonthDayNano), Duration(TimeUnit::Microsecond)) => {
- cast_interval_to_duration::<DurationMicrosecondType>(array, cast_options)
+ cast_month_day_nano_to_duration::<DurationMicrosecondType>(array, cast_options)
}
(Interval(IntervalUnit::MonthDayNano), Duration(TimeUnit::Nanosecond)) => {
- cast_interval_to_duration::<DurationNanosecondType>(array, cast_options)
+ cast_month_day_nano_to_duration::<DurationNanosecondType>(array, cast_options)
}
(Interval(IntervalUnit::YearMonth), Interval(IntervalUnit::MonthDayNano)) => {
cast_interval_year_month_to_interval_month_day_nano(array, cast_options)
@@ -8624,29 +8600,16 @@ mod tests {
}
/// helper function to test casting from duration to interval
- fn cast_from_duration_to_interval<T: ArrowTemporalType>(
+ fn cast_from_duration_to_interval<T: ArrowTemporalType<Native = i64>>(
array: Vec<i64>,
cast_options: &CastOptions,
- ) -> Result<PrimitiveArray<IntervalMonthDayNanoType>, ArrowError>
- where
- arrow_array::PrimitiveArray<T>: From<Vec<i64>>,
- {
- let array = PrimitiveArray::<T>::from(array);
+ ) -> Result<PrimitiveArray<IntervalMonthDayNanoType>, ArrowError> {
+ let array = PrimitiveArray::<T>::new(array.into(), None);
let array = Arc::new(array) as ArrayRef;
- let casted_array = cast_with_options(
- &array,
- &DataType::Interval(IntervalUnit::MonthDayNano),
- cast_options,
- )?;
- casted_array
- .as_any()
- .downcast_ref::<IntervalMonthDayNanoArray>()
- .ok_or_else(|| {
- ArrowError::ComputeError(
- "Failed to downcast to IntervalMonthDayNanoArray".to_string(),
- )
- })
- .cloned()
+ let interval = DataType::Interval(IntervalUnit::MonthDayNano);
+ let out = cast_with_options(&array, &interval, cast_options)?;
+ let out = out.as_primitive::<IntervalMonthDayNanoType>().clone();
+ Ok(out)
}
#[test]
@@ -8768,11 +8731,9 @@ mod tests {
/// helper function to test casting from interval to duration
fn cast_from_interval_to_duration<T: ArrowTemporalType>(
- array: Vec<i128>,
+ array: &IntervalMonthDayNanoArray,
cast_options: &CastOptions,
) -> Result<PrimitiveArray<T>, ArrowError> {
- let array = IntervalMonthDayNanoArray::from(array);
- let array = Arc::new(array) as ArrayRef;
let casted_array = cast_with_options(&array, &T::DATA_TYPE, cast_options)?;
casted_array
.as_any()
@@ -8788,125 +8749,89 @@ mod tests {
#[test]
fn test_cast_from_interval_to_duration() {
+ let nullable = CastOptions::default();
+ let fallible = CastOptions {
+ safe: false,
+ format_options: FormatOptions::default(),
+ };
+
// from interval month day nano to duration second
- let array = vec![1234567];
- let casted_array = cast_from_interval_to_duration::<DurationSecondType>(
- array,
- &CastOptions::default(),
- )
- .unwrap();
- assert_eq!(
- casted_array.data_type(),
- &DataType::Duration(TimeUnit::Second)
- );
+ let array = vec![1234567].into();
+ let casted_array: DurationSecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert_eq!(casted_array.value(0), 0);
- let array = vec![i128::MAX];
- let casted_array = cast_from_interval_to_duration::<DurationSecondType>(
- array.clone(),
- &CastOptions::default(),
- )
- .unwrap();
+ let array = vec![i128::MAX].into();
+ let casted_array: DurationSecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert!(!casted_array.is_valid(0));
- let casted_array = cast_from_interval_to_duration::<DurationSecondType>(
- array,
- &CastOptions {
- safe: false,
- format_options: FormatOptions::default(),
- },
- );
- assert!(casted_array.is_err());
+ let res = cast_from_interval_to_duration::<DurationSecondType>(&array, &fallible);
+ assert!(res.is_err());
// from interval month day nano to duration millisecond
- let array = vec![1234567];
- let casted_array = cast_from_interval_to_duration::<DurationMillisecondType>(
- array,
- &CastOptions::default(),
- )
- .unwrap();
+ let array = vec![1234567].into();
+ let casted_array: DurationMillisecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert_eq!(casted_array.value(0), 1);
- let array = vec![i128::MAX];
- let casted_array = cast_from_interval_to_duration::<DurationMillisecondType>(
- array.clone(),
- &CastOptions::default(),
- )
- .unwrap();
+ let array = vec![i128::MAX].into();
+ let casted_array: DurationMillisecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert!(!casted_array.is_valid(0));
- let casted_array = cast_from_interval_to_duration::<DurationMillisecondType>(
- array,
- &CastOptions {
- safe: false,
- format_options: FormatOptions::default(),
- },
- );
- assert!(casted_array.is_err());
+ let res =
+ cast_from_interval_to_duration::<DurationMillisecondType>(&array, &fallible);
+ assert!(res.is_err());
// from interval month day nano to duration microsecond
- let array = vec![1234567];
- let casted_array = cast_from_interval_to_duration::<DurationMicrosecondType>(
- array,
- &CastOptions::default(),
- )
- .unwrap();
- assert_eq!(
- casted_array.data_type(),
- &DataType::Duration(TimeUnit::Microsecond)
- );
+ let array = vec![1234567].into();
+ let casted_array: DurationMicrosecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert_eq!(casted_array.value(0), 1234);
- let array = vec![i128::MAX];
- let casted_array = cast_from_interval_to_duration::<DurationMicrosecondType>(
- array.clone(),
- &CastOptions::default(),
- )
- .unwrap();
+ let array = vec![i128::MAX].into();
+ let casted_array =
+ cast_from_interval_to_duration::<DurationMicrosecondType>(&array, &nullable)
+ .unwrap();
assert!(!casted_array.is_valid(0));
- let casted_array = cast_from_interval_to_duration::<DurationMicrosecondType>(
- array,
- &CastOptions {
- safe: false,
- format_options: FormatOptions::default(),
- },
- );
+ let casted_array =
+ cast_from_interval_to_duration::<DurationMicrosecondType>(&array, &fallible);
assert!(casted_array.is_err());
// from interval month day nano to duration nanosecond
- let array = vec![1234567];
- let casted_array = cast_from_interval_to_duration::<DurationNanosecondType>(
- array,
- &CastOptions::default(),
- )
- .unwrap();
- assert_eq!(
- casted_array.data_type(),
- &DataType::Duration(TimeUnit::Nanosecond)
- );
+ let array = vec![1234567].into();
+ let casted_array: DurationNanosecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert_eq!(casted_array.value(0), 1234567);
- let array = vec![i128::MAX];
- let casted_array = cast_from_interval_to_duration::<DurationNanosecondType>(
- array.clone(),
- &CastOptions::default(),
- )
- .unwrap();
- assert_eq!(
- casted_array.data_type(),
- &DataType::Duration(TimeUnit::Nanosecond)
- );
+ let array = vec![i128::MAX].into();
+ let casted_array: DurationNanosecondArray =
+ cast_from_interval_to_duration(&array, &nullable).unwrap();
assert!(!casted_array.is_valid(0));
- let casted_array = cast_from_interval_to_duration::<DurationNanosecondType>(
- array,
- &CastOptions {
- safe: false,
- format_options: FormatOptions::default(),
- },
- );
+ let casted_array =
+ cast_from_interval_to_duration::<DurationNanosecondType>(&array, &fallible);
assert!(casted_array.is_err());
+
+ let array = vec![
+ IntervalMonthDayNanoType::make_value(0, 1, 0),
+ IntervalMonthDayNanoType::make_value(-1, 0, 0),
+ IntervalMonthDayNanoType::make_value(1, 1, 0),
+ IntervalMonthDayNanoType::make_value(1, 0, 1),
+ IntervalMonthDayNanoType::make_value(0, 0, -1),
+ ]
+ .into();
+ let casted_array =
+ cast_from_interval_to_duration::<DurationNanosecondType>(&array, &nullable)
+ .unwrap();
+ assert!(!casted_array.is_valid(0));
+ assert!(!casted_array.is_valid(1));
+ assert!(!casted_array.is_valid(2));
+ assert!(!casted_array.is_valid(3));
+ assert!(casted_array.is_valid(4));
+ assert_eq!(casted_array.value(4), -1);
}
/// helper function to test casting from interval year month to interval month day nano