You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2024/03/17 10:31:23 UTC
(arrow-datafusion) branch main updated: Optimize make_date (#9089) (#9600)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 9f364a44bd Optimize make_date (#9089) (#9600)
9f364a44bd is described below
commit 9f364a44bd413c0343924a9576262a71947bbffb
Author: Vojtěch Toman <vo...@gmail.com>
AuthorDate: Sun Mar 17 11:31:17 2024 +0100
Optimize make_date (#9089) (#9600)
* Optimize make_date (#9089)
* replace the expensive calculation of unix_days_from_ce with a constant
* do not use PrimitiveArray builder for the scalar case
* Improve function name
---------
Co-authored-by: Vojtech Toman <vo...@graphaware.com>
Co-authored-by: Andrew Lamb <an...@nerdnetworks.org>
---
datafusion/functions/src/datetime/make_date.rs | 140 ++++++++++++-------------
1 file changed, 66 insertions(+), 74 deletions(-)
diff --git a/datafusion/functions/src/datetime/make_date.rs b/datafusion/functions/src/datetime/make_date.rs
index b8010f64d2..8f892aafb4 100644
--- a/datafusion/functions/src/datetime/make_date.rs
+++ b/datafusion/functions/src/datetime/make_date.rs
@@ -80,47 +80,10 @@ impl ScalarUDFImpl for MakeDateFunc {
ColumnarValue::Array(a) => Some(a.len()),
});
- let is_scalar = len.is_none();
- let array_size = if is_scalar { 1 } else { len.unwrap() };
-
let years = args[0].cast_to(&DataType::Int32, None)?;
let months = args[1].cast_to(&DataType::Int32, None)?;
let days = args[2].cast_to(&DataType::Int32, None)?;
- // since the epoch for the date32 datatype is the unix epoch
- // we need to subtract the unix epoch from the current date
- // note this can result in a negative value
- let unix_days_from_ce = NaiveDate::from_ymd_opt(1970, 1, 1)
- .unwrap()
- .num_days_from_ce();
-
- let mut builder: PrimitiveBuilder<Date32Type> =
- PrimitiveArray::builder(array_size);
-
- let construct_date_fn = |builder: &mut PrimitiveBuilder<Date32Type>,
- year: i32,
- month: i32,
- day: i32,
- unix_days_from_ce: i32|
- -> Result<()> {
- let Ok(m) = u32::try_from(month) else {
- return exec_err!("Month value '{month:?}' is out of range");
- };
- let Ok(d) = u32::try_from(day) else {
- return exec_err!("Day value '{day:?}' is out of range");
- };
-
- let date = NaiveDate::from_ymd_opt(year, m, d);
-
- match date {
- Some(d) => builder.append_value(d.num_days_from_ce() - unix_days_from_ce),
- None => {
- return exec_err!("Unable to parse date from {year}, {month}, {day}")
- }
- };
- Ok(())
- };
-
let scalar_value_fn = |col: &ColumnarValue| -> Result<i32> {
let ColumnarValue::Scalar(s) = col else {
return exec_err!("Expected scalar value");
@@ -131,54 +94,83 @@ impl ScalarUDFImpl for MakeDateFunc {
Ok(*i)
};
- // For scalar only columns the operation is faster without using the PrimitiveArray
- if is_scalar {
- construct_date_fn(
- &mut builder,
- scalar_value_fn(&years)?,
- scalar_value_fn(&months)?,
- scalar_value_fn(&days)?,
- unix_days_from_ce,
- )?;
- } else {
- let to_primitive_array = |col: &ColumnarValue,
- scalar_count: usize|
- -> Result<PrimitiveArray<Int32Type>> {
- match col {
- ColumnarValue::Array(a) => {
- Ok(a.as_primitive::<Int32Type>().to_owned())
+ let value = if let Some(array_size) = len {
+ let to_primitive_array_fn =
+ |col: &ColumnarValue| -> PrimitiveArray<Int32Type> {
+ match col {
+ ColumnarValue::Array(a) => {
+ a.as_primitive::<Int32Type>().to_owned()
+ }
+ _ => {
+ let v = scalar_value_fn(col).unwrap();
+ PrimitiveArray::<Int32Type>::from_value(v, array_size)
+ }
}
- _ => {
- let v = scalar_value_fn(col).unwrap();
- Ok(PrimitiveArray::<Int32Type>::from_value(v, scalar_count))
- }
- }
- };
+ };
- let years = to_primitive_array(&years, array_size).unwrap();
- let months = to_primitive_array(&months, array_size).unwrap();
- let days = to_primitive_array(&days, array_size).unwrap();
+ let years = to_primitive_array_fn(&years);
+ let months = to_primitive_array_fn(&months);
+ let days = to_primitive_array_fn(&days);
+
+ let mut builder: PrimitiveBuilder<Date32Type> =
+ PrimitiveArray::builder(array_size);
for i in 0..array_size {
- construct_date_fn(
- &mut builder,
+ make_date_inner(
years.value(i),
months.value(i),
days.value(i),
- unix_days_from_ce,
+ |days: i32| builder.append_value(days),
)?;
}
- }
- let arr = builder.finish();
+ let arr = builder.finish();
- if is_scalar {
- // If all inputs are scalar, keeps output as scalar
- Ok(ColumnarValue::Scalar(ScalarValue::Date32(Some(
- arr.value(0),
- ))))
+ ColumnarValue::Array(Arc::new(arr))
} else {
- Ok(ColumnarValue::Array(Arc::new(arr)))
- }
+ // For scalar only columns the operation is faster without using the PrimitiveArray.
+ // Also, keep the output as scalar since all inputs are scalar.
+ let mut value = 0;
+ make_date_inner(
+ scalar_value_fn(&years)?,
+ scalar_value_fn(&months)?,
+ scalar_value_fn(&days)?,
+ |days: i32| value = days,
+ )?;
+
+ ColumnarValue::Scalar(ScalarValue::Date32(Some(value)))
+ };
+
+ Ok(value)
+ }
+}
+
+/// Converts the year/month/day fields to an `i32` representing the days from
+/// the unix epoch and invokes `date_consumer_fn` with the value
+fn make_date_inner<F: FnMut(i32)>(
+ year: i32,
+ month: i32,
+ day: i32,
+ mut date_consumer_fn: F,
+) -> Result<()> {
+ let Ok(m) = u32::try_from(month) else {
+ return exec_err!("Month value '{month:?}' is out of range");
+ };
+ let Ok(d) = u32::try_from(day) else {
+ return exec_err!("Day value '{day:?}' is out of range");
+ };
+
+ if let Some(date) = NaiveDate::from_ymd_opt(year, m, d) {
+ // The number of days until the start of the unix epoch in the proleptic Gregorian calendar
+ // (with January 1, Year 1 (CE) as day 1). See [Datelike::num_days_from_ce].
+ const UNIX_DAYS_FROM_CE: i32 = 719_163;
+
+ // since the epoch for the date32 datatype is the unix epoch
+ // we need to subtract the unix epoch from the current date
+ // note that this can result in a negative value
+ date_consumer_fn(date.num_days_from_ce() - UNIX_DAYS_FROM_CE);
+ Ok(())
+ } else {
+ exec_err!("Unable to parse date from {year}, {month}, {day}")
}
}