You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by al...@apache.org on 2022/07/27 18:55:43 UTC
[arrow-datafusion] branch master updated: Update to arrow `19.0.0` (#2955)
This is an automated email from the ASF dual-hosted git repository.
alamb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
The following commit(s) were added to refs/heads/master by this push:
new cd3164918 Update to arrow `19.0.0` (#2955)
cd3164918 is described below
commit cd3164918b0415b072a3109f3ccf654da7518ec1
Author: Andrew Lamb <an...@nerdnetworks.org>
AuthorDate: Wed Jul 27 14:55:37 2022 -0400
Update to arrow `19.0.0` (#2955)
* Update to use arrow 19.0.0
* Update for infallable builder and Decimal128Array
* Update avro for new API
* Update more APIs for arrow 19.0.0
* Fix tests and clippy
---
datafusion-cli/Cargo.toml | 2 +-
datafusion-examples/Cargo.toml | 2 +-
datafusion-examples/examples/custom_datasource.rs | 4 +-
datafusion/common/Cargo.toml | 4 +-
datafusion/common/src/scalar.rs | 149 ++++++++----------
datafusion/core/Cargo.toml | 4 +-
datafusion/core/fuzz-utils/Cargo.toml | 2 +-
.../core/src/avro_to_arrow/arrow_array_reader.rs | 102 ++++++-------
datafusion/core/src/avro_to_arrow/schema.rs | 1 +
datafusion/core/src/catalog/information_schema.rs | 86 ++++-------
datafusion/core/src/datasource/listing/helpers.rs | 8 +-
datafusion/core/src/physical_optimizer/pruning.rs | 6 +-
.../core/src/physical_plan/aggregates/hash.rs | 2 +-
.../core/src/physical_plan/aggregates/row_hash.rs | 2 +-
datafusion/core/src/physical_plan/analyze.rs | 18 +--
datafusion/core/src/physical_plan/explain.rs | 6 +-
datafusion/core/src/physical_plan/hash_join.rs | 43 +++---
datafusion/core/src/physical_plan/hash_utils.rs | 6 +-
datafusion/core/src/physical_plan/repartition.rs | 3 +-
.../core/src/physical_plan/sort_merge_join.rs | 20 +--
datafusion/core/src/test/mod.rs | 4 +-
datafusion/core/tests/provider_filter_pushdown.rs | 2 +-
datafusion/core/tests/sql/mod.rs | 16 +-
datafusion/core/tests/sql/select.rs | 14 +-
datafusion/expr/Cargo.toml | 2 +-
datafusion/jit/Cargo.toml | 2 +-
datafusion/optimizer/Cargo.toml | 2 +-
datafusion/physical-expr/Cargo.toml | 2 +-
datafusion/physical-expr/src/aggregate/average.rs | 6 +-
.../physical-expr/src/aggregate/count_distinct.rs | 16 +-
datafusion/physical-expr/src/aggregate/min_max.rs | 24 +--
datafusion/physical-expr/src/aggregate/sum.rs | 14 +-
.../physical-expr/src/aggregate/sum_distinct.rs | 2 +-
datafusion/physical-expr/src/array_expressions.rs | 23 +--
.../physical-expr/src/datetime_expressions.rs | 10 +-
datafusion/physical-expr/src/expressions/binary.rs | 170 ++++++++++++---------
datafusion/physical-expr/src/expressions/cast.rs | 47 +++---
.../physical-expr/src/expressions/datetime.rs | 2 +-
.../src/expressions/get_indexed_field.rs | 13 +-
.../physical-expr/src/expressions/in_list.rs | 12 +-
.../physical-expr/src/expressions/try_cast.rs | 48 +++---
datafusion/physical-expr/src/regex_expressions.rs | 32 ++--
datafusion/proto/Cargo.toml | 2 +-
datafusion/proto/src/to_proto.rs | 6 +-
datafusion/row/Cargo.toml | 2 +-
datafusion/row/src/reader.rs | 16 +-
datafusion/sql/Cargo.toml | 2 +-
47 files changed, 469 insertions(+), 492 deletions(-)
diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml
index 1684ecf50..6ac3a30c8 100644
--- a/datafusion-cli/Cargo.toml
+++ b/datafusion-cli/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.59"
readme = "README.md"
[dependencies]
-arrow = { version = "18.0.0" }
+arrow = { version = "19.0.0" }
clap = { version = "3", features = ["derive", "cargo"] }
datafusion = { path = "../datafusion/core", version = "10.0.0" }
dirs = "4.0.0"
diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml
index 00d2a68c9..ed23512f3 100644
--- a/datafusion-examples/Cargo.toml
+++ b/datafusion-examples/Cargo.toml
@@ -34,7 +34,7 @@ path = "examples/avro_sql.rs"
required-features = ["datafusion/avro"]
[dev-dependencies]
-arrow-flight = { version = "18.0.0" }
+arrow-flight = { version = "19.0.0" }
async-trait = "0.1.41"
datafusion = { path = "../datafusion/core" }
futures = "0.3"
diff --git a/datafusion-examples/examples/custom_datasource.rs b/datafusion-examples/examples/custom_datasource.rs
index b68936a7c..2be4f194c 100644
--- a/datafusion-examples/examples/custom_datasource.rs
+++ b/datafusion-examples/examples/custom_datasource.rs
@@ -247,8 +247,8 @@ impl ExecutionPlan for CustomExec {
let mut account_array = UInt64Builder::new(users.len());
for user in users {
- id_array.append_value(user.id)?;
- account_array.append_value(user.bank_account)?;
+ id_array.append_value(user.id);
+ account_array.append_value(user.bank_account);
}
Ok(Box::pin(MemoryStream::try_new(
diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml
index 0e1a79445..e121a55a4 100644
--- a/datafusion/common/Cargo.toml
+++ b/datafusion/common/Cargo.toml
@@ -38,11 +38,11 @@ jit = ["cranelift-module"]
pyarrow = ["pyo3"]
[dependencies]
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
cranelift-module = { version = "0.86.1", optional = true }
object_store = { version = "0.3", optional = true }
ordered-float = "3.0"
-parquet = { version = "18.0.0", features = ["arrow"], optional = true }
+parquet = { version = "19.0.0", features = ["arrow"], optional = true }
pyo3 = { version = "0.16", optional = true }
sqlparser = "0.18"
diff --git a/datafusion/common/src/scalar.rs b/datafusion/common/src/scalar.rs
index c2b0e875b..b30238762 100644
--- a/datafusion/common/src/scalar.rs
+++ b/datafusion/common/src/scalar.rs
@@ -527,15 +527,15 @@ macro_rules! build_values_list {
for scalar_value in $VALUES {
match scalar_value {
ScalarValue::$SCALAR_TY(Some(v)) => {
- builder.values().append_value(v.clone()).unwrap()
+ builder.values().append_value(v.clone());
}
ScalarValue::$SCALAR_TY(None) => {
- builder.values().append_null().unwrap();
+ builder.values().append_null();
}
_ => panic!("Incompatible ScalarValue for list"),
};
}
- builder.append(true).unwrap();
+ builder.append(true);
}
builder.finish()
@@ -550,15 +550,15 @@ macro_rules! build_values_list_tz {
for scalar_value in $VALUES {
match scalar_value {
ScalarValue::$SCALAR_TY(Some(v), _) => {
- builder.values().append_value(v.clone()).unwrap()
+ builder.values().append_value(v.clone());
}
ScalarValue::$SCALAR_TY(None, _) => {
- builder.values().append_null().unwrap();
+ builder.values().append_null();
}
_ => panic!("Incompatible ScalarValue for list"),
};
}
- builder.append(true).unwrap();
+ builder.append(true);
}
builder.finish()
@@ -879,10 +879,10 @@ impl ScalarValue {
for s in xs {
match s {
ScalarValue::$SCALAR_TY(Some(val)) => {
- builder.values().append_value(val)?;
+ builder.values().append_value(val);
}
ScalarValue::$SCALAR_TY(None) => {
- builder.values().append_null()?;
+ builder.values().append_null();
}
sv => {
return Err(DataFusionError::Internal(format!(
@@ -893,10 +893,10 @@ impl ScalarValue {
}
}
}
- builder.append(true)?;
+ builder.append(true);
}
ScalarValue::List(None, _) => {
- builder.append(false)?;
+ builder.append(false);
}
sv => {
return Err(DataFusionError::Internal(format!(
@@ -917,6 +917,11 @@ impl ScalarValue {
ScalarValue::iter_to_decimal_array(scalars, precision, scale)?;
Arc::new(decimal_array)
}
+ DataType::Decimal256(_, _) => {
+ return Err(DataFusionError::Internal(
+ "Decimal256 is not supported for ScalarValue".to_string(),
+ ))
+ }
DataType::Null => ScalarValue::iter_to_null_array(scalars),
DataType::Boolean => build_array_primitive!(BooleanArray, Boolean),
DataType::Float32 => build_array_primitive!(Float32Array, Float32),
@@ -1112,14 +1117,14 @@ impl ScalarValue {
scalars: impl IntoIterator<Item = ScalarValue>,
precision: &usize,
scale: &usize,
- ) -> Result<DecimalArray> {
+ ) -> Result<Decimal128Array> {
let array = scalars
.into_iter()
.map(|element: ScalarValue| match element {
ScalarValue::Decimal128(v1, _, _) => v1,
_ => unreachable!(),
})
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(*precision, *scale)?;
Ok(array)
}
@@ -1129,9 +1134,7 @@ impl ScalarValue {
data_type: &DataType,
) -> Result<GenericListArray<i32>> {
let mut offsets = Int32Array::builder(0);
- if let Err(err) = offsets.append_value(0) {
- return Err(DataFusionError::ArrowError(err));
- }
+ offsets.append_value(0);
let mut elements: Vec<ArrayRef> = Vec::new();
let mut valid = BooleanBufferBuilder::new(0);
@@ -1144,9 +1147,7 @@ impl ScalarValue {
// Add new offset index
flat_len += element_array.len() as i32;
- if let Err(err) = offsets.append_value(flat_len) {
- return Err(DataFusionError::ArrowError(err));
- }
+ offsets.append_value(flat_len);
elements.push(element_array);
@@ -1155,9 +1156,7 @@ impl ScalarValue {
}
None => {
// Repeat previous offset index
- if let Err(err) = offsets.append_value(flat_len) {
- return Err(DataFusionError::ArrowError(err));
- }
+ offsets.append_value(flat_len);
// Element is null
valid.append(false);
@@ -1196,10 +1195,10 @@ impl ScalarValue {
precision: &usize,
scale: &usize,
size: usize,
- ) -> DecimalArray {
+ ) -> Decimal128Array {
std::iter::repeat(value)
.take(size)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(*precision, *scale)
.unwrap()
}
@@ -1405,7 +1404,7 @@ impl ScalarValue {
precision: &usize,
scale: &usize,
) -> ScalarValue {
- let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
if array.is_null(index) {
ScalarValue::Decimal128(None, *precision, *scale)
} else {
@@ -1593,7 +1592,7 @@ impl ScalarValue {
precision: usize,
scale: usize,
) -> bool {
- let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
if array.precision() != precision || array.scale() != scale {
return false;
}
@@ -2139,14 +2138,14 @@ mod tests {
// decimal scalar to array
let array = decimal_value.to_array();
- let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(1, array.len());
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
assert_eq!(123i128, array.value(0).as_i128());
// decimal scalar to array with size
let array = decimal_value.to_array_of_size(10);
- let array_decimal = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array_decimal = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
assert_eq!(10, array.len());
assert_eq!(DataType::Decimal(10, 1), array.data_type().clone());
assert_eq!(123i128, array_decimal.value(0).as_i128());
@@ -3106,116 +3105,100 @@ mod tests {
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
- .append_value("First")
- .unwrap();
+ .append_value("First");
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(1)
- .unwrap();
+ .append_value(1);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(2)
- .unwrap();
+ .append_value(2);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(3)
- .unwrap();
+ .append_value(3);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
- .append(true)
- .unwrap();
- list_builder.values().append(true).unwrap();
+ .append(true);
+ list_builder.values().append(true);
list_builder
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
- .append_value("Second")
- .unwrap();
+ .append_value("Second");
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(4)
- .unwrap();
+ .append_value(4);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(5)
- .unwrap();
+ .append_value(5);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
- .append(true)
- .unwrap();
- list_builder.values().append(true).unwrap();
- list_builder.append(true).unwrap();
+ .append(true);
+ list_builder.values().append(true);
+ list_builder.append(true);
list_builder
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
- .append_value("Third")
- .unwrap();
+ .append_value("Third");
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(6)
- .unwrap();
+ .append_value(6);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
- .append(true)
- .unwrap();
- list_builder.values().append(true).unwrap();
- list_builder.append(true).unwrap();
+ .append(true);
+ list_builder.values().append(true);
+ list_builder.append(true);
list_builder
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
- .append_value("Second")
- .unwrap();
+ .append_value("Second");
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(4)
- .unwrap();
+ .append_value(4);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
.values()
- .append_value(5)
- .unwrap();
+ .append_value(5);
list_builder
.values()
.field_builder::<ListBuilder<PrimitiveBuilder<Int32Type>>>(1)
.unwrap()
- .append(true)
- .unwrap();
- list_builder.values().append(true).unwrap();
- list_builder.append(true).unwrap();
+ .append(true);
+ list_builder.values().append(true);
+ list_builder.append(true);
let expected = list_builder.finish();
@@ -3285,27 +3268,27 @@ mod tests {
let middle_builder = ListBuilder::new(inner_builder);
let mut outer_builder = ListBuilder::new(middle_builder);
- outer_builder.values().values().append_value(1).unwrap();
- outer_builder.values().values().append_value(2).unwrap();
- outer_builder.values().values().append_value(3).unwrap();
- outer_builder.values().append(true).unwrap();
+ outer_builder.values().values().append_value(1);
+ outer_builder.values().values().append_value(2);
+ outer_builder.values().values().append_value(3);
+ outer_builder.values().append(true);
- outer_builder.values().values().append_value(4).unwrap();
- outer_builder.values().values().append_value(5).unwrap();
- outer_builder.values().append(true).unwrap();
- outer_builder.append(true).unwrap();
+ outer_builder.values().values().append_value(4);
+ outer_builder.values().values().append_value(5);
+ outer_builder.values().append(true);
+ outer_builder.append(true);
- outer_builder.values().values().append_value(6).unwrap();
- outer_builder.values().append(true).unwrap();
+ outer_builder.values().values().append_value(6);
+ outer_builder.values().append(true);
- outer_builder.values().values().append_value(7).unwrap();
- outer_builder.values().values().append_value(8).unwrap();
- outer_builder.values().append(true).unwrap();
- outer_builder.append(true).unwrap();
+ outer_builder.values().values().append_value(7);
+ outer_builder.values().values().append_value(8);
+ outer_builder.values().append(true);
+ outer_builder.append(true);
- outer_builder.values().values().append_value(9).unwrap();
- outer_builder.values().append(true).unwrap();
- outer_builder.append(true).unwrap();
+ outer_builder.values().values().append_value(9);
+ outer_builder.values().append(true);
+ outer_builder.append(true);
let expected = outer_builder.finish();
diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml
index 351e72fc2..4385badb1 100644
--- a/datafusion/core/Cargo.toml
+++ b/datafusion/core/Cargo.toml
@@ -55,7 +55,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
avro-rs = { version = "0.13", features = ["snappy"], optional = true }
bytes = "1.1"
@@ -78,7 +78,7 @@ num_cpus = "1.13.0"
object_store = "0.3.0"
ordered-float = "3.0"
parking_lot = "0.12"
-parquet = { version = "18.0.0", features = ["arrow", "async"] }
+parquet = { version = "19.0.0", features = ["arrow", "async"] }
paste = "^1.0"
pin-project-lite = "^0.2.7"
pyo3 = { version = "0.16", optional = true }
diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml
index 8544d88f1..0d66a6999 100644
--- a/datafusion/core/fuzz-utils/Cargo.toml
+++ b/datafusion/core/fuzz-utils/Cargo.toml
@@ -23,6 +23,6 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
env_logger = "0.9.0"
rand = "0.8"
diff --git a/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs b/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs
index e82f732aa..358a6d048 100644
--- a/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs
+++ b/datafusion/core/src/avro_to_arrow/arrow_array_reader.rs
@@ -130,58 +130,52 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
arrays.and_then(|arr| RecordBatch::try_new(projected_schema, arr).map(Some))
}
- fn build_boolean_array(
- &self,
- rows: RecordSlice,
- col_name: &str,
- ) -> ArrowResult<ArrayRef> {
+ fn build_boolean_array(&self, rows: RecordSlice, col_name: &str) -> ArrayRef {
let mut builder = BooleanBuilder::new(rows.len());
for row in rows {
if let Some(value) = self.field_lookup(col_name, row) {
if let Some(boolean) = resolve_boolean(value) {
- builder.append_value(boolean)?
+ builder.append_value(boolean)
} else {
- builder.append_null()?;
+ builder.append_null();
}
} else {
- builder.append_null()?;
+ builder.append_null();
}
}
- Ok(Arc::new(builder.finish()))
+ Arc::new(builder.finish())
}
- #[allow(clippy::unnecessary_wraps)]
fn build_primitive_array<T: ArrowPrimitiveType + Resolver>(
&self,
rows: RecordSlice,
col_name: &str,
- ) -> ArrowResult<ArrayRef>
+ ) -> ArrayRef
where
T: ArrowNumericType,
T::Native: num_traits::cast::NumCast,
{
- Ok(Arc::new(
+ Arc::new(
rows.iter()
.map(|row| {
self.field_lookup(col_name, row)
.and_then(|value| resolve_item::<T>(value))
})
.collect::<PrimitiveArray<T>>(),
- ))
+ )
}
#[inline(always)]
- #[allow(clippy::unnecessary_wraps)]
fn build_string_dictionary_builder<T>(
&self,
row_len: usize,
- ) -> ArrowResult<StringDictionaryBuilder<T>>
+ ) -> StringDictionaryBuilder<T>
where
T: ArrowPrimitiveType + ArrowDictionaryKeyType,
{
let key_builder = PrimitiveBuilder::<T>::new(row_len);
let values_builder = StringBuilder::new(row_len * 5);
- Ok(StringDictionaryBuilder::new(key_builder, values_builder))
+ StringDictionaryBuilder::new(key_builder, values_builder)
}
fn build_wrapped_list_array(
@@ -271,7 +265,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
}
DataType::Dictionary(_, _) => {
let values_builder =
- self.build_string_dictionary_builder::<D>(rows.len() * 5)?;
+ self.build_string_dictionary_builder::<D>(rows.len() * 5);
Box::new(ListBuilder::new(values_builder))
}
e => {
@@ -316,14 +310,14 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
))?;
for val in vals {
if let Some(v) = val {
- builder.values().append_value(&v)?
+ builder.values().append_value(&v)
} else {
- builder.values().append_null()?
+ builder.values().append_null()
};
}
// Append to the list
- builder.append(true)?;
+ builder.append(true);
}
DataType::Dictionary(_, _) => {
let builder = builder.as_any_mut().downcast_mut::<ListBuilder<StringDictionaryBuilder<D>>>().ok_or_else(||ArrowError::SchemaError(
@@ -333,12 +327,12 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
if let Some(v) = val {
let _ = builder.values().append(&v)?;
} else {
- builder.values().append_null()?
+ builder.values().append_null()
};
}
// Append to the list
- builder.append(true)?;
+ builder.append(true);
}
e => {
return Err(SchemaError(format!(
@@ -364,16 +358,16 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
T: ArrowPrimitiveType + ArrowDictionaryKeyType,
{
let mut builder: StringDictionaryBuilder<T> =
- self.build_string_dictionary_builder(rows.len())?;
+ self.build_string_dictionary_builder(rows.len());
for row in rows {
if let Some(value) = self.field_lookup(col_name, row) {
if let Ok(str_v) = resolve_string(value) {
builder.append(str_v).map(drop)?
} else {
- builder.append_null()?
+ builder.append_null()
}
} else {
- builder.append_null()?
+ builder.append_null()
}
}
Ok(Arc::new(builder.finish()) as ArrayRef)
@@ -609,10 +603,8 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
.iter()
.filter(|field| projection.is_empty() || projection.contains(field.name()))
.map(|field| {
- match field.data_type() {
- DataType::Null => {
- Ok(Arc::new(NullArray::new(rows.len())) as ArrayRef)
- }
+ let arr = match field.data_type() {
+ DataType::Null => Arc::new(NullArray::new(rows.len())) as ArrayRef,
DataType::Boolean => self.build_boolean_array(rows, field.name()),
DataType::Float64 => {
self.build_primitive_array::<Float64Type>(rows, field.name())
@@ -684,10 +676,12 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
rows,
field.name(),
),
- t => Err(ArrowError::SchemaError(format!(
- "TimeUnit {:?} not supported with Time64",
- t
- ))),
+ t => {
+ return Err(ArrowError::SchemaError(format!(
+ "TimeUnit {:?} not supported with Time64",
+ t
+ )))
+ }
},
DataType::Time32(unit) => match unit {
TimeUnit::Second => self
@@ -700,12 +694,14 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
rows,
field.name(),
),
- t => Err(ArrowError::SchemaError(format!(
- "TimeUnit {:?} not supported with Time32",
- t
- ))),
+ t => {
+ return Err(ArrowError::SchemaError(format!(
+ "TimeUnit {:?} not supported with Time32",
+ t
+ )))
+ }
},
- DataType::Utf8 | DataType::LargeUtf8 => Ok(Arc::new(
+ DataType::Utf8 | DataType::LargeUtf8 => Arc::new(
rows.iter()
.map(|row| {
let maybe_value = self.field_lookup(field.name(), row);
@@ -713,8 +709,8 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
})
.collect::<ArrowResult<StringArray>>()?,
)
- as ArrayRef),
- DataType::Binary | DataType::LargeBinary => Ok(Arc::new(
+ as ArrayRef,
+ DataType::Binary | DataType::LargeBinary => Arc::new(
rows.iter()
.map(|row| {
let maybe_value = self.field_lookup(field.name(), row);
@@ -722,11 +718,11 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
})
.collect::<BinaryArray>(),
)
- as ArrayRef),
+ as ArrayRef,
DataType::List(ref list_field) => {
match list_field.data_type() {
DataType::Dictionary(ref key_ty, _) => {
- self.build_wrapped_list_array(rows, field.name(), key_ty)
+ self.build_wrapped_list_array(rows, field.name(), key_ty)?
}
_ => {
// extract rows by name
@@ -740,7 +736,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
self.build_nested_list_array::<i32>(
extracted_rows.as_slice(),
list_field,
- )
+ )?
}
}
}
@@ -750,7 +746,7 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
field.name(),
key_ty,
val_ty,
- ),
+ )?,
DataType::Struct(fields) => {
let len = rows.len();
let num_bytes = bit_util::ceil(len, 8);
@@ -778,15 +774,17 @@ impl<'a, R: Read> AvroArrowArrayReader<'a, R> {
.child_data(
arrays.into_iter().map(|a| a.data().clone()).collect(),
)
- .build()
- .unwrap();
- Ok(make_array(data))
+ .build()?;
+ make_array(data)
}
- _ => Err(ArrowError::SchemaError(format!(
- "type {:?} not supported",
- field.data_type()
- ))),
- }
+ _ => {
+ return Err(ArrowError::SchemaError(format!(
+ "type {:?} not supported",
+ field.data_type()
+ )))
+ }
+ };
+ Ok(arr)
})
.collect();
arrays
diff --git a/datafusion/core/src/avro_to_arrow/schema.rs b/datafusion/core/src/avro_to_arrow/schema.rs
index dd8b90776..c5b4febbb 100644
--- a/datafusion/core/src/avro_to_arrow/schema.rs
+++ b/datafusion/core/src/avro_to_arrow/schema.rs
@@ -217,6 +217,7 @@ fn default_field_name(dt: &DataType) -> &str {
DataType::Dictionary(_, _) => "map",
DataType::Map(_, _) => unimplemented!("Map support not implemented"),
DataType::Decimal(_, _) => "decimal",
+ DataType::Decimal256(_, _) => "decimal",
}
}
diff --git a/datafusion/core/src/catalog/information_schema.rs b/datafusion/core/src/catalog/information_schema.rs
index 0467dffa7..d4944c2d5 100644
--- a/datafusion/core/src/catalog/information_schema.rs
+++ b/datafusion/core/src/catalog/information_schema.rs
@@ -266,20 +266,14 @@ impl InformationSchemaTablesBuilder {
table_type: TableType,
) {
// Note: append_value is actually infallable.
- self.catalog_names
- .append_value(catalog_name.as_ref())
- .unwrap();
- self.schema_names
- .append_value(schema_name.as_ref())
- .unwrap();
- self.table_names.append_value(table_name.as_ref()).unwrap();
- self.table_types
- .append_value(match table_type {
- TableType::Base => "BASE TABLE",
- TableType::View => "VIEW",
- TableType::Temporary => "LOCAL TEMPORARY",
- })
- .unwrap();
+ self.catalog_names.append_value(catalog_name.as_ref());
+ self.schema_names.append_value(schema_name.as_ref());
+ self.table_names.append_value(table_name.as_ref());
+ self.table_types.append_value(match table_type {
+ TableType::Base => "BASE TABLE",
+ TableType::View => "VIEW",
+ TableType::Temporary => "LOCAL TEMPORARY",
+ });
}
}
@@ -347,14 +341,10 @@ impl InformationSchemaViewBuilder {
definition: Option<impl AsRef<str>>,
) {
// Note: append_value is actually infallable.
- self.catalog_names
- .append_value(catalog_name.as_ref())
- .unwrap();
- self.schema_names
- .append_value(schema_name.as_ref())
- .unwrap();
- self.table_names.append_value(table_name.as_ref()).unwrap();
- self.definitions.append_option(definition.as_ref()).unwrap();
+ self.catalog_names.append_value(catalog_name.as_ref());
+ self.schema_names.append_value(schema_name.as_ref());
+ self.table_names.append_value(table_name.as_ref());
+ self.definitions.append_option(definition.as_ref());
}
}
@@ -450,33 +440,23 @@ impl InformationSchemaColumnsBuilder {
use DataType::*;
// Note: append_value is actually infallable.
- self.catalog_names
- .append_value(catalog_name.as_ref())
- .unwrap();
- self.schema_names
- .append_value(schema_name.as_ref())
- .unwrap();
- self.table_names.append_value(table_name.as_ref()).unwrap();
-
- self.column_names
- .append_value(column_name.as_ref())
- .unwrap();
-
- self.ordinal_positions
- .append_value(column_position as u64)
- .unwrap();
+ self.catalog_names.append_value(catalog_name.as_ref());
+ self.schema_names.append_value(schema_name.as_ref());
+ self.table_names.append_value(table_name.as_ref());
+
+ self.column_names.append_value(column_name.as_ref());
+
+ self.ordinal_positions.append_value(column_position as u64);
// DataFusion does not support column default values, so null
- self.column_defaults.append_null().unwrap();
+ self.column_defaults.append_null();
// "YES if the column is possibly nullable, NO if it is known not nullable. "
let nullable_str = if is_nullable { "YES" } else { "NO" };
- self.is_nullables.append_value(nullable_str).unwrap();
+ self.is_nullables.append_value(nullable_str);
// "System supplied type" --> Use debug format of the datatype
- self.data_types
- .append_value(format!("{:?}", data_type))
- .unwrap();
+ self.data_types.append_value(format!("{:?}", data_type));
// "If data_type identifies a character or bit string type, the
// declared maximum length; null for all other data types or
@@ -484,9 +464,7 @@ impl InformationSchemaColumnsBuilder {
//
// Arrow has no equivalent of VARCHAR(20), so we leave this as Null
let max_chars = None;
- self.character_maximum_lengths
- .append_option(max_chars)
- .unwrap();
+ self.character_maximum_lengths.append_option(max_chars);
// "Maximum length, in bytes, for binary data, character data,
// or text and image data."
@@ -495,9 +473,7 @@ impl InformationSchemaColumnsBuilder {
LargeBinary | LargeUtf8 => Some(i64::MAX as u64),
_ => None,
};
- self.character_octet_lengths
- .append_option(char_len)
- .unwrap();
+ self.character_octet_lengths.append_option(char_len);
// numeric_precision: "If data_type identifies a numeric type, this column
// contains the (declared or implicit) precision of the type
@@ -538,16 +514,12 @@ impl InformationSchemaColumnsBuilder {
_ => (None, None, None),
};
- self.numeric_precisions
- .append_option(numeric_precision)
- .unwrap();
- self.numeric_precision_radixes
- .append_option(numeric_radix)
- .unwrap();
- self.numeric_scales.append_option(numeric_scale).unwrap();
+ self.numeric_precisions.append_option(numeric_precision);
+ self.numeric_precision_radixes.append_option(numeric_radix);
+ self.numeric_scales.append_option(numeric_scale);
- self.datetime_precisions.append_option(None).unwrap();
- self.interval_types.append_null().unwrap();
+ self.datetime_precisions.append_option(None);
+ self.interval_types.append_null();
}
}
diff --git a/datafusion/core/src/datasource/listing/helpers.rs b/datafusion/core/src/datasource/listing/helpers.rs
index b4b93d5fc..80fece684 100644
--- a/datafusion/core/src/datasource/listing/helpers.rs
+++ b/datafusion/core/src/datasource/listing/helpers.rs
@@ -247,11 +247,11 @@ fn paths_to_batch(
&file_meta.location,
table_partition_cols,
) {
- key_builder.append_value(file_meta.location.as_ref())?;
- length_builder.append_value(file_meta.size as u64)?;
- modified_builder.append_value(file_meta.last_modified.timestamp_millis())?;
+ key_builder.append_value(file_meta.location.as_ref());
+ length_builder.append_value(file_meta.size as u64);
+ modified_builder.append_value(file_meta.last_modified.timestamp_millis());
for (i, part_val) in partition_values.iter().enumerate() {
- partition_builders[i].append_value(part_val)?;
+ partition_builders[i].append_value(part_val);
}
} else {
debug!("No partitioning for path {}", file_meta.location);
diff --git a/datafusion/core/src/physical_optimizer/pruning.rs b/datafusion/core/src/physical_optimizer/pruning.rs
index 2265675b3..b1242b150 100644
--- a/datafusion/core/src/physical_optimizer/pruning.rs
+++ b/datafusion/core/src/physical_optimizer/pruning.rs
@@ -800,7 +800,7 @@ mod tests {
use crate::from_slice::FromSlice;
use crate::logical_plan::{col, lit};
use crate::{assert_batches_eq, physical_optimizer::pruning::StatisticsType};
- use arrow::array::DecimalArray;
+ use arrow::array::Decimal128Array;
use arrow::{
array::{BinaryArray, Int32Array, Int64Array, StringArray},
datatypes::{DataType, TimeUnit},
@@ -825,13 +825,13 @@ mod tests {
Self {
min: Arc::new(
min.into_iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(precision, scale)
.unwrap(),
),
max: Arc::new(
max.into_iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(precision, scale)
.unwrap(),
),
diff --git a/datafusion/core/src/physical_plan/aggregates/hash.rs b/datafusion/core/src/physical_plan/aggregates/hash.rs
index ddf9af18f..c21109495 100644
--- a/datafusion/core/src/physical_plan/aggregates/hash.rs
+++ b/datafusion/core/src/physical_plan/aggregates/hash.rs
@@ -273,7 +273,7 @@ fn group_aggregate_batch(
let mut offset_so_far = 0;
for group_idx in groups_with_rows.iter() {
let indices = &accumulators.group_states[*group_idx].indices;
- batch_indices.append_slice(indices)?;
+ batch_indices.append_slice(indices);
offset_so_far += indices.len();
offsets.push(offset_so_far);
}
diff --git a/datafusion/core/src/physical_plan/aggregates/row_hash.rs b/datafusion/core/src/physical_plan/aggregates/row_hash.rs
index 5353bc745..2ef1d16d4 100644
--- a/datafusion/core/src/physical_plan/aggregates/row_hash.rs
+++ b/datafusion/core/src/physical_plan/aggregates/row_hash.rs
@@ -288,7 +288,7 @@ fn group_aggregate_batch(
let mut offset_so_far = 0;
for group_idx in groups_with_rows.iter() {
let indices = &aggr_state.group_states[*group_idx].indices;
- batch_indices.append_slice(indices)?;
+ batch_indices.append_slice(indices);
offset_so_far += indices.len();
offsets.push(offset_so_far);
}
diff --git a/datafusion/core/src/physical_plan/analyze.rs b/datafusion/core/src/physical_plan/analyze.rs
index c2f08c69a..46273aa89 100644
--- a/datafusion/core/src/physical_plan/analyze.rs
+++ b/datafusion/core/src/physical_plan/analyze.rs
@@ -160,32 +160,30 @@ impl ExecutionPlan for AnalyzeExec {
let mut plan_builder = StringBuilder::new(1);
// TODO use some sort of enum rather than strings?
- type_builder.append_value("Plan with Metrics").unwrap();
+ type_builder.append_value("Plan with Metrics");
let annotated_plan =
DisplayableExecutionPlan::with_metrics(captured_input.as_ref())
.indent()
.to_string();
- plan_builder.append_value(annotated_plan).unwrap();
+ plan_builder.append_value(annotated_plan);
// Verbose output
// TODO make this more sophisticated
if verbose {
- type_builder.append_value("Plan with Full Metrics").unwrap();
+ type_builder.append_value("Plan with Full Metrics");
let annotated_plan =
DisplayableExecutionPlan::with_full_metrics(captured_input.as_ref())
.indent()
.to_string();
- plan_builder.append_value(annotated_plan).unwrap();
+ plan_builder.append_value(annotated_plan);
- type_builder.append_value("Output Rows").unwrap();
- plan_builder.append_value(total_rows.to_string()).unwrap();
+ type_builder.append_value("Output Rows");
+ plan_builder.append_value(total_rows.to_string());
- type_builder.append_value("Duration").unwrap();
- plan_builder
- .append_value(format!("{:?}", end - start))
- .unwrap();
+ type_builder.append_value("Duration");
+ plan_builder.append_value(format!("{:?}", end - start));
}
let maybe_batch = RecordBatch::try_new(
diff --git a/datafusion/core/src/physical_plan/explain.rs b/datafusion/core/src/physical_plan/explain.rs
index fdc139a7e..555b9e1e7 100644
--- a/datafusion/core/src/physical_plan/explain.rs
+++ b/datafusion/core/src/physical_plan/explain.rs
@@ -133,13 +133,13 @@ impl ExecutionPlan for ExplainExec {
let mut prev: Option<&StringifiedPlan> = None;
for p in plans_to_print {
- type_builder.append_value(p.plan_type.to_string())?;
+ type_builder.append_value(p.plan_type.to_string());
match prev {
Some(prev) if !should_show(prev, p) => {
- plan_builder.append_value("SAME TEXT AS ABOVE")?;
+ plan_builder.append_value("SAME TEXT AS ABOVE");
}
Some(_) | None => {
- plan_builder.append_value(&*p.plan)?;
+ plan_builder.append_value(&*p.plan);
}
}
prev = Some(p);
diff --git a/datafusion/core/src/physical_plan/hash_join.rs b/datafusion/core/src/physical_plan/hash_join.rs
index 6db504034..6540cc130 100644
--- a/datafusion/core/src/physical_plan/hash_join.rs
+++ b/datafusion/core/src/physical_plan/hash_join.rs
@@ -23,7 +23,7 @@ use ahash::RandomState;
use arrow::{
array::{
as_dictionary_array, as_string_array, ArrayData, ArrayRef, BasicDecimalArray,
- BooleanArray, Date32Array, Date64Array, DecimalArray, DictionaryArray,
+ BooleanArray, Date32Array, Date64Array, Decimal128Array, DictionaryArray,
LargeStringArray, PrimitiveArray, TimestampMicrosecondArray,
TimestampMillisecondArray, TimestampSecondArray, UInt32BufferBuilder,
UInt32Builder, UInt64BufferBuilder, UInt64Builder,
@@ -787,8 +787,8 @@ fn build_join_indexes(
&keys_values,
*null_equals_null,
)? {
- left_indices.append_value(i)?;
- right_indices.append_value(row as u32)?;
+ left_indices.append_value(i);
+ right_indices.append_value(row as u32);
}
}
};
@@ -811,22 +811,22 @@ fn build_join_indexes(
&keys_values,
*null_equals_null,
)? {
- left_indices.append_value(i)?;
- right_indices.append_value(row as u32)?;
+ left_indices.append_value(i);
+ right_indices.append_value(row as u32);
no_match = false;
}
}
// If no rows matched left, still must keep the right
// with all nulls for left
if no_match {
- left_indices.append_null()?;
- right_indices.append_value(row as u32)?;
+ left_indices.append_null();
+ right_indices.append_value(row as u32);
}
}
None => {
// when no match, add the row with None for the left side
- left_indices.append_null()?;
- right_indices.append_value(row as u32)?;
+ left_indices.append_null();
+ right_indices.append_value(row as u32);
}
}
}
@@ -901,13 +901,13 @@ fn apply_join_filter(
// If row index changes and row doesnt have match
// append (idx, null)
if right_indices.value(pos) != state.0 && !state.1 {
- right_rebuilt.append_value(state.0)?;
- left_rebuilt.append_null()?;
+ right_rebuilt.append_value(state.0);
+ left_rebuilt.append_null();
}
// If has match append matched row indices
if mask.value(pos) {
- right_rebuilt.append_value(right_indices.value(pos))?;
- left_rebuilt.append_value(left_indices.value(pos))?;
+ right_rebuilt.append_value(right_indices.value(pos));
+ left_rebuilt.append_value(left_indices.value(pos));
};
// Calculate if current row index has match
@@ -921,12 +921,11 @@ fn apply_join_filter(
},
)
// Append last row from right side if no match found
- .and_then(|(row_idx, has_match)| {
+ .map(|(row_idx, has_match)| {
if !has_match {
- right_rebuilt.append_value(row_idx)?;
- left_rebuilt.append_null()?;
+ right_rebuilt.append_value(row_idx);
+ left_rebuilt.append_null();
}
- Ok(())
})?;
Ok((left_rebuilt.finish(), right_rebuilt.finish()))
@@ -1103,7 +1102,7 @@ fn equal_rows(
DataType::Decimal(_, rscale) => {
if lscale == rscale {
equal_rows_elem!(
- DecimalArray,
+ Decimal128Array,
l,
r,
left,
@@ -2378,12 +2377,12 @@ mod tests {
)?;
let mut left_ids = UInt64Builder::new(0);
- left_ids.append_value(0)?;
- left_ids.append_value(1)?;
+ left_ids.append_value(0);
+ left_ids.append_value(1);
let mut right_ids = UInt32Builder::new(0);
- right_ids.append_value(0)?;
- right_ids.append_value(1)?;
+ right_ids.append_value(0);
+ right_ids.append_value(1);
assert_eq!(left_ids.finish(), l);
diff --git a/datafusion/core/src/physical_plan/hash_utils.rs b/datafusion/core/src/physical_plan/hash_utils.rs
index 81bf5c1f3..b9c34ec9b 100644
--- a/datafusion/core/src/physical_plan/hash_utils.rs
+++ b/datafusion/core/src/physical_plan/hash_utils.rs
@@ -21,7 +21,7 @@ use crate::error::{DataFusionError, Result};
use ahash::{CallHasher, RandomState};
use arrow::array::{
Array, ArrayRef, BasicDecimalArray, BooleanArray, Date32Array, Date64Array,
- DecimalArray, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
+ Decimal128Array, DictionaryArray, Float32Array, Float64Array, Int16Array, Int32Array,
Int64Array, Int8Array, LargeStringArray, StringArray, TimestampMicrosecondArray,
TimestampMillisecondArray, TimestampNanosecondArray, TimestampSecondArray,
UInt16Array, UInt32Array, UInt64Array, UInt8Array,
@@ -58,7 +58,7 @@ fn hash_decimal128<'a>(
hashes_buffer: &'a mut [u64],
mul_col: bool,
) {
- let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
if array.null_count() == 0 {
if mul_col {
for (i, hash) in hashes_buffer.iter_mut().enumerate() {
@@ -626,7 +626,7 @@ mod tests {
let array = vec![1, 2, 3, 4]
.into_iter()
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(20, 3)
.unwrap();
let array_ref = Arc::new(array);
diff --git a/datafusion/core/src/physical_plan/repartition.rs b/datafusion/core/src/physical_plan/repartition.rs
index 82efe2c4f..552e1820a 100644
--- a/datafusion/core/src/physical_plan/repartition.rs
+++ b/datafusion/core/src/physical_plan/repartition.rs
@@ -156,8 +156,7 @@ impl BatchPartitioner {
for (index, hash) in hash_buffer.iter().enumerate() {
indices[(*hash % *partitions as u64) as usize]
- .append_value(index as u64)
- .unwrap();
+ .append_value(index as u64);
}
for (partition, mut indices) in indices.into_iter().enumerate() {
diff --git a/datafusion/core/src/physical_plan/sort_merge_join.rs b/datafusion/core/src/physical_plan/sort_merge_join.rs
index 39ff4c2ec..4eb1616c9 100644
--- a/datafusion/core/src/physical_plan/sort_merge_join.rs
+++ b/datafusion/core/src/physical_plan/sort_merge_join.rs
@@ -353,7 +353,7 @@ impl StreamedBatch {
&mut self,
buffered_batch_idx: Option<usize>,
buffered_idx: Option<usize>,
- ) -> ArrowResult<()> {
+ ) {
if self.output_indices.is_empty() || self.buffered_batch_idx != buffered_batch_idx
{
self.output_indices.push(StreamedJoinedChunk {
@@ -365,16 +365,12 @@ impl StreamedBatch {
};
let current_chunk = self.output_indices.last_mut().unwrap();
- current_chunk
- .streamed_indices
- .append_value(self.idx as u64)?;
+ current_chunk.streamed_indices.append_value(self.idx as u64);
if let Some(idx) = buffered_idx {
- current_chunk.buffered_indices.append_value(idx as u64)?;
+ current_chunk.buffered_indices.append_value(idx as u64);
} else {
- current_chunk.buffered_indices.append_null()?;
+ current_chunk.buffered_indices.append_null();
}
-
- Ok(())
}
}
@@ -808,7 +804,7 @@ impl SMJStream {
self.streamed_batch.append_output_pair(
Some(self.buffered_data.scanning_batch_idx),
Some(scanning_idx),
- )?;
+ );
} else {
self.buffered_data
.scanning_batch_mut()
@@ -832,7 +828,7 @@ impl SMJStream {
};
self.streamed_batch
- .append_output_pair(scanning_batch_idx, None)?;
+ .append_output_pair(scanning_batch_idx, None);
self.output_size += 1;
self.buffered_data.scanning_finish();
self.streamed_joined = true;
@@ -1102,7 +1098,7 @@ fn compare_join_arrays(
DataType::Float64 => compare_value!(Float64Array),
DataType::Utf8 => compare_value!(StringArray),
DataType::LargeUtf8 => compare_value!(LargeStringArray),
- DataType::Decimal(..) => compare_value!(DecimalArray),
+ DataType::Decimal(..) => compare_value!(Decimal128Array),
DataType::Timestamp(time_unit, None) => match time_unit {
TimeUnit::Second => compare_value!(TimestampSecondArray),
TimeUnit::Millisecond => compare_value!(TimestampMillisecondArray),
@@ -1168,7 +1164,7 @@ fn is_join_arrays_equal(
DataType::Float64 => compare_value!(Float64Array),
DataType::Utf8 => compare_value!(StringArray),
DataType::LargeUtf8 => compare_value!(LargeStringArray),
- DataType::Decimal(..) => compare_value!(DecimalArray),
+ DataType::Decimal(..) => compare_value!(Decimal128Array),
DataType::Timestamp(time_unit, None) => match time_unit {
TimeUnit::Second => compare_value!(TimestampSecondArray),
TimeUnit::Millisecond => compare_value!(TimestampMillisecondArray),
diff --git a/datafusion/core/src/test/mod.rs b/datafusion/core/src/test/mod.rs
index 598d81879..6c107f669 100644
--- a/datafusion/core/src/test/mod.rs
+++ b/datafusion/core/src/test/mod.rs
@@ -27,7 +27,7 @@ use crate::physical_plan::file_format::{CsvExec, FileScanConfig};
use crate::test::object_store::local_unpartitioned_file;
use crate::test_util::aggr_test_schema;
use array::{Array, ArrayRef};
-use arrow::array::{self, DecimalBuilder, Int32Array};
+use arrow::array::{self, Decimal128Builder, Int32Array};
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
use arrow::record_batch::RecordBatch;
use futures::{Future, FutureExt};
@@ -200,7 +200,7 @@ pub fn table_with_decimal() -> Arc<dyn TableProvider> {
}
fn make_decimal() -> RecordBatch {
- let mut decimal_builder = DecimalBuilder::new(20, 10, 3);
+ let mut decimal_builder = Decimal128Builder::new(20, 10, 3);
for i in 110000..110010 {
decimal_builder.append_value(i as i128).unwrap();
}
diff --git a/datafusion/core/tests/provider_filter_pushdown.rs b/datafusion/core/tests/provider_filter_pushdown.rs
index 9b9ba84d3..3ebfec996 100644
--- a/datafusion/core/tests/provider_filter_pushdown.rs
+++ b/datafusion/core/tests/provider_filter_pushdown.rs
@@ -36,7 +36,7 @@ use std::sync::Arc;
fn create_batch(value: i32, num_rows: usize) -> Result<RecordBatch> {
let mut builder = Int32Builder::new(num_rows);
for _ in 0..num_rows {
- builder.append_value(value)?;
+ builder.append_value(value);
}
Ok(RecordBatch::try_new(
diff --git a/datafusion/core/tests/sql/mod.rs b/datafusion/core/tests/sql/mod.rs
index 186584aeb..7f235b1ba 100644
--- a/datafusion/core/tests/sql/mod.rs
+++ b/datafusion/core/tests/sql/mod.rs
@@ -290,7 +290,7 @@ fn create_hashjoin_datatype_context() -> Result<SessionContext> {
None,
])),
Arc::new(
- DecimalArray::from_iter_values([123, 45600, 78900, -12312])
+ Decimal128Array::from_iter_values([123, 45600, 78900, -12312])
.with_precision_and_scale(5, 2)
.unwrap(),
),
@@ -323,7 +323,7 @@ fn create_hashjoin_datatype_context() -> Result<SessionContext> {
None,
])),
Arc::new(
- DecimalArray::from_iter_values([-12312, 10000000, 0, 78900])
+ Decimal128Array::from_iter_values([-12312, 10000000, 0, 78900])
.with_precision_and_scale(10, 2)
.unwrap(),
),
@@ -571,25 +571,25 @@ async fn register_tpch_csv_data(
match field.data_type() {
DataType::Utf8 => {
let sb = col.as_any_mut().downcast_mut::<StringBuilder>().unwrap();
- sb.append_value(val)?;
+ sb.append_value(val);
}
DataType::Date32 => {
let sb = col.as_any_mut().downcast_mut::<Date32Builder>().unwrap();
let dt = NaiveDate::parse_from_str(val.trim(), "%Y-%m-%d").unwrap();
let dt = dt.sub(NaiveDate::from_ymd(1970, 1, 1)).num_days() as i32;
- sb.append_value(dt)?;
+ sb.append_value(dt);
}
DataType::Int32 => {
let sb = col.as_any_mut().downcast_mut::<Int32Builder>().unwrap();
- sb.append_value(val.trim().parse().unwrap())?;
+ sb.append_value(val.trim().parse().unwrap());
}
DataType::Int64 => {
let sb = col.as_any_mut().downcast_mut::<Int64Builder>().unwrap();
- sb.append_value(val.trim().parse().unwrap())?;
+ sb.append_value(val.trim().parse().unwrap());
}
DataType::Float64 => {
let sb = col.as_any_mut().downcast_mut::<Float64Builder>().unwrap();
- sb.append_value(val.trim().parse().unwrap())?;
+ sb.append_value(val.trim().parse().unwrap());
}
_ => Err(DataFusionError::Plan(format!(
"Not implemented: {}",
@@ -827,7 +827,7 @@ pub fn table_with_decimal() -> Arc<dyn TableProvider> {
}
fn make_decimal() -> RecordBatch {
- let mut decimal_builder = DecimalBuilder::new(20, 10, 3);
+ let mut decimal_builder = Decimal128Builder::new(20, 10, 3);
for i in 110000..110010 {
decimal_builder.append_value(i as i128).unwrap();
}
diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs
index 9119e21e8..5056fa02f 100644
--- a/datafusion/core/tests/sql/select.rs
+++ b/datafusion/core/tests/sql/select.rs
@@ -517,9 +517,9 @@ async fn query_get_indexed_field() -> Result<()> {
for int_vec in vec![vec![0, 1, 2], vec![4, 5, 6], vec![7, 8, 9]] {
let builder = lb.values();
for int in int_vec {
- builder.append_value(int).unwrap();
+ builder.append_value(int);
}
- lb.append(true).unwrap();
+ lb.append(true);
}
let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(lb.finish())])?;
@@ -568,11 +568,11 @@ async fn query_nested_get_indexed_field() -> Result<()> {
for int_vec in int_vec_vec {
let builder = nested_builder.values();
for int in int_vec {
- builder.append_value(int).unwrap();
+ builder.append_value(int);
}
- nested_builder.append(true).unwrap();
+ nested_builder.append(true);
}
- lb.append(true).unwrap();
+ lb.append(true);
}
let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(lb.finish())])?;
@@ -628,9 +628,9 @@ async fn query_nested_get_indexed_field_on_struct() -> Result<()> {
for int_vec in vec![vec![0, 1, 2, 3], vec![4, 5, 6, 7], vec![8, 9, 10, 11]] {
let lb = sb.field_builder::<ListBuilder<Int64Builder>>(0).unwrap();
for int in int_vec {
- lb.values().append_value(int).unwrap();
+ lb.values().append_value(int);
}
- lb.append(true).unwrap();
+ lb.append(true);
}
let data = RecordBatch::try_new(schema.clone(), vec![Arc::new(sb.finish())])?;
let table = MemTable::try_new(schema, vec![vec![data]])?;
diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml
index ce65717d2..b91e9c470 100644
--- a/datafusion/expr/Cargo.toml
+++ b/datafusion/expr/Cargo.toml
@@ -36,6 +36,6 @@ path = "src/lib.rs"
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "10.0.0" }
sqlparser = "0.18"
diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml
index eb539cbfc..e30895a4d 100644
--- a/datafusion/jit/Cargo.toml
+++ b/datafusion/jit/Cargo.toml
@@ -36,7 +36,7 @@ path = "src/lib.rs"
jit = []
[dependencies]
-arrow = { version = "18.0.0" }
+arrow = { version = "19.0.0" }
cranelift = "0.86.1"
cranelift-jit = "0.86.1"
cranelift-module = "0.86.1"
diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml
index 24d2f1812..58c6b2d0d 100644
--- a/datafusion/optimizer/Cargo.toml
+++ b/datafusion/optimizer/Cargo.toml
@@ -37,7 +37,7 @@ default = ["unicode_expressions"]
unicode_expressions = []
[dependencies]
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
async-trait = "0.1.41"
chrono = { version = "0.4", default-features = false }
datafusion-common = { path = "../common", version = "10.0.0" }
diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml
index e5abf12af..e32422f98 100644
--- a/datafusion/physical-expr/Cargo.toml
+++ b/datafusion/physical-expr/Cargo.toml
@@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"]
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
blake2 = { version = "^0.10.2", optional = true }
blake3 = { version = "1.0", optional = true }
chrono = { version = "0.4", default-features = false }
diff --git a/datafusion/physical-expr/src/aggregate/average.rs b/datafusion/physical-expr/src/aggregate/average.rs
index 3eee84bb5..1b1d99525 100644
--- a/datafusion/physical-expr/src/aggregate/average.rs
+++ b/datafusion/physical-expr/src/aggregate/average.rs
@@ -292,7 +292,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..7)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
@@ -310,7 +310,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
@@ -328,7 +328,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(6)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
diff --git a/datafusion/physical-expr/src/aggregate/count_distinct.rs b/datafusion/physical-expr/src/aggregate/count_distinct.rs
index 83c50b4cd..744d9b90d 100644
--- a/datafusion/physical-expr/src/aggregate/count_distinct.rs
+++ b/datafusion/physical-expr/src/aggregate/count_distinct.rs
@@ -267,22 +267,20 @@ mod tests {
Some(values) => {
for value in values.iter() {
match value {
- Some(v) => builder.values().append_value((*v).into())?,
- None => builder.values().append_null()?,
+ Some(v) => builder.values().append_value((*v).into()),
+ None => builder.values().append_null(),
}
}
- builder.append(true)?;
+ builder.append(true);
}
None => {
- builder.append(false)?;
+ builder.append(false);
}
}
}
- let array = Arc::new(builder.finish()) as ArrayRef;
-
- Ok(array) as Result<ArrayRef>
+ Arc::new(builder.finish()) as ArrayRef
}};
}
@@ -696,7 +694,7 @@ mod tests {
Some(vec![Some(-2_i32), Some(-3_i32)]),
],
Int32Builder
- )?;
+ );
let state_in2 = build_list!(
vec![
@@ -704,7 +702,7 @@ mod tests {
Some(vec![Some(5_u64), Some(7_u64)]),
],
UInt64Builder
- )?;
+ );
let (states, result) = run_merge_batch(&[state_in1, state_in2])?;
diff --git a/datafusion/physical-expr/src/aggregate/min_max.rs b/datafusion/physical-expr/src/aggregate/min_max.rs
index 811aa473c..bd56973b1 100644
--- a/datafusion/physical-expr/src/aggregate/min_max.rs
+++ b/datafusion/physical-expr/src/aggregate/min_max.rs
@@ -41,7 +41,7 @@ use datafusion_expr::Accumulator;
use crate::aggregate::row_accumulator::RowAccumulator;
use crate::expressions::format_state_name;
use arrow::array::Array;
-use arrow::array::DecimalArray;
+use arrow::array::Decimal128Array;
use datafusion_row::accessor::RowAccessor;
// Min/max aggregation can take Dictionary encode input but always produces unpacked
@@ -176,7 +176,7 @@ macro_rules! typed_min_max_batch_decimal128 {
if null_count == $VALUES.len() {
ScalarValue::Decimal128(None, *$PRECISION, *$SCALE)
} else {
- let array = $VALUES.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = $VALUES.as_any().downcast_ref::<Decimal128Array>().unwrap();
if null_count == 0 {
// there is no null value
let mut result = array.value(0);
@@ -777,7 +777,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
@@ -788,7 +788,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(0)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
let result = min_batch(&array)?;
@@ -798,7 +798,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
@@ -816,7 +816,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(6)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
@@ -834,7 +834,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
@@ -867,7 +867,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 5)?,
);
let result = max_batch(&array)?;
@@ -877,7 +877,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(0)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
let result = max_batch(&array)?;
@@ -887,7 +887,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
@@ -904,7 +904,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
@@ -921,7 +921,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(6)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
generic_test_op!(
diff --git a/datafusion/physical-expr/src/aggregate/sum.rs b/datafusion/physical-expr/src/aggregate/sum.rs
index ee8b224fe..866e90f1e 100644
--- a/datafusion/physical-expr/src/aggregate/sum.rs
+++ b/datafusion/physical-expr/src/aggregate/sum.rs
@@ -37,7 +37,7 @@ use datafusion_expr::Accumulator;
use crate::aggregate::row_accumulator::RowAccumulator;
use crate::expressions::format_state_name;
use arrow::array::Array;
-use arrow::array::DecimalArray;
+use arrow::array::Decimal128Array;
use arrow::compute::cast;
use datafusion_row::accessor::RowAccessor;
@@ -157,7 +157,7 @@ fn sum_decimal_batch(
precision: &usize,
scale: &usize,
) -> Result<ScalarValue> {
- let array = values.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = values.as_any().downcast_ref::<Decimal128Array>().unwrap();
if array.null_count() == array.len() {
return Ok(ScalarValue::Decimal128(None, *precision, *scale));
@@ -541,7 +541,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
let result = sum_batch(&array, &DataType::Decimal(10, 0))?;
@@ -551,7 +551,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(Some)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
@@ -576,7 +576,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
let result = sum_batch(&array, &DataType::Decimal(10, 0))?;
@@ -586,7 +586,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(35, 0)?,
);
generic_test_op!(
@@ -610,7 +610,7 @@ mod tests {
let array: ArrayRef = Arc::new(
std::iter::repeat(None)
.take(6)
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?,
);
let result = sum_batch(&array, &DataType::Decimal(10, 0))?;
diff --git a/datafusion/physical-expr/src/aggregate/sum_distinct.rs b/datafusion/physical-expr/src/aggregate/sum_distinct.rs
index ddca35775..a64b4b497 100644
--- a/datafusion/physical-expr/src/aggregate/sum_distinct.rs
+++ b/datafusion/physical-expr/src/aggregate/sum_distinct.rs
@@ -283,7 +283,7 @@ mod tests {
let array: ArrayRef = Arc::new(
(1..6)
.map(|i| if i == 2 { None } else { Some(i % 2) })
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(35, 0)?,
);
generic_test_sum_distinct!(
diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs
index 45ec881ba..84e6732e3 100644
--- a/datafusion/physical-expr/src/array_expressions.rs
+++ b/datafusion/physical-expr/src/array_expressions.rs
@@ -48,14 +48,14 @@ macro_rules! array {
for index in 0..args[0].len() {
for arg in &args {
if arg.is_null(index) {
- builder.values().append_null()?;
+ builder.values().append_null();
} else {
- builder.values().append_value(arg.value(index))?;
+ builder.values().append_value(arg.value(index));
}
}
- builder.append(true)?;
+ builder.append(true);
}
- Ok(Arc::new(builder.finish()))
+ Arc::new(builder.finish())
}};
}
@@ -67,7 +67,7 @@ fn array_array(args: &[ArrayRef]) -> Result<ArrayRef> {
));
}
- match args[0].data_type() {
+ let res = match args[0].data_type() {
DataType::Utf8 => array!(args, StringArray, StringBuilder),
DataType::LargeUtf8 => array!(args, LargeStringArray, LargeStringBuilder),
DataType::Boolean => array!(args, BooleanArray, BooleanBuilder),
@@ -81,11 +81,14 @@ fn array_array(args: &[ArrayRef]) -> Result<ArrayRef> {
DataType::UInt16 => array!(args, UInt16Array, UInt16Builder),
DataType::UInt32 => array!(args, UInt32Array, UInt32Builder),
DataType::UInt64 => array!(args, UInt64Array, UInt64Builder),
- data_type => Err(DataFusionError::NotImplemented(format!(
- "Array is not implemented for type '{:?}'.",
- data_type
- ))),
- }
+ data_type => {
+ return Err(DataFusionError::NotImplemented(format!(
+ "Array is not implemented for type '{:?}'.",
+ data_type
+ )))
+ }
+ };
+ Ok(res)
}
/// put values in an array.
diff --git a/datafusion/physical-expr/src/datetime_expressions.rs b/datafusion/physical-expr/src/datetime_expressions.rs
index 1e1cfd07f..eba3eca03 100644
--- a/datafusion/physical-expr/src/datetime_expressions.rs
+++ b/datafusion/physical-expr/src/datetime_expressions.rs
@@ -387,11 +387,11 @@ mod tests {
let mut string_builder = StringBuilder::new(2);
let mut ts_builder = TimestampNanosecondArray::builder(2);
- string_builder.append_value("2020-09-08T13:42:29.190855Z")?;
- ts_builder.append_value(1599572549190855000)?;
+ string_builder.append_value("2020-09-08T13:42:29.190855Z");
+ ts_builder.append_value(1599572549190855000);
- string_builder.append_null()?;
- ts_builder.append_null()?;
+ string_builder.append_null();
+ ts_builder.append_null();
let expected_timestamps = &ts_builder.finish() as &dyn Array;
let string_array =
@@ -508,7 +508,7 @@ mod tests {
// that we get an error.
let mut builder = Int64Array::builder(1);
- builder.append_value(1)?;
+ builder.append_value(1);
let int64array = ColumnarValue::Array(Arc::new(builder.finish()));
let expected_err =
diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs
index 93bcfae2e..b7492445b 100644
--- a/datafusion/physical-expr/src/expressions/binary.rs
+++ b/datafusion/physical-expr/src/expressions/binary.rs
@@ -123,7 +123,7 @@ fn is_not_distinct_from_bool(
/// Creates an BooleanArray the same size as `left`,
/// applying `op` to all non-null elements of left
fn compare_decimal_scalar<F>(
- left: &DecimalArray,
+ left: &Decimal128Array,
right: i128,
op: F,
) -> Result<BooleanArray>
@@ -139,8 +139,8 @@ where
/// Creates an BooleanArray the same size as `left`,
/// by applying `op` to all non-null elements of left and right
fn compare_decimal<F>(
- left: &DecimalArray,
- right: &DecimalArray,
+ left: &Decimal128Array,
+ right: &Decimal128Array,
op: F,
) -> Result<BooleanArray>
where
@@ -160,62 +160,68 @@ where
}
pub(super) fn eq_decimal_scalar(
- left: &DecimalArray,
+ left: &Decimal128Array,
right: i128,
) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left == right)
}
pub(super) fn eq_decimal(
- left: &DecimalArray,
- right: &DecimalArray,
+ left: &Decimal128Array,
+ right: &Decimal128Array,
) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left == right)
}
-fn neq_decimal_scalar(left: &DecimalArray, right: i128) -> Result<BooleanArray> {
+fn neq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left != right)
}
-fn neq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<BooleanArray> {
+fn neq_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left != right)
}
-fn lt_decimal_scalar(left: &DecimalArray, right: i128) -> Result<BooleanArray> {
+fn lt_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left < right)
}
-fn lt_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<BooleanArray> {
+fn lt_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left < right)
}
-fn lt_eq_decimal_scalar(left: &DecimalArray, right: i128) -> Result<BooleanArray> {
+fn lt_eq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left <= right)
}
-fn lt_eq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<BooleanArray> {
+fn lt_eq_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left <= right)
}
-fn gt_decimal_scalar(left: &DecimalArray, right: i128) -> Result<BooleanArray> {
+fn gt_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left > right)
}
-fn gt_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<BooleanArray> {
+fn gt_decimal(left: &Decimal128Array, right: &Decimal128Array) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left > right)
}
-fn gt_eq_decimal_scalar(left: &DecimalArray, right: i128) -> Result<BooleanArray> {
+fn gt_eq_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<BooleanArray> {
compare_decimal_scalar(left, right, |left, right| left >= right)
}
-fn gt_eq_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<BooleanArray> {
+fn gt_eq_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<BooleanArray> {
compare_decimal(left, right, |left, right| left >= right)
}
fn is_distinct_from_decimal(
- left: &DecimalArray,
- right: &DecimalArray,
+ left: &Decimal128Array,
+ right: &Decimal128Array,
) -> Result<BooleanArray> {
Ok(left
.iter()
@@ -229,8 +235,8 @@ fn is_distinct_from_decimal(
}
fn is_not_distinct_from_decimal(
- left: &DecimalArray,
- right: &DecimalArray,
+ left: &Decimal128Array,
+ right: &Decimal128Array,
) -> Result<BooleanArray> {
Ok(left
.iter()
@@ -243,13 +249,13 @@ fn is_not_distinct_from_decimal(
.collect())
}
-/// Creates an DecimalArray the same size as `left`,
+/// Creates an Decimal128Array the same size as `left`,
/// by applying `op` to all non-null elements of left and right
fn arith_decimal<F>(
- left: &DecimalArray,
- right: &DecimalArray,
+ left: &Decimal128Array,
+ right: &Decimal128Array,
op: F,
-) -> Result<DecimalArray>
+) -> Result<Decimal128Array>
where
F: Fn(i128, i128) -> Result<i128>,
{
@@ -266,10 +272,10 @@ where
}
fn arith_decimal_scalar<F>(
- left: &DecimalArray,
+ left: &Decimal128Array,
right: i128,
op: F,
-) -> Result<DecimalArray>
+) -> Result<Decimal128Array>
where
F: Fn(i128, i128) -> Result<i128>,
{
@@ -284,38 +290,53 @@ where
.collect()
}
-fn add_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalArray> {
+fn add_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<Decimal128Array> {
let array = arith_decimal(left, right, |left, right| Ok(left + right))?
.with_precision_and_scale(left.precision(), left.scale())?;
Ok(array)
}
-fn add_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArray> {
+fn add_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<Decimal128Array> {
let array = arith_decimal_scalar(left, right, |left, right| Ok(left + right))?
.with_precision_and_scale(left.precision(), left.scale())?;
Ok(array)
}
-fn subtract_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalArray> {
+fn subtract_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<Decimal128Array> {
let array = arith_decimal(left, right, |left, right| Ok(left - right))?
.with_precision_and_scale(left.precision(), left.scale())?;
Ok(array)
}
-fn subtract_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArray> {
+fn subtract_decimal_scalar(
+ left: &Decimal128Array,
+ right: i128,
+) -> Result<Decimal128Array> {
let array = arith_decimal_scalar(left, right, |left, right| Ok(left - right))?
.with_precision_and_scale(left.precision(), left.scale())?;
Ok(array)
}
-fn multiply_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalArray> {
+fn multiply_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<Decimal128Array> {
let divide = 10_i128.pow(left.scale() as u32);
let array = arith_decimal(left, right, |left, right| Ok(left * right / divide))?
.with_precision_and_scale(left.precision(), left.scale())?;
Ok(array)
}
-fn multiply_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArray> {
+fn multiply_decimal_scalar(
+ left: &Decimal128Array,
+ right: i128,
+) -> Result<Decimal128Array> {
let divide = 10_i128.pow(left.scale() as u32);
let array =
arith_decimal_scalar(left, right, |left, right| Ok(left * right / divide))?
@@ -323,7 +344,10 @@ fn multiply_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalAr
Ok(array)
}
-fn divide_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalArray> {
+fn divide_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<Decimal128Array> {
let mul = 10_f64.powi(left.scale() as i32);
let array = arith_decimal(left, right, |left, right| {
let l_value = left as f64;
@@ -335,7 +359,7 @@ fn divide_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalAr
Ok(array)
}
-fn divide_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArray> {
+fn divide_decimal_scalar(left: &Decimal128Array, right: i128) -> Result<Decimal128Array> {
let mul = 10_f64.powi(left.scale() as i32);
let array = arith_decimal_scalar(left, right, |left, right| {
let l_value = left as f64;
@@ -347,7 +371,10 @@ fn divide_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArra
Ok(array)
}
-fn modulus_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalArray> {
+fn modulus_decimal(
+ left: &Decimal128Array,
+ right: &Decimal128Array,
+) -> Result<Decimal128Array> {
let array = arith_decimal(left, right, |left, right| {
if right == 0 {
Err(DataFusionError::ArrowError(DivideByZero))
@@ -359,7 +386,10 @@ fn modulus_decimal(left: &DecimalArray, right: &DecimalArray) -> Result<DecimalA
Ok(array)
}
-fn modulus_decimal_scalar(left: &DecimalArray, right: i128) -> Result<DecimalArray> {
+fn modulus_decimal_scalar(
+ left: &Decimal128Array,
+ right: i128,
+) -> Result<Decimal128Array> {
if right == 0 {
return Err(DataFusionError::ArrowError(DivideByZero));
}
@@ -794,7 +824,7 @@ macro_rules! binary_primitive_array_op {
match $LEFT.data_type() {
// TODO support decimal type
// which is not the primitive type
- DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, DecimalArray),
+ DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, Decimal128Array),
DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
@@ -819,7 +849,7 @@ macro_rules! binary_primitive_array_op {
macro_rules! binary_primitive_array_op_scalar {
($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
let result: Result<Arc<dyn Array>> = match $LEFT.data_type() {
- DataType::Decimal(_,_) => compute_decimal_op_scalar!($LEFT, $RIGHT, $OP, DecimalArray),
+ DataType::Decimal(_,_) => compute_decimal_op_scalar!($LEFT, $RIGHT, $OP, Decimal128Array),
DataType::Int8 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int8Array),
DataType::Int16 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int16Array),
DataType::Int32 => compute_op_scalar!($LEFT, $RIGHT, $OP, Int32Array),
@@ -846,7 +876,7 @@ macro_rules! binary_array_op {
($LEFT:expr, $RIGHT:expr, $OP:ident) => {{
match $LEFT.data_type() {
DataType::Null => compute_null_op!($LEFT, $RIGHT, $OP, NullArray),
- DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, DecimalArray),
+ DataType::Decimal(_,_) => compute_decimal_op!($LEFT, $RIGHT, $OP, Decimal128Array),
DataType::Int8 => compute_op!($LEFT, $RIGHT, $OP, Int8Array),
DataType::Int16 => compute_op!($LEFT, $RIGHT, $OP, Int16Array),
DataType::Int32 => compute_op!($LEFT, $RIGHT, $OP, Int32Array),
@@ -1079,7 +1109,7 @@ macro_rules! binary_array_op_dyn_scalar {
let result: Result<Arc<dyn Array>> = match right {
ScalarValue::Boolean(b) => compute_bool_op_dyn_scalar!($LEFT, b, $OP, $OP_TYPE),
- ScalarValue::Decimal128(..) => compute_decimal_op_scalar!($LEFT, right, $OP, DecimalArray),
+ ScalarValue::Decimal128(..) => compute_decimal_op_scalar!($LEFT, right, $OP, Decimal128Array),
ScalarValue::Utf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
ScalarValue::LargeUtf8(v) => compute_utf8_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
ScalarValue::Int8(v) => compute_op_dyn_scalar!($LEFT, v, $OP, $OP_TYPE),
@@ -1809,7 +1839,7 @@ mod tests {
let mut dict_builder = StringDictionaryBuilder::new(keys_builder, values_builder);
dict_builder.append("one")?;
- dict_builder.append_null()?;
+ dict_builder.append_null();
dict_builder.append("three")?;
dict_builder.append("four")?;
let dict_array = dict_builder.finish();
@@ -2629,19 +2659,19 @@ mod tests {
array: &[Option<i128>],
precision: usize,
scale: usize,
- ) -> Result<DecimalArray> {
- let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale);
+ ) -> Decimal128Array {
+ let mut decimal_builder = Decimal128Builder::new(array.len(), precision, scale);
for value in array {
match value {
None => {
- decimal_builder.append_null()?;
+ decimal_builder.append_null();
}
Some(v) => {
- decimal_builder.append_value(*v)?;
+ decimal_builder.append_value(*v).expect("valid value");
}
}
}
- Ok(decimal_builder.finish())
+ decimal_builder.finish()
}
#[test]
@@ -2656,7 +2686,7 @@ mod tests {
],
25,
3,
- )?;
+ );
// eq: array = i128
let result = eq_decimal_scalar(&decimal_array, value_i128)?;
assert_eq!(
@@ -2704,7 +2734,7 @@ mod tests {
],
25,
3,
- )?;
+ );
// eq: left == right
let result = eq_decimal(&left_decimal_array, &right_decimal_array)?;
assert_eq!(
@@ -2860,7 +2890,7 @@ mod tests {
],
10,
0,
- )?) as ArrayRef;
+ )) as ArrayRef;
let int64_array = Arc::new(Int64Array::from(vec![
Some(value),
@@ -2903,7 +2933,7 @@ mod tests {
],
10,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
let float64_array = Arc::new(Float64Array::from(vec![
Some(1.23),
Some(1.22),
@@ -2982,7 +3012,7 @@ mod tests {
],
25,
3,
- )?;
+ );
let right_decimal_array = create_decimal_array(
&[
Some(value_i128),
@@ -2992,59 +3022,59 @@ mod tests {
],
25,
3,
- )?;
+ );
// add
let result = add_decimal(&left_decimal_array, &right_decimal_array)?;
let expect =
- create_decimal_array(&[Some(246), None, Some(245), Some(247)], 25, 3)?;
+ create_decimal_array(&[Some(246), None, Some(245), Some(247)], 25, 3);
assert_eq!(expect, result);
let result = add_decimal_scalar(&left_decimal_array, 10)?;
let expect =
- create_decimal_array(&[Some(133), None, Some(132), Some(134)], 25, 3)?;
+ create_decimal_array(&[Some(133), None, Some(132), Some(134)], 25, 3);
assert_eq!(expect, result);
// subtract
let result = subtract_decimal(&left_decimal_array, &right_decimal_array)?;
- let expect = create_decimal_array(&[Some(0), None, Some(-1), Some(1)], 25, 3)?;
+ let expect = create_decimal_array(&[Some(0), None, Some(-1), Some(1)], 25, 3);
assert_eq!(expect, result);
let result = subtract_decimal_scalar(&left_decimal_array, 10)?;
let expect =
- create_decimal_array(&[Some(113), None, Some(112), Some(114)], 25, 3)?;
+ create_decimal_array(&[Some(113), None, Some(112), Some(114)], 25, 3);
assert_eq!(expect, result);
// multiply
let result = multiply_decimal(&left_decimal_array, &right_decimal_array)?;
- let expect = create_decimal_array(&[Some(15), None, Some(15), Some(15)], 25, 3)?;
+ let expect = create_decimal_array(&[Some(15), None, Some(15), Some(15)], 25, 3);
assert_eq!(expect, result);
let result = multiply_decimal_scalar(&left_decimal_array, 10)?;
- let expect = create_decimal_array(&[Some(1), None, Some(1), Some(1)], 25, 3)?;
+ let expect = create_decimal_array(&[Some(1), None, Some(1), Some(1)], 25, 3);
assert_eq!(expect, result);
// divide
let left_decimal_array = create_decimal_array(
&[Some(1234567), None, Some(1234567), Some(1234567)],
25,
3,
- )?;
+ );
let right_decimal_array =
- create_decimal_array(&[Some(10), Some(100), Some(55), Some(-123)], 25, 3)?;
+ create_decimal_array(&[Some(10), Some(100), Some(55), Some(-123)], 25, 3);
let result = divide_decimal(&left_decimal_array, &right_decimal_array)?;
let expect = create_decimal_array(
&[Some(123456700), None, Some(22446672), Some(-10037130)],
25,
3,
- )?;
+ );
assert_eq!(expect, result);
let result = divide_decimal_scalar(&left_decimal_array, 10)?;
let expect = create_decimal_array(
&[Some(123456700), None, Some(123456700), Some(123456700)],
25,
3,
- )?;
+ );
assert_eq!(expect, result);
// modulus
let result = modulus_decimal(&left_decimal_array, &right_decimal_array)?;
- let expect = create_decimal_array(&[Some(7), None, Some(37), Some(16)], 25, 3)?;
+ let expect = create_decimal_array(&[Some(7), None, Some(37), Some(16)], 25, 3);
assert_eq!(expect, result);
let result = modulus_decimal_scalar(&left_decimal_array, 10)?;
- let expect = create_decimal_array(&[Some(7), None, Some(7), Some(7)], 25, 3)?;
+ let expect = create_decimal_array(&[Some(7), None, Some(7), Some(7)], 25, 3);
assert_eq!(expect, result);
Ok(())
@@ -3083,7 +3113,7 @@ mod tests {
],
10,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
let int32_array = Arc::new(Int32Array::from(vec![
Some(123),
Some(122),
@@ -3096,7 +3126,7 @@ mod tests {
&[Some(12423), None, Some(12422), Some(12524)],
13,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
apply_arithmetic_op(
&schema,
&int32_array,
@@ -3115,7 +3145,7 @@ mod tests {
&[Some(-12177), None, Some(-12178), Some(-12276)],
13,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
apply_arithmetic_op(
&schema,
&int32_array,
@@ -3130,7 +3160,7 @@ mod tests {
&[Some(15129), None, Some(15006), Some(15376)],
21,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
apply_arithmetic_op(
&schema,
&int32_array,
@@ -3153,7 +3183,7 @@ mod tests {
],
23,
11,
- )?) as ArrayRef;
+ )) as ArrayRef;
apply_arithmetic_op(
&schema,
&int32_array,
@@ -3171,7 +3201,7 @@ mod tests {
&[Some(000), None, Some(100), Some(000)],
10,
2,
- )?) as ArrayRef;
+ )) as ArrayRef;
apply_arithmetic_op(
&schema,
&int32_array,
diff --git a/datafusion/physical-expr/src/expressions/cast.rs b/datafusion/physical-expr/src/expressions/cast.rs
index de1909676..72503bbdb 100644
--- a/datafusion/physical-expr/src/expressions/cast.rs
+++ b/datafusion/physical-expr/src/expressions/cast.rs
@@ -161,7 +161,7 @@ mod tests {
use crate::expressions::col;
use arrow::{
array::{
- Array, BasicDecimalArray, DecimalArray, Float32Array, Float64Array,
+ Array, BasicDecimalArray, Decimal128Array, Float32Array, Float64Array,
Int16Array, Int32Array, Int64Array, Int8Array, StringArray,
Time64NanosecondArray, TimestampNanosecondArray, UInt32Array,
},
@@ -271,12 +271,18 @@ mod tests {
#[test]
fn test_cast_decimal_to_decimal() -> Result<()> {
- let array = vec![1234, 2222, 3, 4000, 5000];
+ let array = vec![
+ Some(1234),
+ Some(2222),
+ Some(3),
+ Some(4000),
+ Some(5000),
+ None,
+ ];
let decimal_array = array
.iter()
- .map(|v| Some(*v))
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 3)?;
// closure that converts to i128
@@ -285,7 +291,7 @@ mod tests {
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 3),
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 6),
vec![
Some(convert(1_234_000)),
@@ -300,15 +306,14 @@ mod tests {
let decimal_array = array
.iter()
- .map(|v| Some(*v))
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 3)?;
let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 3),
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 2),
vec![
Some(convert(123)),
@@ -330,7 +335,7 @@ mod tests {
// decimal to i8
let decimal_array = array
.iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -351,7 +356,7 @@ mod tests {
// decimal to i16
let decimal_array = array
.iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -372,7 +377,7 @@ mod tests {
// decimal to i32
let decimal_array = array
.iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -393,7 +398,7 @@ mod tests {
// decimal to i64
let decimal_array = array
.iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 0)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -422,7 +427,7 @@ mod tests {
];
let decimal_array = array
.iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(10, 3)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -443,7 +448,7 @@ mod tests {
// decimal to float64
let decimal_array = array
.into_iter()
- .collect::<DecimalArray>()
+ .collect::<Decimal128Array>()
.with_precision_and_scale(20, 6)?;
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -471,7 +476,7 @@ mod tests {
Int8Array,
DataType::Int8,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(3, 0),
vec![
Some(convert(1)),
@@ -489,7 +494,7 @@ mod tests {
Int16Array,
DataType::Int16,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(5, 0),
vec![
Some(convert(1)),
@@ -507,7 +512,7 @@ mod tests {
Int32Array,
DataType::Int32,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 0),
vec![
Some(convert(1)),
@@ -525,7 +530,7 @@ mod tests {
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 0),
vec![
Some(convert(1)),
@@ -543,7 +548,7 @@ mod tests {
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 2),
vec![
Some(convert(100)),
@@ -561,7 +566,7 @@ mod tests {
Float32Array,
DataType::Float32,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 2),
vec![
Some(convert(150)),
@@ -579,7 +584,7 @@ mod tests {
Float64Array,
DataType::Float64,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 4),
vec![
Some(convert(15000)),
diff --git a/datafusion/physical-expr/src/expressions/datetime.rs b/datafusion/physical-expr/src/expressions/datetime.rs
index d4486a3ff..3c59a90b0 100644
--- a/datafusion/physical-expr/src/expressions/datetime.rs
+++ b/datafusion/physical-expr/src/expressions/datetime.rs
@@ -406,7 +406,7 @@ mod tests {
fn exercise(dt: &Expr, op: Operator, interval: &Expr) -> Result<ColumnarValue> {
let mut builder = Date32Builder::new(1);
- builder.append_value(0).unwrap();
+ builder.append_value(0);
let a: ArrayRef = Arc::new(builder.finish());
let schema = Schema::new(vec![Field::new("a", DataType::Date32, false)]);
let batch = RecordBatch::try_new(Arc::new(schema.clone()), vec![a])?;
diff --git a/datafusion/physical-expr/src/expressions/get_indexed_field.rs b/datafusion/physical-expr/src/expressions/get_indexed_field.rs
index bb43229d1..40216b455 100644
--- a/datafusion/physical-expr/src/expressions/get_indexed_field.rs
+++ b/datafusion/physical-expr/src/expressions/get_indexed_field.rs
@@ -155,9 +155,8 @@ mod tests {
None => builder.append_null(),
Some(v) => builder.append_value(v),
}
- .unwrap()
}
- lb.append(true).unwrap();
+ lb.append(true);
}
lb.finish()
@@ -271,9 +270,8 @@ mod tests {
match int_value {
None => fb.append_null(),
Some(v) => fb.append_value(v),
- }
- .unwrap();
- builder.append(true).unwrap();
+ };
+ builder.append(true);
let lb = builder
.field_builder::<ListBuilder<StringBuilder>>(1)
.unwrap();
@@ -281,10 +279,9 @@ mod tests {
match str_value {
None => lb.values().append_null(),
Some(v) => lb.values().append_value(v),
- }
- .unwrap();
+ };
}
- lb.append(true).unwrap();
+ lb.append(true);
}
builder.finish()
}
diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs
index 40a253c33..899a20835 100644
--- a/datafusion/physical-expr/src/expressions/in_list.rs
+++ b/datafusion/physical-expr/src/expressions/in_list.rs
@@ -292,7 +292,7 @@ fn try_cast_static_filter_to_set(
}
fn make_list_contains_decimal(
- array: &DecimalArray,
+ array: &Decimal128Array,
list: Vec<ColumnarValue>,
negated: bool,
) -> ColumnarValue {
@@ -319,7 +319,7 @@ fn make_list_contains_decimal(
}
fn make_set_contains_decimal(
- array: &DecimalArray,
+ array: &Decimal128Array,
set: &HashSet<ScalarValue>,
negated: bool,
) -> ColumnarValue {
@@ -632,7 +632,7 @@ impl PhysicalExpr for InListExpr {
Ok(set_contains_utf8(array, set, self.negated))
}
DataType::Decimal(_, _) => {
- let array = array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ let array = array.as_any().downcast_ref::<Decimal128Array>().unwrap();
Ok(make_set_contains_decimal(array, set, self.negated))
}
datatype => Result::Err(DataFusionError::NotImplemented(format!(
@@ -762,7 +762,7 @@ impl PhysicalExpr for InListExpr {
}
DataType::Decimal(_, _) => {
let decimal_array =
- array.as_any().downcast_ref::<DecimalArray>().unwrap();
+ array.as_any().downcast_ref::<Decimal128Array>().unwrap();
Ok(make_list_contains_decimal(
decimal_array,
list_values,
@@ -1035,7 +1035,7 @@ mod tests {
let schema = Schema::new(vec![Field::new("a", DataType::Decimal(13, 4), true)]);
let array = vec![Some(100_0000_i128), None, Some(200_5000_i128)]
.into_iter()
- .collect::<DecimalArray>();
+ .collect::<Decimal128Array>();
let array = array.with_precision_and_scale(13, 4).unwrap();
let col_a = col("a", &schema)?;
let batch =
@@ -1281,7 +1281,7 @@ mod tests {
let schema = Schema::new(vec![Field::new("a", DataType::Decimal(13, 4), true)]);
let array = vec![Some(100_0000_i128), Some(200_5000_i128), None]
.into_iter()
- .collect::<DecimalArray>();
+ .collect::<Decimal128Array>();
let array = array.with_precision_and_scale(13, 4).unwrap();
let col_a = col("a", &schema)?;
let batch =
diff --git a/datafusion/physical-expr/src/expressions/try_cast.rs b/datafusion/physical-expr/src/expressions/try_cast.rs
index 2509c1d6b..5e8cc30fe 100644
--- a/datafusion/physical-expr/src/expressions/try_cast.rs
+++ b/datafusion/physical-expr/src/expressions/try_cast.rs
@@ -119,7 +119,7 @@ mod tests {
use super::*;
use crate::expressions::col;
use arrow::array::{
- BasicDecimalArray, DecimalArray, DecimalBuilder, StringArray,
+ BasicDecimalArray, Decimal128Array, Decimal128Builder, StringArray,
Time64NanosecondArray,
};
use arrow::util::decimal::{BasicDecimal, Decimal128};
@@ -233,12 +233,12 @@ mod tests {
fn test_try_cast_decimal_to_decimal() -> Result<()> {
// try cast one decimal data type to another decimal data type
let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
- let decimal_array = create_decimal_array(&array, 10, 3)?;
+ let decimal_array = create_decimal_array(&array, 10, 3);
let convert = |v: i128| Decimal128::new(20, 6, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 3),
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 6),
vec![
Some(convert(1_234_000)),
@@ -250,12 +250,12 @@ mod tests {
]
);
- let decimal_array = create_decimal_array(&array, 10, 3)?;
+ let decimal_array = create_decimal_array(&array, 10, 3);
let convert = |v: i128| Decimal128::new(10, 2, &v.to_le_bytes());
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 3),
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 2),
vec![
Some(convert(123)),
@@ -272,10 +272,10 @@ mod tests {
#[test]
fn test_try_cast_decimal_to_numeric() -> Result<()> {
- // TODO we should add function to create DecimalArray with value and metadata
+ // TODO we should add function to create Decimal128Array with value and metadata
// https://github.com/apache/arrow-rs/issues/1009
let array: Vec<i128> = vec![1, 2, 3, 4, 5];
- let decimal_array = create_decimal_array(&array, 10, 0)?;
+ let decimal_array = create_decimal_array(&array, 10, 0);
// decimal to i8
generic_decimal_to_other_test_cast!(
decimal_array,
@@ -293,7 +293,7 @@ mod tests {
);
// decimal to i16
- let decimal_array = create_decimal_array(&array, 10, 0)?;
+ let decimal_array = create_decimal_array(&array, 10, 0);
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 0),
@@ -310,7 +310,7 @@ mod tests {
);
// decimal to i32
- let decimal_array = create_decimal_array(&array, 10, 0)?;
+ let decimal_array = create_decimal_array(&array, 10, 0);
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 0),
@@ -327,7 +327,7 @@ mod tests {
);
// decimal to i64
- let decimal_array = create_decimal_array(&array, 10, 0)?;
+ let decimal_array = create_decimal_array(&array, 10, 0);
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 0),
@@ -345,7 +345,7 @@ mod tests {
// decimal to float32
let array: Vec<i128> = vec![1234, 2222, 3, 4000, 5000];
- let decimal_array = create_decimal_array(&array, 10, 3)?;
+ let decimal_array = create_decimal_array(&array, 10, 3);
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(10, 3),
@@ -361,7 +361,7 @@ mod tests {
]
);
// decimal to float64
- let decimal_array = create_decimal_array(&array, 20, 6)?;
+ let decimal_array = create_decimal_array(&array, 20, 6);
generic_decimal_to_other_test_cast!(
decimal_array,
DataType::Decimal(20, 6),
@@ -388,7 +388,7 @@ mod tests {
Int8Array,
DataType::Int8,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(3, 0),
vec![
Some(convert(1)),
@@ -405,7 +405,7 @@ mod tests {
Int16Array,
DataType::Int16,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(5, 0),
vec![
Some(convert(1)),
@@ -422,7 +422,7 @@ mod tests {
Int32Array,
DataType::Int32,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 0),
vec![
Some(convert(1)),
@@ -439,7 +439,7 @@ mod tests {
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 0),
vec![
Some(convert(1)),
@@ -456,7 +456,7 @@ mod tests {
Int64Array,
DataType::Int64,
vec![1, 2, 3, 4, 5],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 2),
vec![
Some(convert(100)),
@@ -473,7 +473,7 @@ mod tests {
Float32Array,
DataType::Float32,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(10, 2),
vec![
Some(convert(150)),
@@ -490,7 +490,7 @@ mod tests {
Float64Array,
DataType::Float64,
vec![1.5, 2.5, 3.0, 1.123_456_8, 5.50],
- DecimalArray,
+ Decimal128Array,
DataType::Decimal(20, 4),
vec![
Some(convert(15000)),
@@ -581,12 +581,12 @@ mod tests {
array: &[i128],
precision: usize,
scale: usize,
- ) -> Result<DecimalArray> {
- let mut decimal_builder = DecimalBuilder::new(array.len(), precision, scale);
+ ) -> Decimal128Array {
+ let mut decimal_builder = Decimal128Builder::new(array.len(), precision, scale);
for value in array {
- decimal_builder.append_value(*value)?
+ decimal_builder.append_value(*value).expect("valid value");
}
- decimal_builder.append_null()?;
- Ok(decimal_builder.finish())
+ decimal_builder.append_null();
+ decimal_builder.finish()
}
}
diff --git a/datafusion/physical-expr/src/regex_expressions.rs b/datafusion/physical-expr/src/regex_expressions.rs
index c53c94615..2705d7ffb 100644
--- a/datafusion/physical-expr/src/regex_expressions.rs
+++ b/datafusion/physical-expr/src/regex_expressions.rs
@@ -191,13 +191,13 @@ mod tests {
let elem_builder: GenericStringBuilder<i32> = GenericStringBuilder::new(0);
let mut expected_builder = ListBuilder::new(elem_builder);
- expected_builder.values().append_value("a").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.append(false).unwrap();
- expected_builder.values().append_value("b").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.append(false).unwrap();
- expected_builder.append(false).unwrap();
+ expected_builder.values().append_value("a");
+ expected_builder.append(true);
+ expected_builder.append(false);
+ expected_builder.values().append_value("b");
+ expected_builder.append(true);
+ expected_builder.append(false);
+ expected_builder.append(false);
let expected = expected_builder.finish();
let re = regexp_match::<i32>(&[Arc::new(values), Arc::new(patterns)]).unwrap();
@@ -214,15 +214,15 @@ mod tests {
let elem_builder: GenericStringBuilder<i32> = GenericStringBuilder::new(0);
let mut expected_builder = ListBuilder::new(elem_builder);
- expected_builder.values().append_value("a").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.values().append_value("a").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.values().append_value("b").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.values().append_value("b").unwrap();
- expected_builder.append(true).unwrap();
- expected_builder.append(false).unwrap();
+ expected_builder.values().append_value("a");
+ expected_builder.append(true);
+ expected_builder.values().append_value("a");
+ expected_builder.append(true);
+ expected_builder.values().append_value("b");
+ expected_builder.append(true);
+ expected_builder.values().append_value("b");
+ expected_builder.append(true);
+ expected_builder.append(false);
let expected = expected_builder.finish();
let re =
diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml
index 5c3026374..f63aae728 100644
--- a/datafusion/proto/Cargo.toml
+++ b/datafusion/proto/Cargo.toml
@@ -37,7 +37,7 @@ default = []
json = ["pbjson", "pbjson-build", "serde", "serde_json"]
[dependencies]
-arrow = { version = "18.0.0" }
+arrow = { version = "19.0.0" }
datafusion = { path = "../core", version = "10.0.0" }
datafusion-common = { path = "../common", version = "10.0.0" }
datafusion-expr = { path = "../expr", version = "10.0.0" }
diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs
index fd5276ca8..60f4079da 100644
--- a/datafusion/proto/src/to_proto.rs
+++ b/datafusion/proto/src/to_proto.rs
@@ -223,6 +223,9 @@ impl From<&DataType> for protobuf::arrow_type::ArrowTypeEnum {
whole: *whole as u64,
fractional: *fractional as u64,
}),
+ DataType::Decimal256(_, _) => {
+ unimplemented!("The Decimal256 data type is not yet supported")
+ }
DataType::Map(_, _) => {
unimplemented!("The Map data type is not yet supported")
}
@@ -1237,7 +1240,8 @@ impl TryFrom<&DataType> for protobuf::scalar_type::Datatype {
| DataType::Union(_, _, _)
| DataType::Dictionary(_, _)
| DataType::Map(_, _)
- | DataType::Decimal(_, _) => {
+ | DataType::Decimal(_, _)
+ | DataType::Decimal256(_, _) => {
return Err(Error::invalid_scalar_type(val));
}
};
diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml
index f2a524d31..930969948 100644
--- a/datafusion/row/Cargo.toml
+++ b/datafusion/row/Cargo.toml
@@ -37,7 +37,7 @@ path = "src/lib.rs"
jit = ["datafusion-jit"]
[dependencies]
-arrow = { version = "18.0.0" }
+arrow = { version = "19.0.0" }
datafusion-common = { path = "../common", version = "10.0.0" }
datafusion-jit = { path = "../jit", version = "10.0.0", optional = true }
paste = "^1.0"
diff --git a/datafusion/row/src/reader.rs b/datafusion/row/src/reader.rs
index 7982ca120..634b814ad 100644
--- a/datafusion/row/src/reader.rs
+++ b/datafusion/row/src/reader.rs
@@ -300,9 +300,7 @@ macro_rules! fn_read_field {
.as_any_mut()
.downcast_mut::<$ARRAY>()
.unwrap();
- to.append_option(row.[<get_ $NATIVE _opt>](col_idx))
- .map_err(DataFusionError::ArrowError)
- .unwrap();
+ to.append_option(row.[<get_ $NATIVE _opt>](col_idx));
}
pub(crate) fn [<read_field_ $NATIVE _null_free>](to: &mut Box<dyn ArrayBuilder>, col_idx: usize, row: &RowReader) {
@@ -310,9 +308,7 @@ macro_rules! fn_read_field {
.as_any_mut()
.downcast_mut::<$ARRAY>()
.unwrap();
- to.append_value(row.[<get_ $NATIVE>](col_idx))
- .map_err(DataFusionError::ArrowError)
- .unwrap();
+ to.append_value(row.[<get_ $NATIVE>](col_idx));
}
}
};
@@ -340,9 +336,9 @@ pub(crate) fn read_field_binary(
) {
let to = to.as_any_mut().downcast_mut::<BinaryBuilder>().unwrap();
if row.is_valid_at(col_idx) {
- to.append_value(row.get_binary(col_idx)).unwrap();
+ to.append_value(row.get_binary(col_idx));
} else {
- to.append_null().unwrap();
+ to.append_null();
}
}
@@ -352,9 +348,7 @@ pub(crate) fn read_field_binary_null_free(
row: &RowReader,
) {
let to = to.as_any_mut().downcast_mut::<BinaryBuilder>().unwrap();
- to.append_value(row.get_binary(col_idx))
- .map_err(DataFusionError::ArrowError)
- .unwrap();
+ to.append_value(row.get_binary(col_idx));
}
fn read_field(
diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml
index c3854172f..db8720d3e 100644
--- a/datafusion/sql/Cargo.toml
+++ b/datafusion/sql/Cargo.toml
@@ -38,7 +38,7 @@ unicode_expressions = []
[dependencies]
ahash = { version = "0.7", default-features = false }
-arrow = { version = "18.0.0", features = ["prettyprint"] }
+arrow = { version = "19.0.0", features = ["prettyprint"] }
datafusion-common = { path = "../common", version = "10.0.0" }
datafusion-expr = { path = "../expr", version = "10.0.0" }
hashbrown = "0.12"