You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2022/10/14 21:28:55 UTC
[arrow-rs] branch master updated: Don't validate decimal precision in ArrayData (#2637) (#2873)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 4d7d411af Don't validate decimal precision in ArrayData (#2637) (#2873)
4d7d411af is described below
commit 4d7d411af340bb768afe3ebf590cd6c425dbc064
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Sat Oct 15 10:28:51 2022 +1300
Don't validate decimal precision in ArrayData (#2637) (#2873)
* Don't validate decimal precision in ArrayData (#2637)
* Format
---
arrow-data/src/data.rs | 19 -------------------
arrow/tests/array_validation.rs | 11 ++++++++---
2 files changed, 8 insertions(+), 22 deletions(-)
diff --git a/arrow-data/src/data.rs b/arrow-data/src/data.rs
index 37c059748..b53e9f0af 100644
--- a/arrow-data/src/data.rs
+++ b/arrow-data/src/data.rs
@@ -18,9 +18,6 @@
//! Contains `ArrayData`, a generic representation of Arrow array data which encapsulates
//! common attributes and operations for Arrow array.
-use crate::decimal::{
- validate_decimal256_precision_with_lt_bytes, validate_decimal_precision,
-};
use crate::{bit_iterator::BitSliceIterator, bitmap::Bitmap};
use arrow_buffer::{bit_util, ArrowNativeType, Buffer, MutableBuffer};
use arrow_schema::{ArrowError, DataType, IntervalUnit, UnionMode};
@@ -1004,22 +1001,6 @@ impl ArrayData {
pub fn validate_values(&self) -> Result<(), ArrowError> {
match &self.data_type {
- DataType::Decimal128(p, _) => {
- let values_buffer: &[i128] = self.typed_buffer(0, self.len)?;
- for value in values_buffer {
- validate_decimal_precision(*value, *p)?;
- }
- Ok(())
- }
- DataType::Decimal256(p, _) => {
- let values = self.buffers()[0].as_slice();
- for pos in 0..self.len() {
- let offset = pos * 32;
- let raw_bytes = &values[offset..offset + 32];
- validate_decimal256_precision_with_lt_bytes(raw_bytes, *p)?;
- }
- Ok(())
- }
DataType::Utf8 => self.validate_utf8::<i32>(),
DataType::LargeUtf8 => self.validate_utf8::<i64>(),
DataType::Binary => self.validate_offsets_full::<i32>(self.buffers[1].len()),
diff --git a/arrow/tests/array_validation.rs b/arrow/tests/array_validation.rs
index f4dcda2e8..16f031a1e 100644
--- a/arrow/tests/array_validation.rs
+++ b/arrow/tests/array_validation.rs
@@ -20,6 +20,7 @@ use arrow::array::{
Int32Array, Int32Builder, Int64Array, StringArray, StructBuilder, UInt64Array,
UInt8Builder,
};
+use arrow_array::Decimal128Array;
use arrow_buffer::{ArrowNativeType, Buffer};
use arrow_data::ArrayData;
use arrow_schema::{DataType, Field, UnionMode};
@@ -1038,7 +1039,6 @@ fn test_string_data_from_foreign() {
}
#[test]
-#[cfg(not(feature = "force_validate"))]
fn test_decimal_full_validation() {
let values_builder = UInt8Builder::with_capacity(10);
let byte_width = 16;
@@ -1055,8 +1055,13 @@ fn test_decimal_full_validation() {
.len(fixed_size_array.len())
.add_buffer(fixed_size_array.data_ref().child_data()[0].buffers()[0].clone());
let array_data = unsafe { builder.build_unchecked() };
- let validation_result = array_data.validate_full();
- let error = validation_result.unwrap_err();
+ array_data.validate_full().unwrap();
+
+ let array = Decimal128Array::from(array_data);
+ let error = array
+ .validate_decimal_precision(array.precision())
+ .unwrap_err();
+
assert_eq!(
"Invalid argument error: 123456 is too large to store in a Decimal128 of precision 5. Max is 99999",
error.to_string()