You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by tu...@apache.org on 2023/06/02 16:12:44 UTC
[arrow-rs] branch master updated: Don't infer 16-byte decimal as decimal256 (#4349)
This is an automated email from the ASF dual-hosted git repository.
tustvold pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/master by this push:
new 008cf9c27 Don't infer 16-byte decimal as decimal256 (#4349)
008cf9c27 is described below
commit 008cf9c27424d581a67ba97f338a22b6eace9cc1
Author: Raphael Taylor-Davies <17...@users.noreply.github.com>
AuthorDate: Fri Jun 2 17:12:37 2023 +0100
Don't infer 16-byte decimal as decimal256 (#4349)
---
parquet/src/arrow/schema/mod.rs | 14 ++++++++++++--
parquet/src/arrow/schema/primitive.rs | 4 ++--
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/parquet/src/arrow/schema/mod.rs b/parquet/src/arrow/schema/mod.rs
index 7469d86dc..cd6e8046c 100644
--- a/parquet/src/arrow/schema/mod.rs
+++ b/parquet/src/arrow/schema/mod.rs
@@ -607,6 +607,9 @@ mod tests {
REQUIRED INT64 decimal2 (DECIMAL(12,2));
REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal3 (DECIMAL(30,2));
REQUIRED BYTE_ARRAY decimal4 (DECIMAL(33,2));
+ REQUIRED BYTE_ARRAY decimal5 (DECIMAL(38,2));
+ REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal6 (DECIMAL(39,2));
+ REQUIRED BYTE_ARRAY decimal7 (DECIMAL(39,2));
}
";
@@ -619,8 +622,11 @@ mod tests {
let arrow_fields = Fields::from(vec![
Field::new("decimal1", DataType::Decimal128(4, 2), false),
Field::new("decimal2", DataType::Decimal128(12, 2), false),
- Field::new("decimal3", DataType::Decimal256(30, 2), false),
+ Field::new("decimal3", DataType::Decimal128(30, 2), false),
Field::new("decimal4", DataType::Decimal128(33, 2), false),
+ Field::new("decimal5", DataType::Decimal128(38, 2), false),
+ Field::new("decimal6", DataType::Decimal256(39, 2), false),
+ Field::new("decimal7", DataType::Decimal256(39, 2), false),
]);
assert_eq!(&arrow_fields, converted_arrow_schema.fields());
}
@@ -1389,6 +1395,8 @@ mod tests {
REQUIRED INT32 decimal_int32 (DECIMAL(8,2));
REQUIRED INT64 decimal_int64 (DECIMAL(16,2));
REQUIRED FIXED_LEN_BYTE_ARRAY (13) decimal_fix_length (DECIMAL(30,2));
+ REQUIRED FIXED_LEN_BYTE_ARRAY (16) decimal128 (DECIMAL(38,2));
+ REQUIRED FIXED_LEN_BYTE_ARRAY (17) decimal256 (DECIMAL(39,2));
}
";
let parquet_group_type = parse_message_type(message_type).unwrap();
@@ -1473,7 +1481,9 @@ mod tests {
),
Field::new("decimal_int32", DataType::Decimal128(8, 2), false),
Field::new("decimal_int64", DataType::Decimal128(16, 2), false),
- Field::new("decimal_fix_length", DataType::Decimal256(30, 2), false),
+ Field::new("decimal_fix_length", DataType::Decimal128(30, 2), false),
+ Field::new("decimal128", DataType::Decimal128(38, 2), false),
+ Field::new("decimal256", DataType::Decimal256(39, 2), false),
];
let arrow_schema = Schema::new(arrow_fields);
let converted_arrow_schema = arrow_to_parquet_schema(&arrow_schema).unwrap();
diff --git a/parquet/src/arrow/schema/primitive.rs b/parquet/src/arrow/schema/primitive.rs
index c67f78076..83d84b77e 100644
--- a/parquet/src/arrow/schema/primitive.rs
+++ b/parquet/src/arrow/schema/primitive.rs
@@ -285,14 +285,14 @@ fn from_fixed_len_byte_array(
// TODO: This should check the type length for the decimal and interval types
match (info.logical_type(), info.converted_type()) {
(Some(LogicalType::Decimal { scale, precision }), _) => {
- if type_length < 16 {
+ if type_length <= 16 {
decimal_128_type(scale, precision)
} else {
decimal_256_type(scale, precision)
}
}
(None, ConvertedType::DECIMAL) => {
- if type_length < 16 {
+ if type_length <= 16 {
decimal_128_type(scale, precision)
} else {
decimal_256_type(scale, precision)