You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by em...@apache.org on 2020/09/16 06:03:11 UTC
[arrow] branch decimal256 updated: Basic support of Decimal256 (PR
for merging into decimal256 branch NOT master) (#8190)
This is an automated email from the ASF dual-hosted git repository.
emkornfield pushed a commit to branch decimal256
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/decimal256 by this push:
new 12e81e6 Basic support of Decimal256 (PR for merging into decimal256 branch NOT master) (#8190)
12e81e6 is described below
commit 12e81e666ffbadf6d817921ec052b23f2d269dad
Author: Mingyu Zhong <69...@users.noreply.github.com>
AuthorDate: Tue Sep 15 23:02:39 2020 -0700
Basic support of Decimal256 (PR for merging into decimal256 branch NOT master) (#8190)
Initial merge of C++ Decimal256 functionality into decimal256 branch.
---
c_glib/test/test-decimal128.rb | 2 +-
cpp/src/arrow/array/array_base.cc | 4 +
cpp/src/arrow/array/array_decimal.cc | 18 +++-
cpp/src/arrow/array/array_decimal.h | 16 ++++
cpp/src/arrow/array/array_dict_test.cc | 66 +++++++++-----
cpp/src/arrow/array/array_test.cc | 59 ++++++++++--
cpp/src/arrow/array/builder_decimal.cc | 35 +++++++
cpp/src/arrow/array/builder_decimal.h | 29 ++++++
cpp/src/arrow/array/concatenate.cc | 2 +-
cpp/src/arrow/array/validate.cc | 7 ++
cpp/src/arrow/builder.cc | 1 +
cpp/src/arrow/c/bridge.cc | 2 +-
cpp/src/arrow/compare.cc | 16 ++++
cpp/src/arrow/dataset/filter.cc | 1 +
cpp/src/arrow/ipc/json_simple.cc | 24 +++--
cpp/src/arrow/ipc/json_simple_test.cc | 41 +++++----
cpp/src/arrow/pretty_print.cc | 5 +
cpp/src/arrow/pretty_print_test.cc | 11 ++-
cpp/src/arrow/python/arrow_to_pandas.cc | 25 +++++
cpp/src/arrow/python/decimal.cc | 42 +++++++--
cpp/src/arrow/python/decimal.h | 18 ++++
cpp/src/arrow/python/inference.cc | 13 ++-
cpp/src/arrow/python/python_test.cc | 67 ++++++++------
cpp/src/arrow/python/python_to_arrow.cc | 19 +++-
cpp/src/arrow/scalar.cc | 8 ++
cpp/src/arrow/scalar.h | 11 +++
cpp/src/arrow/scalar_test.cc | 21 ++++-
cpp/src/arrow/testing/gtest_util.cc | 3 +-
cpp/src/arrow/testing/json_internal.cc | 25 ++++-
cpp/src/arrow/type.cc | 36 +++++++-
cpp/src/arrow/type.h | 29 +++++-
cpp/src/arrow/type_fwd.h | 26 +++++-
cpp/src/arrow/type_test.cc | 90 ++++++++++++++----
cpp/src/arrow/type_traits.h | 14 ++-
cpp/src/arrow/util/basic_decimal.cc | 81 ++++++++++++++++
cpp/src/arrow/util/basic_decimal.h | 56 ++++++++++++
cpp/src/arrow/util/decimal.cc | 132 +++++++++++++++++++++++----
cpp/src/arrow/util/decimal.h | 60 ++++++++++++
cpp/src/arrow/util/decimal_test.cc | 91 ++++++++++++++++++
cpp/src/arrow/visitor.cc | 3 +
cpp/src/arrow/visitor.h | 3 +
cpp/src/arrow/visitor_inline.h | 1 +
python/pyarrow/__init__.py | 9 +-
python/pyarrow/array.pxi | 9 +-
python/pyarrow/includes/libarrow.pxd | 22 ++++-
python/pyarrow/lib.pxd | 9 ++
python/pyarrow/lib.pyx | 3 +-
python/pyarrow/public-api.pxi | 4 +-
python/pyarrow/scalar.pxi | 23 ++++-
python/pyarrow/tests/strategies.py | 10 +-
python/pyarrow/tests/test_array.py | 6 +-
python/pyarrow/tests/test_convert_builtin.py | 41 +++++----
python/pyarrow/tests/test_scalars.py | 23 ++++-
python/pyarrow/tests/test_schema.py | 1 +
python/pyarrow/tests/test_types.py | 31 ++++++-
python/pyarrow/types.pxi | 49 +++++++++-
python/pyarrow/types.py | 17 +++-
r/R/enums.R | 25 ++---
58 files changed, 1280 insertions(+), 215 deletions(-)
diff --git a/c_glib/test/test-decimal128.rb b/c_glib/test/test-decimal128.rb
index 0e4bc82..98789d3 100644
--- a/c_glib/test/test-decimal128.rb
+++ b/c_glib/test/test-decimal128.rb
@@ -214,7 +214,7 @@ class TestDecimal128 < Test::Unit::TestCase
decimal = Arrow::Decimal128.new(10)
message =
"[decimal128][rescale]: Invalid: " +
- "Rescaling decimal value would cause data loss"
+ "Rescaling Decimal128 value would cause data loss"
assert_raise(Arrow::Error::Invalid.new(message)) do
decimal.rescale(1, -1)
end
diff --git a/cpp/src/arrow/array/array_base.cc b/cpp/src/arrow/array/array_base.cc
index 0781dd4..c62d20e 100644
--- a/cpp/src/arrow/array/array_base.cc
+++ b/cpp/src/arrow/array/array_base.cc
@@ -73,6 +73,10 @@ struct ScalarFromArraySlotImpl {
return Finish(Decimal128(a.GetValue(index_)));
}
+ Status Visit(const Decimal256Array& a) {
+ return Finish(Decimal256(a.GetValue(index_)));
+ }
+
template <typename T>
Status Visit(const BaseBinaryArray<T>& a) {
return Finish(a.GetString(index_));
diff --git a/cpp/src/arrow/array/array_decimal.cc b/cpp/src/arrow/array/array_decimal.cc
index 1e813f2..d65f6ee 100644
--- a/cpp/src/arrow/array/array_decimal.cc
+++ b/cpp/src/arrow/array/array_decimal.cc
@@ -33,11 +33,11 @@ namespace arrow {
using internal::checked_cast;
// ----------------------------------------------------------------------
-// Decimal
+// Decimal128
Decimal128Array::Decimal128Array(const std::shared_ptr<ArrayData>& data)
: FixedSizeBinaryArray(data) {
- ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL);
+ ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL128);
}
std::string Decimal128Array::FormatValue(int64_t i) const {
@@ -46,4 +46,18 @@ std::string Decimal128Array::FormatValue(int64_t i) const {
return value.ToString(type_.scale());
}
+// ----------------------------------------------------------------------
+// Decimal256
+
+Decimal256Array::Decimal256Array(const std::shared_ptr<ArrayData>& data)
+ : FixedSizeBinaryArray(data) {
+ ARROW_CHECK_EQ(data->type->id(), Type::DECIMAL256);
+}
+
+std::string Decimal256Array::FormatValue(int64_t i) const {
+ const auto& type_ = checked_cast<const Decimal256Type&>(*type());
+ const Decimal256 value(GetValue(i));
+ return value.ToString(type_.scale());
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/array/array_decimal.h b/cpp/src/arrow/array/array_decimal.h
index 6d5e884..8d7d1c5 100644
--- a/cpp/src/arrow/array/array_decimal.h
+++ b/cpp/src/arrow/array/array_decimal.h
@@ -47,4 +47,20 @@ class ARROW_EXPORT Decimal128Array : public FixedSizeBinaryArray {
// Backward compatibility
using DecimalArray = Decimal128Array;
+// ----------------------------------------------------------------------
+// Decimal256Array
+
+/// Concrete Array class for 256-bit decimal data
+class ARROW_EXPORT Decimal256Array : public FixedSizeBinaryArray {
+ public:
+ using TypeClass = Decimal256Type;
+
+ using FixedSizeBinaryArray::FixedSizeBinaryArray;
+
+ /// \brief Construct Decimal256Array from ArrayData instance
+ explicit Decimal256Array(const std::shared_ptr<ArrayData>& data);
+
+ std::string FormatValue(int64_t i) const;
+};
+
} // namespace arrow
diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc
index 5d36849..50622c59 100644
--- a/cpp/src/arrow/array/array_dict_test.cc
+++ b/cpp/src/arrow/array/array_dict_test.cc
@@ -826,13 +826,13 @@ TEST(TestFixedSizeBinaryDictionaryBuilder, AppendArrayInvalidType) {
}
#endif
-TEST(TestDecimalDictionaryBuilder, Basic) {
+template <typename DecimalValue>
+void TestDecimalDictionaryBuilderBasic(std::shared_ptr<DataType> decimal_type) {
// Build the dictionary Array
- auto decimal_type = arrow::decimal(2, 0);
DictionaryBuilder<FixedSizeBinaryType> builder(decimal_type);
// Test data
- std::vector<Decimal128> test{12, 12, 11, 12};
+ std::vector<DecimalValue> test{12, 12, 11, 12};
for (const auto& value : test) {
ASSERT_OK(builder.Append(value.ToBytes().data()));
}
@@ -848,40 +848,48 @@ TEST(TestDecimalDictionaryBuilder, Basic) {
ASSERT_TRUE(expected.Equals(result));
}
-TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
- const auto& decimal_type = arrow::decimal(21, 0);
+TEST(TestDecimal128DictionaryBuilder, Basic) {
+ TestDecimalDictionaryBuilderBasic<Decimal128>(arrow::decimal128(2, 0));
+}
+
+TEST(TestDecimal256DictionaryBuilder, Basic) {
+ TestDecimalDictionaryBuilderBasic<Decimal256>(arrow::decimal256(76, 0));
+}
+void TestDecimalDictionaryBuilderDoubleTableSize(
+ std::shared_ptr<DataType> decimal_type, FixedSizeBinaryBuilder& decimal_builder) {
// Build the dictionary Array
DictionaryBuilder<FixedSizeBinaryType> dict_builder(decimal_type);
// Build expected data
- Decimal128Builder decimal_builder(decimal_type);
Int16Builder int_builder;
// Fill with 1024 different values
for (int64_t i = 0; i < 1024; i++) {
- const uint8_t bytes[] = {0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 0,
- 12,
- 12,
- static_cast<uint8_t>(i / 128),
- static_cast<uint8_t>(i % 128)};
+ // Decimal256Builder takes 32 bytes, while Decimal128Builder takes only the first 16
+ // bytes.
+ const uint8_t bytes[32] = {0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 0,
+ 12,
+ 12,
+ static_cast<uint8_t>(i / 128),
+ static_cast<uint8_t>(i % 128)};
ASSERT_OK(dict_builder.Append(bytes));
ASSERT_OK(decimal_builder.Append(bytes));
ASSERT_OK(int_builder.Append(static_cast<uint16_t>(i)));
}
// Fill with an already existing value
- const uint8_t known_value[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
+ const uint8_t known_value[32] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 0, 1};
for (int64_t i = 0; i < 1024; i++) {
ASSERT_OK(dict_builder.Append(known_value));
ASSERT_OK(int_builder.Append(1));
@@ -902,6 +910,18 @@ TEST(TestDecimalDictionaryBuilder, DoubleTableSize) {
ASSERT_TRUE(expected.Equals(result));
}
+TEST(TestDecimal128DictionaryBuilder, DoubleTableSize) {
+ const auto& decimal_type = arrow::decimal128(21, 0);
+ Decimal128Builder decimal_builder(decimal_type);
+ TestDecimalDictionaryBuilderDoubleTableSize(decimal_type, decimal_builder);
+}
+
+TEST(TestDecimal256DictionaryBuilder, DoubleTableSize) {
+ const auto& decimal_type = arrow::decimal256(21, 0);
+ Decimal256Builder decimal_builder(decimal_type);
+ TestDecimalDictionaryBuilderDoubleTableSize(decimal_type, decimal_builder);
+}
+
TEST(TestNullDictionaryBuilder, Basic) {
// MakeBuilder
auto dict_type = dictionary(int8(), null());
diff --git a/cpp/src/arrow/array/array_test.cc b/cpp/src/arrow/array/array_test.cc
index ed0fe4a..9f893af 100644
--- a/cpp/src/arrow/array/array_test.cc
+++ b/cpp/src/arrow/array/array_test.cc
@@ -428,6 +428,7 @@ TEST_F(TestArray, TestMakeArrayFromScalar) {
std::make_shared<FixedSizeBinaryScalar>(
hello, fixed_size_binary(static_cast<int32_t>(hello->size()))),
std::make_shared<Decimal128Scalar>(Decimal128(10), decimal(16, 4)),
+ std::make_shared<Decimal256Scalar>(Decimal256(10), decimal(76, 38)),
std::make_shared<StringScalar>(hello),
std::make_shared<LargeStringScalar>(hello),
std::make_shared<ListScalar>(ArrayFromJSON(int8(), "[1, 2, 3]")),
@@ -2338,10 +2339,14 @@ TEST(TestAdaptiveUIntBuilderWithStartIntSize, TestReset) {
// ----------------------------------------------------------------------
// Test Decimal arrays
-using DecimalVector = std::vector<Decimal128>;
-
+template <typename TYPE>
class DecimalTest : public ::testing::TestWithParam<int> {
public:
+ using DecimalBuilder = typename TypeTraits<TYPE>::BuilderType;
+ using DecimalValue = typename TypeTraits<TYPE>::ScalarType::ValueType;
+ using DecimalArray = typename TypeTraits<TYPE>::ArrayType;
+ using DecimalVector = std::vector<DecimalValue>;
+
DecimalTest() {}
template <size_t BYTE_WIDTH = 16>
@@ -2357,8 +2362,8 @@ class DecimalTest : public ::testing::TestWithParam<int> {
template <size_t BYTE_WIDTH = 16>
void TestCreate(int32_t precision, const DecimalVector& draw,
const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
- auto type = std::make_shared<Decimal128Type>(precision, 4);
- auto builder = std::make_shared<Decimal128Builder>(type);
+ auto type = std::make_shared<TYPE>(precision, 4);
+ auto builder = std::make_shared<DecimalBuilder>(type);
size_t null_count = 0;
@@ -2389,7 +2394,7 @@ class DecimalTest : public ::testing::TestWithParam<int> {
ASSERT_OK_AND_ASSIGN(expected_null_bitmap, internal::BytesToBits(valid_bytes));
int64_t expected_null_count = CountNulls(valid_bytes);
- auto expected = std::make_shared<Decimal128Array>(
+ auto expected = std::make_shared<DecimalArray>(
type, size, expected_data, expected_null_bitmap, expected_null_count);
std::shared_ptr<Array> lhs = out->Slice(offset);
@@ -2398,7 +2403,9 @@ class DecimalTest : public ::testing::TestWithParam<int> {
}
};
-TEST_P(DecimalTest, NoNulls) {
+using Decimal128Test = DecimalTest<Decimal128Type>;
+
+TEST_P(Decimal128Test, NoNulls) {
int32_t precision = GetParam();
std::vector<Decimal128> draw = {Decimal128(1), Decimal128(-2), Decimal128(2389),
Decimal128(4), Decimal128(-12348)};
@@ -2407,7 +2414,7 @@ TEST_P(DecimalTest, NoNulls) {
this->TestCreate(precision, draw, valid_bytes, 2);
}
-TEST_P(DecimalTest, WithNulls) {
+TEST_P(Decimal128Test, WithNulls) {
int32_t precision = GetParam();
std::vector<Decimal128> draw = {Decimal128(1), Decimal128(2), Decimal128(-1),
Decimal128(4), Decimal128(-1), Decimal128(1),
@@ -2426,7 +2433,43 @@ TEST_P(DecimalTest, WithNulls) {
this->TestCreate(precision, draw, valid_bytes, 2);
}
-INSTANTIATE_TEST_SUITE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
+INSTANTIATE_TEST_SUITE_P(Decimal128Test, Decimal128Test, ::testing::Range(1, 38));
+
+using Decimal256Test = DecimalTest<Decimal256Type>;
+
+TEST_P(Decimal256Test, NoNulls) {
+ int32_t precision = GetParam();
+ std::vector<Decimal256> draw = {Decimal256(1), Decimal256(-2), Decimal256(2389),
+ Decimal256(4), Decimal256(-12348)};
+ std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+ this->TestCreate(precision, draw, valid_bytes, 0);
+ this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+TEST_P(Decimal256Test, WithNulls) {
+ int32_t precision = GetParam();
+ std::vector<Decimal256> draw = {Decimal256(1), Decimal256(2), Decimal256(-1),
+ Decimal256(4), Decimal256(-1), Decimal256(1),
+ Decimal256(2)};
+ Decimal256 big; // (pow(2, 255) - 1) / pow(10, 38)
+ ASSERT_OK_AND_ASSIGN(big,
+ Decimal256::FromString("578960446186580977117854925043439539266."
+ "34992332820282019728792003956564819967"));
+ draw.push_back(big);
+
+ Decimal256 big_negative; // -pow(2, 255) / pow(10, 38)
+ ASSERT_OK_AND_ASSIGN(big_negative,
+ Decimal256::FromString("-578960446186580977117854925043439539266."
+ "34992332820282019728792003956564819968"));
+ draw.push_back(big_negative);
+
+ std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
+ true, true, true, true};
+ this->TestCreate(precision, draw, valid_bytes, 0);
+ this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+INSTANTIATE_TEST_SUITE_P(Decimal256Test, Decimal256Test, ::testing::Range(1, 76));
// ----------------------------------------------------------------------
// Test rechunking
diff --git a/cpp/src/arrow/array/builder_decimal.cc b/cpp/src/arrow/array/builder_decimal.cc
index ea5c9eb..18ec06c 100644
--- a/cpp/src/arrow/array/builder_decimal.cc
+++ b/cpp/src/arrow/array/builder_decimal.cc
@@ -67,4 +67,39 @@ Status Decimal128Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
return Status::OK();
}
+// ----------------------------------------------------------------------
+// Decimal256Builder
+
+Decimal256Builder::Decimal256Builder(const std::shared_ptr<DataType>& type,
+ MemoryPool* pool)
+ : FixedSizeBinaryBuilder(type, pool),
+ decimal_type_(internal::checked_pointer_cast<Decimal256Type>(type)) {}
+
+Status Decimal256Builder::Append(Decimal256 value) {
+ RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
+ UnsafeAppend(value);
+ return Status::OK();
+}
+
+void Decimal256Builder::UnsafeAppend(Decimal256 value) {
+ value.ToBytes(GetMutableValue(length()));
+ byte_builder_.UnsafeAdvance(32);
+ UnsafeAppendToBitmap(true);
+}
+
+void Decimal256Builder::UnsafeAppend(util::string_view value) {
+ FixedSizeBinaryBuilder::UnsafeAppend(value);
+}
+
+Status Decimal256Builder::FinishInternal(std::shared_ptr<ArrayData>* out) {
+ std::shared_ptr<Buffer> data;
+ RETURN_NOT_OK(byte_builder_.Finish(&data));
+ std::shared_ptr<Buffer> null_bitmap;
+ RETURN_NOT_OK(null_bitmap_builder_.Finish(&null_bitmap));
+
+ *out = ArrayData::Make(type(), length_, {null_bitmap, data}, null_count_);
+ capacity_ = length_ = null_count_ = 0;
+ return Status::OK();
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/array/builder_decimal.h b/cpp/src/arrow/array/builder_decimal.h
index 8f0ff83..31d81ef 100644
--- a/cpp/src/arrow/array/builder_decimal.h
+++ b/cpp/src/arrow/array/builder_decimal.h
@@ -58,6 +58,35 @@ class ARROW_EXPORT Decimal128Builder : public FixedSizeBinaryBuilder {
std::shared_ptr<Decimal128Type> decimal_type_;
};
+class ARROW_EXPORT Decimal256Builder : public FixedSizeBinaryBuilder {
+ public:
+ using TypeClass = Decimal256Type;
+
+ explicit Decimal256Builder(const std::shared_ptr<DataType>& type,
+ MemoryPool* pool = default_memory_pool());
+
+ using FixedSizeBinaryBuilder::Append;
+ using FixedSizeBinaryBuilder::AppendValues;
+ using FixedSizeBinaryBuilder::Reset;
+
+ Status Append(Decimal256 val);
+ void UnsafeAppend(Decimal256 val);
+ void UnsafeAppend(util::string_view val);
+
+ Status FinishInternal(std::shared_ptr<ArrayData>* out) override;
+
+ /// \cond FALSE
+ using ArrayBuilder::Finish;
+ /// \endcond
+
+ Status Finish(std::shared_ptr<Decimal256Array>* out) { return FinishTyped(out); }
+
+ std::shared_ptr<DataType> type() const override { return decimal_type_; }
+
+ protected:
+ std::shared_ptr<Decimal256Type> decimal_type_;
+};
+
using DecimalBuilder = Decimal128Builder;
} // namespace arrow
diff --git a/cpp/src/arrow/array/concatenate.cc b/cpp/src/arrow/array/concatenate.cc
index dcfb3f5..30eeeee 100644
--- a/cpp/src/arrow/array/concatenate.cc
+++ b/cpp/src/arrow/array/concatenate.cc
@@ -201,7 +201,7 @@ class ConcatenateImpl {
}
Status Visit(const FixedWidthType& fixed) {
- // Handles numbers, decimal128, fixed_size_binary
+ // Handles numbers, decimal128, decimal256, fixed_size_binary
ARROW_ASSIGN_OR_RAISE(auto buffers, Buffers(1, fixed));
return ConcatenateBuffers(buffers, pool_).Value(&out_->buffers[1]);
}
diff --git a/cpp/src/arrow/array/validate.cc b/cpp/src/arrow/array/validate.cc
index 3063f55..5bc0bf3 100644
--- a/cpp/src/arrow/array/validate.cc
+++ b/cpp/src/arrow/array/validate.cc
@@ -64,6 +64,13 @@ struct ValidateArrayVisitor {
return Status::OK();
}
+ Status Visit(const Decimal256Array& array) {
+ if (array.length() > 0 && array.values() == nullptr) {
+ return Status::Invalid("values is null");
+ }
+ return Status::OK();
+ }
+
Status Visit(const StringArray& array) { return ValidateBinaryArray(array); }
Status Visit(const BinaryArray& array) { return ValidateBinaryArray(array); }
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index acb2739..998ea0f 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -131,6 +131,7 @@ Status MakeBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type,
BUILDER_CASE(LargeBinary);
BUILDER_CASE(FixedSizeBinary);
BUILDER_CASE(Decimal128);
+ BUILDER_CASE(Decimal256);
case Type::DICTIONARY: {
const auto& dict_type = static_cast<const DictionaryType&>(*type);
diff --git a/cpp/src/arrow/c/bridge.cc b/cpp/src/arrow/c/bridge.cc
index 1e602a6..1585b50 100644
--- a/cpp/src/arrow/c/bridge.cc
+++ b/cpp/src/arrow/c/bridge.cc
@@ -303,7 +303,7 @@ struct SchemaExporter {
return SetFormat("w:" + std::to_string(type.byte_width()));
}
- Status Visit(const Decimal128Type& type) {
+ Status Visit(const DecimalType& type) {
return SetFormat("d:" + std::to_string(type.precision()) + "," +
std::to_string(type.scale()));
}
diff --git a/cpp/src/arrow/compare.cc b/cpp/src/arrow/compare.cc
index 421ec13..622f5cb 100644
--- a/cpp/src/arrow/compare.cc
+++ b/cpp/src/arrow/compare.cc
@@ -353,6 +353,10 @@ class RangeEqualsVisitor {
return Visit(checked_cast<const FixedSizeBinaryArray&>(left));
}
+ Status Visit(const Decimal256Array& left) {
+ return Visit(checked_cast<const FixedSizeBinaryArray&>(left));
+ }
+
Status Visit(const NullArray& left) {
ARROW_UNUSED(left);
result_ = true;
@@ -806,6 +810,12 @@ class TypeEqualsVisitor {
return Status::OK();
}
+ Status Visit(const Decimal256Type& left) {
+ const auto& right = checked_cast<const Decimal256Type&>(right_);
+ result_ = left.precision() == right.precision() && left.scale() == right.scale();
+ return Status::OK();
+ }
+
template <typename T>
enable_if_t<is_list_like_type<T>::value || is_struct_type<T>::value, Status> Visit(
const T& left) {
@@ -919,6 +929,12 @@ class ScalarEqualsVisitor {
return Status::OK();
}
+ Status Visit(const Decimal256Scalar& left) {
+ const auto& right = checked_cast<const Decimal256Scalar&>(right_);
+ result_ = left.value == right.value;
+ return Status::OK();
+ }
+
Status Visit(const ListScalar& left) {
const auto& right = checked_cast<const ListScalar&>(right_);
result_ = internal::SharedPtrEquals(left.value, right.value);
diff --git a/cpp/src/arrow/dataset/filter.cc b/cpp/src/arrow/dataset/filter.cc
index 0ef449f..4c0ce50 100644
--- a/cpp/src/arrow/dataset/filter.cc
+++ b/cpp/src/arrow/dataset/filter.cc
@@ -136,6 +136,7 @@ struct CompareVisitor {
}
Status Visit(const Decimal128Type&) { return CompareValues<Decimal128Type>(); }
+ Status Visit(const Decimal256Type&) { return CompareValues<Decimal256Type>(); }
// Explicit because it falls under `physical_unsigned_integer`.
// TODO(bkietz) whenever we vendor a float16, this can be implemented
diff --git a/cpp/src/arrow/ipc/json_simple.cc b/cpp/src/arrow/ipc/json_simple.cc
index d307373..08d7d8c 100644
--- a/cpp/src/arrow/ipc/json_simple.cc
+++ b/cpp/src/arrow/ipc/json_simple.cc
@@ -274,12 +274,16 @@ class FloatConverter final : public ConcreteConverter<FloatConverter<Type>> {
// ------------------------------------------------------------------------
// Converter for decimal arrays
-class DecimalConverter final : public ConcreteConverter<DecimalConverter> {
+template <typename DecimalSubtype, typename DecimalValue>
+class DecimalConverter final
+ : public ConcreteConverter<DecimalConverter<DecimalSubtype, DecimalValue>> {
public:
+ using BuilderType = typename TypeTraits<DecimalSubtype>::BuilderType;
+
explicit DecimalConverter(const std::shared_ptr<DataType>& type) {
this->type_ = type;
- decimal_type_ = checked_cast<Decimal128Type*>(type.get());
- builder_ = std::make_shared<DecimalBuilder>(type);
+ decimal_type_ = checked_cast<DecimalSubtype*>(type.get());
+ builder_ = std::make_shared<BuilderType>(type);
}
Status AppendValue(const rj::Value& json_obj) override {
@@ -288,9 +292,9 @@ class DecimalConverter final : public ConcreteConverter<DecimalConverter> {
}
if (json_obj.IsString()) {
int32_t precision, scale;
- Decimal128 d;
+ DecimalValue d;
auto view = util::string_view(json_obj.GetString(), json_obj.GetStringLength());
- RETURN_NOT_OK(Decimal128::FromString(view, &d, &precision, &scale));
+ RETURN_NOT_OK(DecimalValue::FromString(view, &d, &precision, &scale));
if (scale != decimal_type_->scale()) {
return Status::Invalid("Invalid scale for decimal: expected ",
decimal_type_->scale(), ", got ", scale);
@@ -303,10 +307,13 @@ class DecimalConverter final : public ConcreteConverter<DecimalConverter> {
std::shared_ptr<ArrayBuilder> builder() override { return builder_; }
private:
- std::shared_ptr<DecimalBuilder> builder_;
- Decimal128Type* decimal_type_;
+ std::shared_ptr<BuilderType> builder_;
+ DecimalSubtype* decimal_type_;
};
+using Decimal128Converter = DecimalConverter<Decimal128Type, Decimal128>;
+using Decimal256Converter = DecimalConverter<Decimal256Type, Decimal256>;
+
// ------------------------------------------------------------------------
// Converter for timestamp arrays
@@ -764,7 +771,8 @@ Status GetConverter(const std::shared_ptr<DataType>& type,
SIMPLE_CONVERTER_CASE(Type::LARGE_STRING, StringConverter<LargeStringType>)
SIMPLE_CONVERTER_CASE(Type::LARGE_BINARY, StringConverter<LargeBinaryType>)
SIMPLE_CONVERTER_CASE(Type::FIXED_SIZE_BINARY, FixedSizeBinaryConverter)
- SIMPLE_CONVERTER_CASE(Type::DECIMAL, DecimalConverter)
+ SIMPLE_CONVERTER_CASE(Type::DECIMAL128, Decimal128Converter)
+ SIMPLE_CONVERTER_CASE(Type::DECIMAL256, Decimal256Converter)
SIMPLE_CONVERTER_CASE(Type::SPARSE_UNION, UnionConverter)
SIMPLE_CONVERTER_CASE(Type::DENSE_UNION, UnionConverter)
SIMPLE_CONVERTER_CASE(Type::INTERVAL_MONTHS, IntegerConverter<MonthIntervalType>)
diff --git a/cpp/src/arrow/ipc/json_simple_test.cc b/cpp/src/arrow/ipc/json_simple_test.cc
index fe1b027..d21952a 100644
--- a/cpp/src/arrow/ipc/json_simple_test.cc
+++ b/cpp/src/arrow/ipc/json_simple_test.cc
@@ -440,14 +440,14 @@ TEST(TestFixedSizeBinary, Errors) {
ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"abcd\"]", &array));
}
-TEST(TestDecimal, Basics) {
- std::shared_ptr<DataType> type = decimal(10, 4);
+template <typename DecimalValue, typename DecimalBuilder>
+void TestDecimalBasic(std::shared_ptr<DataType> type) {
std::shared_ptr<Array> expected, actual;
ASSERT_OK(ArrayFromJSON(type, "[]", &actual));
ASSERT_OK(actual->ValidateFull());
{
- Decimal128Builder builder(type);
+ DecimalBuilder builder(type);
ASSERT_OK(builder.Finish(&expected));
}
AssertArraysEqual(*expected, *actual);
@@ -455,9 +455,9 @@ TEST(TestDecimal, Basics) {
ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", \"-78.9000\"]", &actual));
ASSERT_OK(actual->ValidateFull());
{
- Decimal128Builder builder(type);
- ASSERT_OK(builder.Append(Decimal128(1234567)));
- ASSERT_OK(builder.Append(Decimal128(-789000)));
+ DecimalBuilder builder(type);
+ ASSERT_OK(builder.Append(DecimalValue(1234567)));
+ ASSERT_OK(builder.Append(DecimalValue(-789000)));
ASSERT_OK(builder.Finish(&expected));
}
AssertArraysEqual(*expected, *actual);
@@ -465,23 +465,32 @@ TEST(TestDecimal, Basics) {
ASSERT_OK(ArrayFromJSON(type, "[\"123.4567\", null]", &actual));
ASSERT_OK(actual->ValidateFull());
{
- Decimal128Builder builder(type);
- ASSERT_OK(builder.Append(Decimal128(1234567)));
+ DecimalBuilder builder(type);
+ ASSERT_OK(builder.Append(DecimalValue(1234567)));
ASSERT_OK(builder.AppendNull());
ASSERT_OK(builder.Finish(&expected));
}
AssertArraysEqual(*expected, *actual);
}
-TEST(TestDecimal, Errors) {
- std::shared_ptr<DataType> type = decimal(10, 4);
- std::shared_ptr<Array> array;
+TEST(TestDecimal128, Basics) {
+ TestDecimalBasic<Decimal128, Decimal128Builder>(decimal128(10, 4));
+}
- ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
- ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[12.3456]", &array));
- // Bad scale
- ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array));
- ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array));
+TEST(TestDecimal256, Basics) {
+ TestDecimalBasic<Decimal256, Decimal256Builder>(decimal256(10, 4));
+}
+
+TEST(TestDecimal, Errors) {
+ for (std::shared_ptr<DataType> type : {decimal128(10, 4), decimal256(10, 4)}) {
+ std::shared_ptr<Array> array;
+
+ ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[0]", &array));
+ ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[12.3456]", &array));
+ // Bad scale
+ ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.345\"]", &array));
+ ASSERT_RAISES(Invalid, ArrayFromJSON(type, "[\"12.34560\"]", &array));
+ }
}
TEST(TestList, IntegerList) {
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index 4b3fd86..3a85d56 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -226,6 +226,11 @@ class ArrayPrinter : public PrettyPrinter {
return Status::OK();
}
+ Status WriteDataValues(const Decimal256Array& array) {
+ WriteValues(array, [&](int64_t i) { (*sink_) << array.FormatValue(i); });
+ return Status::OK();
+ }
+
template <typename T>
enable_if_list_like<typename T::TypeClass, Status> WriteDataValues(const T& array) {
bool skip_comma = true;
diff --git a/cpp/src/arrow/pretty_print_test.cc b/cpp/src/arrow/pretty_print_test.cc
index 6124b8f..e7fcc75 100644
--- a/cpp/src/arrow/pretty_print_test.cc
+++ b/cpp/src/arrow/pretty_print_test.cc
@@ -498,15 +498,16 @@ TEST_F(TestPrettyPrint, FixedSizeBinaryType) {
CheckArray(*array, {2, 1}, ex_2);
}
-TEST_F(TestPrettyPrint, Decimal128Type) {
+TEST_F(TestPrettyPrint, DecimalTypes) {
int32_t p = 19;
int32_t s = 4;
- auto type = decimal(p, s);
- auto array = ArrayFromJSON(type, "[\"123.4567\", \"456.7891\", null]");
+ for (auto type : {decimal128(p, s), decimal256(p, s)}) {
+ auto array = ArrayFromJSON(type, "[\"123.4567\", \"456.7891\", null]");
- static const char* ex = "[\n 123.4567,\n 456.7891,\n null\n]";
- CheckArray(*array, {0}, ex);
+ static const char* ex = "[\n 123.4567,\n 456.7891,\n null\n]";
+ CheckArray(*array, {0}, ex);
+ }
}
TEST_F(TestPrettyPrint, DictionaryType) {
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index 47b62a3..8def6b8 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -1014,6 +1014,31 @@ struct ObjectWriterVisitor {
return Status::OK();
}
+ Status Visit(const Decimal256Type& type) {
+ OwnedRef decimal;
+ OwnedRef Decimal;
+ RETURN_NOT_OK(internal::ImportModule("decimal", &decimal));
+ RETURN_NOT_OK(internal::ImportFromModule(decimal.obj(), "Decimal", &Decimal));
+ PyObject* decimal_constructor = Decimal.obj();
+
+ for (int c = 0; c < data.num_chunks(); c++) {
+ const auto& arr = checked_cast<const arrow::Decimal256Array&>(*data.chunk(c));
+
+ for (int64_t i = 0; i < arr.length(); ++i) {
+ if (arr.IsNull(i)) {
+ Py_INCREF(Py_None);
+ *out_values++ = Py_None;
+ } else {
+ *out_values++ =
+ internal::DecimalFromString(decimal_constructor, arr.FormatValue(i));
+ RETURN_IF_PYERROR();
+ }
+ }
+ }
+
+ return Status::OK();
+ }
+
template <typename T>
enable_if_t<is_fixed_size_list_type<T>::value || is_var_length_list_type<T>::value,
Status>
diff --git a/cpp/src/arrow/python/decimal.cc b/cpp/src/arrow/python/decimal.cc
index 1871201..6a9be60 100644
--- a/cpp/src/arrow/python/decimal.cc
+++ b/cpp/src/arrow/python/decimal.cc
@@ -109,13 +109,14 @@ PyObject* DecimalFromString(PyObject* decimal_constructor,
namespace {
+template <typename ArrowDecimal>
Status DecimalFromStdString(const std::string& decimal_string,
- const DecimalType& arrow_type, Decimal128* out) {
+ const DecimalType& arrow_type, ArrowDecimal* out) {
int32_t inferred_precision;
int32_t inferred_scale;
- RETURN_NOT_OK(
- Decimal128::FromString(decimal_string, out, &inferred_precision, &inferred_scale));
+ RETURN_NOT_OK(ArrowDecimal::FromString(decimal_string, out, &inferred_precision,
+ &inferred_scale));
const int32_t precision = arrow_type.precision();
const int32_t scale = arrow_type.scale();
@@ -133,10 +134,10 @@ Status DecimalFromStdString(const std::string& decimal_string,
return Status::OK();
}
-} // namespace
-
-Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
- Decimal128* out) {
+template <typename ArrowDecimal>
+Status InternalDecimalFromPythonDecimal(PyObject* python_decimal,
+ const DecimalType& arrow_type,
+ ArrowDecimal* out) {
DCHECK_NE(python_decimal, NULLPTR);
DCHECK_NE(out, NULLPTR);
@@ -145,8 +146,9 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
return DecimalFromStdString(string, arrow_type, out);
}
-Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
- Decimal128* out) {
+template <typename ArrowDecimal>
+Status InternalDecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+ ArrowDecimal* out) {
DCHECK_NE(obj, NULLPTR);
DCHECK_NE(out, NULLPTR);
@@ -163,6 +165,28 @@ Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
}
}
+} // namespace
+
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+ Decimal128* out) {
+ return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out);
+}
+
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+ Decimal128* out) {
+ return InternalDecimalFromPyObject(obj, arrow_type, out);
+}
+
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+ Decimal256* out) {
+ return InternalDecimalFromPythonDecimal(python_decimal, arrow_type, out);
+}
+
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type,
+ Decimal256* out) {
+ return InternalDecimalFromPyObject(obj, arrow_type, out);
+}
+
bool PyDecimal_Check(PyObject* obj) {
static OwnedRef decimal_type;
if (!decimal_type.obj()) {
diff --git a/cpp/src/arrow/python/decimal.h b/cpp/src/arrow/python/decimal.h
index 3d20b01..1187037 100644
--- a/cpp/src/arrow/python/decimal.h
+++ b/cpp/src/arrow/python/decimal.h
@@ -25,6 +25,7 @@
namespace arrow {
class Decimal128;
+class Decimal256;
namespace py {
@@ -72,6 +73,23 @@ Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arr
ARROW_PYTHON_EXPORT
Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal128* out);
+// \brief Convert a Python decimal to an Arrow Decimal256 object
+// \param[in] python_decimal A Python decimal.Decimal instance
+// \param[in] arrow_type An instance of arrow::DecimalType
+// \param[out] out A pointer to a Decimal256
+// \return The status of the operation
+ARROW_PYTHON_EXPORT
+Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
+ Decimal256* out);
+
+// \brief Convert a Python object to an Arrow Decimal256 object
+// \param[in] python_decimal A Python int or decimal.Decimal instance
+// \param[in] arrow_type An instance of arrow::DecimalType
+// \param[out] out A pointer to a Decimal256
+// \return The status of the operation
+ARROW_PYTHON_EXPORT
+Status DecimalFromPyObject(PyObject* obj, const DecimalType& arrow_type, Decimal256* out);
+
// \brief Check whether obj is an instance of Decimal
ARROW_PYTHON_EXPORT
bool PyDecimal_Check(PyObject* obj);
diff --git a/cpp/src/arrow/python/inference.cc b/cpp/src/arrow/python/inference.cc
index d1ce2c2..1ba7dad 100644
--- a/cpp/src/arrow/python/inference.cc
+++ b/cpp/src/arrow/python/inference.cc
@@ -450,9 +450,16 @@ class TypeInferrer {
} else if (struct_count_) {
RETURN_NOT_OK(GetStructType(out));
} else if (decimal_count_) {
- // the default constructor does not validate the precision and scale
- ARROW_ASSIGN_OR_RAISE(*out, Decimal128Type::Make(max_decimal_metadata_.precision(),
- max_decimal_metadata_.scale()));
+ if (max_decimal_metadata_.precision() > Decimal128Type::kMaxPrecision) {
+ // the default constructor does not validate the precision and scale
+ ARROW_ASSIGN_OR_RAISE(*out,
+ Decimal256Type::Make(max_decimal_metadata_.precision(),
+ max_decimal_metadata_.scale()));
+ } else {
+ ARROW_ASSIGN_OR_RAISE(*out,
+ Decimal128Type::Make(max_decimal_metadata_.precision(),
+ max_decimal_metadata_.scale()));
+ }
} else if (float_count_) {
// Prioritize floats before integers
*out = float64();
diff --git a/cpp/src/arrow/python/python_test.cc b/cpp/src/arrow/python/python_test.cc
index b21c16a..9d881d2 100644
--- a/cpp/src/arrow/python/python_test.cc
+++ b/cpp/src/arrow/python/python_test.cc
@@ -28,6 +28,7 @@
#include "arrow/table.h"
#include "arrow/testing/gtest_util.h"
#include "arrow/util/decimal.h"
+#include "arrow/util/optional.h"
#include "arrow/python/arrow_to_pandas.h"
#include "arrow/python/decimal.h"
@@ -333,54 +334,62 @@ TEST(BuiltinConversionTest, TestMixedTypeFails) {
ASSERT_RAISES(TypeError, ConvertPySequence(list, {}, &arr));
}
+template <typename DecimalValue>
+void DecimalTestFromPythonDecimalRescale(std::shared_ptr<DataType> type,
+ OwnedRef python_decimal,
+ ::arrow::util::optional<int> expected) {
+ DecimalValue value;
+ const auto& decimal_type = checked_cast<const DecimalType&>(*type);
+
+ if (expected.has_value()) {
+ ASSERT_OK(
+ internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
+ ASSERT_EQ(expected.value(), value);
+
+ ASSERT_OK(internal::DecimalFromPyObject(python_decimal.obj(), decimal_type, &value));
+ ASSERT_EQ(expected.value(), value);
+ } else {
+ ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
+ decimal_type, &value));
+ ASSERT_RAISES(Invalid, internal::DecimalFromPyObject(python_decimal.obj(),
+ decimal_type, &value));
+ }
+}
+
TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) {
// We fail when truncating values that would lose data if cast to a decimal type with
// lower scale
- Decimal128 value;
- OwnedRef python_decimal(this->CreatePythonDecimal("1.001"));
- auto type = ::arrow::decimal(10, 2);
- const auto& decimal_type = checked_cast<const DecimalType&>(*type);
- ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(),
- decimal_type, &value));
+ DecimalTestFromPythonDecimalRescale<Decimal128>(::arrow::decimal128(10, 2),
+ this->CreatePythonDecimal("1.001"), {});
+ // TODO: Test Decimal256 after implementing scaling.
}
TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) {
// We allow truncation of values that do not lose precision when dividing by 10 * the
// difference between the scales, e.g., 1.000 -> 1.00
- Decimal128 value;
- OwnedRef python_decimal(this->CreatePythonDecimal("1.000"));
- auto type = ::arrow::decimal(10, 2);
- const auto& decimal_type = checked_cast<const DecimalType&>(*type);
- ASSERT_OK(
- internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
- ASSERT_EQ(100, value.low_bits());
- ASSERT_EQ(0, value.high_bits());
-
- ASSERT_OK(internal::DecimalFromPyObject(python_decimal.obj(), decimal_type, &value));
- ASSERT_EQ(100, value.low_bits());
- ASSERT_EQ(0, value.high_bits());
+ DecimalTestFromPythonDecimalRescale<Decimal128>(
+ ::arrow::decimal128(10, 2), this->CreatePythonDecimal("1.000"), 100);
+ // TODO: Test Decimal256 after implementing scaling.
}
TEST_F(DecimalTest, FromPythonNegativeDecimalRescale) {
- Decimal128 value;
- OwnedRef python_decimal(this->CreatePythonDecimal("-1.000"));
- auto type = ::arrow::decimal(10, 9);
- const auto& decimal_type = checked_cast<const DecimalType&>(*type);
- ASSERT_OK(
- internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value));
- ASSERT_EQ(-1000000000, value);
+ DecimalTestFromPythonDecimalRescale<Decimal128>(
+ ::arrow::decimal128(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000);
+ // TODO: Test Decimal256 after implementing scaling.
}
-TEST_F(DecimalTest, FromPythonInteger) {
+TEST_F(DecimalTest, Decimal128FromPythonInteger) {
Decimal128 value;
OwnedRef python_long(PyLong_FromLong(42));
- auto type = ::arrow::decimal(10, 2);
+ auto type = ::arrow::decimal128(10, 2);
const auto& decimal_type = checked_cast<const DecimalType&>(*type);
ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value));
ASSERT_EQ(4200, value);
}
-TEST_F(DecimalTest, TestOverflowFails) {
+// TODO: Test Decimal256 from python after implementing scaling.
+
+TEST_F(DecimalTest, TestDecimal128OverflowFails) {
Decimal128 value;
OwnedRef python_decimal(
this->CreatePythonDecimal("9999999999999999999999999999999999999.9"));
@@ -395,6 +404,8 @@ TEST_F(DecimalTest, TestOverflowFails) {
decimal_type, &value));
}
+// TODO: Test Decimal256 overflow after implementing scaling.
+
TEST_F(DecimalTest, TestNoneAndNaN) {
OwnedRef list_ref(PyList_New(4));
PyObject* list = list_ref.obj();
diff --git a/cpp/src/arrow/python/python_to_arrow.cc b/cpp/src/arrow/python/python_to_arrow.cc
index 949213f..0b128f2 100644
--- a/cpp/src/arrow/python/python_to_arrow.cc
+++ b/cpp/src/arrow/python/python_to_arrow.cc
@@ -1111,10 +1111,10 @@ class StructConverter : public TypedConverter<StructType, null_coding> {
bool ignore_timezone_;
};
-template <NullCoding null_coding>
-class DecimalConverter : public TypedConverter<arrow::Decimal128Type, null_coding> {
+template <NullCoding null_coding, class DecimalSubtype, class DecimalValue>
+class DecimalConverter : public TypedConverter<DecimalSubtype, null_coding> {
public:
- using BASE = TypedConverter<arrow::Decimal128Type, null_coding>;
+ using BASE = TypedConverter<DecimalSubtype, null_coding>;
Status Init(ArrayBuilder* builder) override {
RETURN_NOT_OK(BASE::Init(builder));
@@ -1123,7 +1123,7 @@ class DecimalConverter : public TypedConverter<arrow::Decimal128Type, null_codin
}
Status AppendValue(PyObject* obj) override {
- Decimal128 value;
+ DecimalValue value;
RETURN_NOT_OK(internal::DecimalFromPyObject(obj, *decimal_type_, &value));
return this->typed_builder_->Append(value);
}
@@ -1132,6 +1132,14 @@ class DecimalConverter : public TypedConverter<arrow::Decimal128Type, null_codin
std::shared_ptr<DecimalType> decimal_type_;
};
+template <NullCoding null_coding>
+using Decimal128Converter =
+ DecimalConverter<null_coding, arrow::Decimal128Type, Decimal128>;
+
+template <NullCoding null_coding>
+using Decimal256Converter =
+ DecimalConverter<null_coding, arrow::Decimal256Type, Decimal256>;
+
#define PRIMITIVE(TYPE_ENUM, TYPE) \
case Type::TYPE_ENUM: \
*out = std::unique_ptr<SeqConverter>(new PrimitiveConverter<TYPE, null_coding>); \
@@ -1162,7 +1170,8 @@ Status GetConverterFlat(const std::shared_ptr<DataType>& type, bool strict_conve
PRIMITIVE(DOUBLE, DoubleType);
PRIMITIVE(DATE32, Date32Type);
PRIMITIVE(DATE64, Date64Type);
- SIMPLE_CONVERTER_CASE(DECIMAL, DecimalConverter);
+ SIMPLE_CONVERTER_CASE(DECIMAL128, Decimal128Converter);
+ SIMPLE_CONVERTER_CASE(DECIMAL256, Decimal256Converter);
case Type::BINARY:
*out =
std::unique_ptr<SeqConverter>(new BinaryConverter<BinaryType, null_coding>());
diff --git a/cpp/src/arrow/scalar.cc b/cpp/src/arrow/scalar.cc
index 88e594e..fb7de22 100644
--- a/cpp/src/arrow/scalar.cc
+++ b/cpp/src/arrow/scalar.cc
@@ -69,6 +69,14 @@ struct ScalarHashImpl {
return StdHash(s.value.low_bits()) & StdHash(s.value.high_bits());
}
+ Status Visit(const Decimal256Scalar& s) {
+ Status status = Status::OK();
+ for (uint64_t elem : s.value.little_endian_array()) {
+ status &= StdHash(elem);
+ }
+ return status;
+ }
+
Status Visit(const BaseListScalar& s) { return ArrayHash(*s.value); }
Status Visit(const StructScalar& s) {
diff --git a/cpp/src/arrow/scalar.h b/cpp/src/arrow/scalar.h
index 4a007dd..a6ec9d3 100644
--- a/cpp/src/arrow/scalar.h
+++ b/cpp/src/arrow/scalar.h
@@ -347,6 +347,17 @@ struct ARROW_EXPORT Decimal128Scalar : public Scalar {
Decimal128 value;
};
+struct ARROW_EXPORT Decimal256Scalar : public Scalar {
+ using Scalar::Scalar;
+ using TypeClass = Decimal256Type;
+ using ValueType = Decimal256;
+
+ Decimal256Scalar(Decimal256 value, std::shared_ptr<DataType> type)
+ : Scalar(std::move(type), true), value(value) {}
+
+ Decimal256 value;
+};
+
struct ARROW_EXPORT BaseListScalar : public Scalar {
using Scalar::Scalar;
using ValueType = std::shared_ptr<Array>;
diff --git a/cpp/src/arrow/scalar_test.cc b/cpp/src/arrow/scalar_test.cc
index f1ab6d6..7eb365d 100644
--- a/cpp/src/arrow/scalar_test.cc
+++ b/cpp/src/arrow/scalar_test.cc
@@ -127,8 +127,8 @@ TYPED_TEST(TestNumericScalar, MakeScalar) {
ASSERT_EQ(ScalarType(3), *three);
}
-TEST(TestDecimalScalar, Basics) {
- auto ty = decimal(3, 2);
+TEST(TestDecimal128Scalar, Basics) {
+ auto ty = decimal128(3, 2);
auto pi = Decimal128Scalar(Decimal128("3.14"), ty);
auto null = MakeNullScalar(ty);
@@ -144,6 +144,23 @@ TEST(TestDecimalScalar, Basics) {
ASSERT_FALSE(second->Equals(null));
}
+TEST(TestDecimal256Scalar, Basics) {
+ auto ty = decimal256(3, 2);
+ auto pi = Decimal256Scalar(Decimal256("3.14"), ty);
+ auto null = MakeNullScalar(ty);
+
+ ASSERT_EQ(pi.value, Decimal256("3.14"));
+
+ // test Array.GetScalar
+ auto arr = ArrayFromJSON(ty, "[null, \"3.14\"]");
+ ASSERT_OK_AND_ASSIGN(auto first, arr->GetScalar(0));
+ ASSERT_OK_AND_ASSIGN(auto second, arr->GetScalar(1));
+ ASSERT_TRUE(first->Equals(null));
+ ASSERT_FALSE(first->Equals(pi));
+ ASSERT_TRUE(second->Equals(pi));
+ ASSERT_FALSE(second->Equals(null));
+}
+
TEST(TestBinaryScalar, Basics) {
std::string data = "test data";
auto buf = std::make_shared<Buffer>(data);
diff --git a/cpp/src/arrow/testing/gtest_util.cc b/cpp/src/arrow/testing/gtest_util.cc
index d17cc0a..51ebe30 100644
--- a/cpp/src/arrow/testing/gtest_util.cc
+++ b/cpp/src/arrow/testing/gtest_util.cc
@@ -68,7 +68,8 @@ std::vector<Type::type> AllTypeIds() {
Type::HALF_FLOAT,
Type::FLOAT,
Type::DOUBLE,
- Type::DECIMAL,
+ Type::DECIMAL128,
+ Type::DECIMAL256,
Type::DATE32,
Type::DATE64,
Type::TIME32,
diff --git a/cpp/src/arrow/testing/json_internal.cc b/cpp/src/arrow/testing/json_internal.cc
index 9bd2f14..4a191a6 100644
--- a/cpp/src/arrow/testing/json_internal.cc
+++ b/cpp/src/arrow/testing/json_internal.cc
@@ -303,6 +303,13 @@ class SchemaWriter {
writer_->Int(type.scale());
}
+ void WriteTypeMetadata(const Decimal256Type& type) {
+ writer_->Key("precision");
+ writer_->Int(type.precision());
+ writer_->Key("scale");
+ writer_->Int(type.scale());
+ }
+
void WriteTypeMetadata(const UnionType& type) {
writer_->Key("mode");
switch (type.mode()) {
@@ -376,6 +383,7 @@ class SchemaWriter {
}
Status Visit(const Decimal128Type& type) { return WritePrimitive("decimal", type); }
+ Status Visit(const Decimal256Type& type) { return WritePrimitive("decimal256", type); }
Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
Status Visit(const DurationType& type) { return WritePrimitive(kDuration, type); }
Status Visit(const MonthIntervalType& type) { return WritePrimitive("interval", type); }
@@ -546,6 +554,18 @@ class ArrayWriter {
}
}
+ void WriteDataValues(const Decimal256Array& arr) {
+ static const char null_string[] = "0";
+ for (int64_t i = 0; i < arr.length(); ++i) {
+ if (arr.IsValid(i)) {
+ const Decimal256 value(arr.GetValue(i));
+ writer_->String(value.ToIntegerString());
+ } else {
+ writer_->String(null_string, sizeof(null_string));
+ }
+ }
+ }
+
void WriteDataValues(const BooleanArray& arr) {
for (int64_t i = 0; i < arr.length(); ++i) {
if (arr.IsValid(i)) {
@@ -1296,8 +1316,9 @@ class ArrayReader {
DCHECK_GT(val.GetStringLength(), 0)
<< "Empty string found when parsing Decimal128 value";
- Decimal128 value;
- ARROW_ASSIGN_OR_RAISE(value, Decimal128::FromString(val.GetString()));
+ using Value = typename TypeTraits<T>::ScalarType::ValueType;
+ Value value;
+ ARROW_ASSIGN_OR_RAISE(value, Value::FromString(val.GetString()));
RETURN_NOT_OK(builder.Append(value));
}
}
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index 5482dea..f27f967 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -70,6 +70,8 @@ constexpr Type::type StructType::type_id;
constexpr Type::type Decimal128Type::type_id;
+constexpr Type::type Decimal256Type::type_id;
+
constexpr Type::type SparseUnionType::type_id;
constexpr Type::type DenseUnionType::type_id;
@@ -131,6 +133,7 @@ std::string ToString(Type::type id) {
TO_STRING_CASE(FLOAT)
TO_STRING_CASE(DOUBLE)
TO_STRING_CASE(DECIMAL)
+ TO_STRING_CASE(DECIMAL256)
TO_STRING_CASE(DATE32)
TO_STRING_CASE(DATE64)
TO_STRING_CASE(TIME32)
@@ -748,7 +751,7 @@ std::vector<std::shared_ptr<Field>> StructType::GetAllFieldsByName(
// Decimal128 type
Decimal128Type::Decimal128Type(int32_t precision, int32_t scale)
- : DecimalType(16, precision, scale) {
+ : DecimalType(type_id, 16, precision, scale) {
ARROW_CHECK_GE(precision, kMinPrecision);
ARROW_CHECK_LE(precision, kMaxPrecision);
}
@@ -761,6 +764,22 @@ Result<std::shared_ptr<DataType>> Decimal128Type::Make(int32_t precision, int32_
}
// ----------------------------------------------------------------------
+// Decimal256 type
+
+Decimal256Type::Decimal256Type(int32_t precision, int32_t scale)
+ : DecimalType(type_id, 32, precision, scale) {
+ ARROW_CHECK_GE(precision, kMinPrecision);
+ ARROW_CHECK_LE(precision, kMaxPrecision);
+}
+
+Result<std::shared_ptr<DataType>> Decimal256Type::Make(int32_t precision, int32_t scale) {
+ if (precision < kMinPrecision || precision > kMaxPrecision) {
+ return Status::Invalid("Decimal precision out of range: ", precision);
+ }
+ return std::make_shared<Decimal256Type>(precision, scale);
+}
+
+// ----------------------------------------------------------------------
// Dictionary-encoded type
Status DictionaryType::ValidateParameters(const DataType& index_type,
@@ -2138,13 +2157,28 @@ std::shared_ptr<Field> field(std::string name, std::shared_ptr<DataType> type,
}
std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
+ return precision <= Decimal128Type::kMaxPrecision ? decimal128(precision, scale)
+ : decimal256(precision, scale);
+}
+
+std::shared_ptr<DataType> decimal128(int32_t precision, int32_t scale) {
return std::make_shared<Decimal128Type>(precision, scale);
}
+std::shared_ptr<DataType> decimal256(int32_t precision, int32_t scale) {
+ return std::make_shared<Decimal256Type>(precision, scale);
+}
+
std::string Decimal128Type::ToString() const {
std::stringstream s;
s << "decimal(" << precision_ << ", " << scale_ << ")";
return s.str();
}
+std::string Decimal256Type::ToString() const {
+ std::stringstream s;
+ s << "decimal256(" << precision_ << ", " << scale_ << ")";
+ return s.str();
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index e67cf28..c8a71ab 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -861,10 +861,9 @@ class ARROW_EXPORT StructType : public NestedType {
/// \brief Base type class for (fixed-size) decimal data
class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
public:
- explicit DecimalType(int32_t byte_width, int32_t precision, int32_t scale)
- : FixedSizeBinaryType(byte_width, Type::DECIMAL),
- precision_(precision),
- scale_(scale) {}
+ explicit DecimalType(Type::type type_id, int32_t byte_width, int32_t precision,
+ int32_t scale)
+ : FixedSizeBinaryType(byte_width, type_id), precision_(precision), scale_(scale) {}
int32_t precision() const { return precision_; }
int32_t scale() const { return scale_; }
@@ -879,7 +878,7 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
/// \brief Concrete type class for 128-bit decimal data
class ARROW_EXPORT Decimal128Type : public DecimalType {
public:
- static constexpr Type::type type_id = Type::DECIMAL;
+ static constexpr Type::type type_id = Type::DECIMAL128;
static constexpr const char* type_name() { return "decimal"; }
@@ -896,6 +895,26 @@ class ARROW_EXPORT Decimal128Type : public DecimalType {
static constexpr int32_t kMaxPrecision = 38;
};
+/// \brief Concrete type class for 256-bit decimal data
+class ARROW_EXPORT Decimal256Type : public DecimalType {
+ public:
+ static constexpr Type::type type_id = Type::DECIMAL256;
+
+ static constexpr const char* type_name() { return "decimal256"; }
+
+ /// Decimal256Type constructor that aborts on invalid input.
+ explicit Decimal256Type(int32_t precision, int32_t scale);
+
+ /// Decimal256Type constructor that returns an error on invalid input.
+ static Result<std::shared_ptr<DataType>> Make(int32_t precision, int32_t scale);
+
+ std::string ToString() const override;
+ std::string name() const override { return "decimal256"; }
+
+ static constexpr int32_t kMinPrecision = 1;
+ static constexpr int32_t kMaxPrecision = 76;
+};
+
/// \brief Concrete type class for union data
class ARROW_EXPORT UnionType : public NestedType {
public:
diff --git a/cpp/src/arrow/type_fwd.h b/cpp/src/arrow/type_fwd.h
index fc25b27..e62a8ca 100644
--- a/cpp/src/arrow/type_fwd.h
+++ b/cpp/src/arrow/type_fwd.h
@@ -143,11 +143,16 @@ class StructBuilder;
struct StructScalar;
class Decimal128;
+class Decimal256;
class DecimalType;
class Decimal128Type;
+class Decimal256Type;
class Decimal128Array;
+class Decimal256Array;
class Decimal128Builder;
+class Decimal256Builder;
struct Decimal128Scalar;
+struct Decimal256Scalar;
struct UnionMode {
enum type { SPARSE, DENSE };
@@ -326,9 +331,14 @@ struct Type {
/// DAY_TIME interval in SQL style
INTERVAL_DAY_TIME,
- /// Precision- and scale-based decimal type. Storage type depends on the
- /// parameters.
- DECIMAL,
+ /// Precision- and scale-based decimal type with 128 bits.
+ DECIMAL128,
+
+ /// Defined for backward-compatibility.
+ DECIMAL = DECIMAL128,
+
+ /// Precision- and scale-based decimal type with 256 bits.
+ DECIMAL256,
/// A list of some logical data type
LIST,
@@ -423,10 +433,18 @@ std::shared_ptr<DataType> ARROW_EXPORT date64();
ARROW_EXPORT
std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width);
-/// \brief Create a Decimal128Type instance
+/// \brief Create a Decimal128Type or Decimal256Type instance depending on the precision
ARROW_EXPORT
std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale);
+/// \brief Create a Decimal128Type instance
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal128(int32_t precision, int32_t scale);
+
+/// \brief Create a Decimal256Type instance
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal256(int32_t precision, int32_t scale);
+
/// \brief Create a ListType instance from its child Field type
ARROW_EXPORT
std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_type);
diff --git a/cpp/src/arrow/type_test.cc b/cpp/src/arrow/type_test.cc
index e53d259..d5ece2e 100644
--- a/cpp/src/arrow/type_test.cc
+++ b/cpp/src/arrow/type_test.cc
@@ -1770,43 +1770,85 @@ TEST(TestDictionaryType, UnifyLarge) {
TEST(TypesTest, TestDecimal128Small) {
Decimal128Type t1(8, 4);
- ASSERT_EQ(t1.id(), Type::DECIMAL);
- ASSERT_EQ(t1.precision(), 8);
- ASSERT_EQ(t1.scale(), 4);
+ EXPECT_EQ(t1.id(), Type::DECIMAL128);
+ EXPECT_EQ(t1.precision(), 8);
+ EXPECT_EQ(t1.scale(), 4);
- ASSERT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
+ EXPECT_EQ(t1.ToString(), std::string("decimal(8, 4)"));
// Test properties
- ASSERT_EQ(t1.byte_width(), 16);
- ASSERT_EQ(t1.bit_width(), 128);
+ EXPECT_EQ(t1.byte_width(), 16);
+ EXPECT_EQ(t1.bit_width(), 128);
}
TEST(TypesTest, TestDecimal128Medium) {
Decimal128Type t1(12, 5);
- ASSERT_EQ(t1.id(), Type::DECIMAL);
- ASSERT_EQ(t1.precision(), 12);
- ASSERT_EQ(t1.scale(), 5);
+ EXPECT_EQ(t1.id(), Type::DECIMAL128);
+ EXPECT_EQ(t1.precision(), 12);
+ EXPECT_EQ(t1.scale(), 5);
- ASSERT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
+ EXPECT_EQ(t1.ToString(), std::string("decimal(12, 5)"));
// Test properties
- ASSERT_EQ(t1.byte_width(), 16);
- ASSERT_EQ(t1.bit_width(), 128);
+ EXPECT_EQ(t1.byte_width(), 16);
+ EXPECT_EQ(t1.bit_width(), 128);
}
TEST(TypesTest, TestDecimal128Large) {
Decimal128Type t1(27, 7);
- ASSERT_EQ(t1.id(), Type::DECIMAL);
- ASSERT_EQ(t1.precision(), 27);
- ASSERT_EQ(t1.scale(), 7);
+ EXPECT_EQ(t1.id(), Type::DECIMAL128);
+ EXPECT_EQ(t1.precision(), 27);
+ EXPECT_EQ(t1.scale(), 7);
- ASSERT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
+ EXPECT_EQ(t1.ToString(), std::string("decimal(27, 7)"));
// Test properties
- ASSERT_EQ(t1.byte_width(), 16);
- ASSERT_EQ(t1.bit_width(), 128);
+ EXPECT_EQ(t1.byte_width(), 16);
+ EXPECT_EQ(t1.bit_width(), 128);
+}
+
+TEST(TypesTest, TestDecimal256Small) {
+ Decimal256Type t1(8, 4);
+
+ EXPECT_EQ(t1.id(), Type::DECIMAL256);
+ EXPECT_EQ(t1.precision(), 8);
+ EXPECT_EQ(t1.scale(), 4);
+
+ EXPECT_EQ(t1.ToString(), std::string("decimal256(8, 4)"));
+
+ // Test properties
+ EXPECT_EQ(t1.byte_width(), 32);
+ EXPECT_EQ(t1.bit_width(), 256);
+}
+
+TEST(TypesTest, TestDecimal256Medium) {
+ Decimal256Type t1(12, 5);
+
+ EXPECT_EQ(t1.id(), Type::DECIMAL256);
+ EXPECT_EQ(t1.precision(), 12);
+ EXPECT_EQ(t1.scale(), 5);
+
+ EXPECT_EQ(t1.ToString(), std::string("decimal256(12, 5)"));
+
+ // Test properties
+ EXPECT_EQ(t1.byte_width(), 32);
+ EXPECT_EQ(t1.bit_width(), 256);
+}
+
+TEST(TypesTest, TestDecimal256Large) {
+ Decimal256Type t1(76, 38);
+
+ EXPECT_EQ(t1.id(), Type::DECIMAL256);
+ EXPECT_EQ(t1.precision(), 76);
+ EXPECT_EQ(t1.scale(), 38);
+
+ EXPECT_EQ(t1.ToString(), std::string("decimal256(76, 38)"));
+
+ // Test properties
+ EXPECT_EQ(t1.byte_width(), 32);
+ EXPECT_EQ(t1.bit_width(), 256);
}
TEST(TypesTest, TestDecimalEquals) {
@@ -1815,12 +1857,24 @@ TEST(TypesTest, TestDecimalEquals) {
Decimal128Type t3(8, 5);
Decimal128Type t4(27, 5);
+ Decimal256Type t5(8, 4);
+ Decimal256Type t6(8, 4);
+ Decimal256Type t7(8, 5);
+ Decimal256Type t8(27, 5);
+
FixedSizeBinaryType t9(16);
+ FixedSizeBinaryType t10(32);
AssertTypeEqual(t1, t2);
AssertTypeNotEqual(t1, t3);
AssertTypeNotEqual(t1, t4);
AssertTypeNotEqual(t1, t9);
+
+ AssertTypeEqual(t5, t6);
+ AssertTypeNotEqual(t5, t1);
+ AssertTypeNotEqual(t5, t7);
+ AssertTypeNotEqual(t5, t8);
+ AssertTypeNotEqual(t5, t10);
}
} // namespace arrow
diff --git a/cpp/src/arrow/type_traits.h b/cpp/src/arrow/type_traits.h
index 7cf9503..16f6a72 100644
--- a/cpp/src/arrow/type_traits.h
+++ b/cpp/src/arrow/type_traits.h
@@ -66,7 +66,8 @@ TYPE_ID_TRAIT(TIMESTAMP, TimestampType)
TYPE_ID_TRAIT(INTERVAL_DAY_TIME, DayTimeIntervalType)
TYPE_ID_TRAIT(INTERVAL_MONTHS, MonthIntervalType)
TYPE_ID_TRAIT(DURATION, DurationType)
-TYPE_ID_TRAIT(DECIMAL, Decimal128Type) // XXX or DecimalType?
+TYPE_ID_TRAIT(DECIMAL128, Decimal128Type)
+TYPE_ID_TRAIT(DECIMAL256, Decimal256Type)
TYPE_ID_TRAIT(STRUCT, StructType)
TYPE_ID_TRAIT(LIST, ListType)
TYPE_ID_TRAIT(LARGE_LIST, LargeListType)
@@ -289,6 +290,14 @@ struct TypeTraits<Decimal128Type> {
};
template <>
+struct TypeTraits<Decimal256Type> {
+ using ArrayType = Decimal256Array;
+ using BuilderType = Decimal256Builder;
+ using ScalarType = Decimal256Scalar;
+ constexpr static bool is_parameter_free = false;
+};
+
+template <>
struct TypeTraits<BinaryType> {
using ArrayType = BinaryArray;
using BuilderType = BinaryBuilder;
@@ -829,7 +838,8 @@ static inline bool is_dictionary(Type::type type_id) {
static inline bool is_fixed_size_binary(Type::type type_id) {
switch (type_id) {
- case Type::DECIMAL:
+ case Type::DECIMAL128:
+ case Type::DECIMAL256:
case Type::FIXED_SIZE_BINARY:
return true;
default:
diff --git a/cpp/src/arrow/util/basic_decimal.cc b/cpp/src/arrow/util/basic_decimal.cc
index e46834d..3e7daa3 100644
--- a/cpp/src/arrow/util/basic_decimal.cc
+++ b/cpp/src/arrow/util/basic_decimal.cc
@@ -725,4 +725,85 @@ int32_t BasicDecimal128::CountLeadingBinaryZeros() const {
}
}
+#if ARROW_LITTLE_ENDIAN
+BasicDecimal256::BasicDecimal256(const uint8_t* bytes)
+ : little_endian_array_(
+ std::array<uint64_t, 4>({reinterpret_cast<const uint64_t*>(bytes)[0],
+ reinterpret_cast<const uint64_t*>(bytes)[1],
+ reinterpret_cast<const uint64_t*>(bytes)[2],
+ reinterpret_cast<const uint64_t*>(bytes)[3]})) {}
+#else
+BasicDecimal256::BasicDecimal256(const uint8_t* bytes)
+ : little_endian_array_(
+ std::array<uint64_t, 4>({reinterpret_cast<const uint64_t*>(bytes)[3],
+ reinterpret_cast<const uint64_t*>(bytes)[2],
+ reinterpret_cast<const uint64_t*>(bytes)[1],
+ reinterpret_cast<const uint64_t*>(bytes)[0]})) {
+#endif
+
+BasicDecimal256& BasicDecimal256::Negate() {
+ uint64_t carry = 1;
+ for (uint64_t& elem : little_endian_array_) {
+ elem = ~elem + carry;
+ carry &= (elem == 0);
+ }
+ return *this;
+}
+
+std::array<uint8_t, 32> BasicDecimal256::ToBytes() const {
+ std::array<uint8_t, 32> out{{0}};
+ ToBytes(out.data());
+ return out;
+}
+
+void BasicDecimal256::ToBytes(uint8_t* out) const {
+ DCHECK_NE(out, nullptr);
+#if ARROW_LITTLE_ENDIAN
+ reinterpret_cast<int64_t*>(out)[0] = little_endian_array_[0];
+ reinterpret_cast<int64_t*>(out)[1] = little_endian_array_[1];
+ reinterpret_cast<int64_t*>(out)[2] = little_endian_array_[2];
+ reinterpret_cast<int64_t*>(out)[3] = little_endian_array_[3];
+#else
+ reinterpret_cast<int64_t*>(out)[0] = little_endian_array_[3];
+ reinterpret_cast<int64_t*>(out)[1] = little_endian_array_[2];
+ reinterpret_cast<int64_t*>(out)[2] = little_endian_array_[1];
+ reinterpret_cast<int64_t*>(out)[3] = little_endian_array_[0];
+#endif
+}
+
+DecimalStatus BasicDecimal256::Rescale(int32_t original_scale, int32_t new_scale,
+ BasicDecimal256* out) const {
+ // TODO: implement.
+ return DecimalStatus::kSuccess;
+}
+
+bool operator==(const BasicDecimal256& left, const BasicDecimal256& right) {
+ return left.little_endian_array() == right.little_endian_array();
+}
+
+bool operator!=(const BasicDecimal256& left, const BasicDecimal256& right) {
+ return left.little_endian_array() != right.little_endian_array();
+}
+
+bool operator<(const BasicDecimal256& left, const BasicDecimal256& right) {
+ const std::array<uint64_t, 4>& lhs = left.little_endian_array();
+ const std::array<uint64_t, 4>& rhs = right.little_endian_array();
+ return lhs[3] != rhs[3]
+ ? static_cast<int64_t>(lhs[3]) < static_cast<int64_t>(rhs[3])
+ : lhs[2] != rhs[2] ? lhs[2] < rhs[2]
+ : lhs[1] != rhs[1] ? lhs[1] < rhs[1] : lhs[0] < rhs[0];
+}
+
+bool operator<=(const BasicDecimal256& left, const BasicDecimal256& right) {
+ return !operator>(left, right);
+}
+
+bool operator>(const BasicDecimal256& left, const BasicDecimal256& right) {
+ return operator<(right, left);
+}
+
+bool operator>=(const BasicDecimal256& left, const BasicDecimal256& right) {
+ return !operator<(left, right);
+}
+
} // namespace arrow
diff --git a/cpp/src/arrow/util/basic_decimal.h b/cpp/src/arrow/util/basic_decimal.h
index 23c38bb..54c8808 100644
--- a/cpp/src/arrow/util/basic_decimal.h
+++ b/cpp/src/arrow/util/basic_decimal.h
@@ -178,4 +178,60 @@ ARROW_EXPORT BasicDecimal128 operator/(const BasicDecimal128& left,
ARROW_EXPORT BasicDecimal128 operator%(const BasicDecimal128& left,
const BasicDecimal128& right);
+class ARROW_EXPORT BasicDecimal256 {
+ public:
+ /// \brief Create a BasicDecimal256 from the two's complement representation.
+ constexpr BasicDecimal256(const std::array<uint64_t, 4>& little_endian_array) noexcept
+ : little_endian_array_(little_endian_array) {}
+
+ /// \brief Empty constructor creates a BasicDecimal256 with a value of 0.
+ constexpr BasicDecimal256() noexcept : little_endian_array_({0, 0, 0, 0}) {}
+
+ /// \brief Convert any integer value into a BasicDecimal256.
+ template <typename T,
+ typename = typename std::enable_if<
+ std::is_integral<T>::value && (sizeof(T) <= sizeof(uint64_t)), T>::type>
+ constexpr BasicDecimal256(T value) noexcept
+ : little_endian_array_({static_cast<uint64_t>(value), extend(value), extend(value),
+ extend(value)}) {}
+
+ /// \brief Create a BasicDecimal256 from an array of bytes. Bytes are assumed to be in
+ /// native-endian byte order.
+ explicit BasicDecimal256(const uint8_t* bytes);
+
+ /// \brief Negate the current value (in-place)
+ BasicDecimal256& Negate();
+
+ /// \brief Get the bits of the two's complement representation of the number. The 4
+ /// elements are in little endian order. The bits within each uint64_t element are in
+ /// native endian order. For example,
+ /// BasicDecimal256(123).little_endian_array() = {123, 0, 0, 0};
+ /// BasicDecimal256(-2).little_endian_array() = {0xFF...FE, 0xFF...FF, 0xFF...FF,
+ /// 0xFF...FF}.
+ inline const std::array<uint64_t, 4>& little_endian_array() const {
+ return little_endian_array_;
+ }
+
+ /// \brief Return the raw bytes of the value in native-endian byte order.
+ std::array<uint8_t, 32> ToBytes() const;
+ void ToBytes(uint8_t* out) const;
+
+ /// \brief Convert BasicDecimal128 from one scale to another
+ DecimalStatus Rescale(int32_t original_scale, int32_t new_scale,
+ BasicDecimal256* out) const;
+
+ private:
+ template <typename T>
+ inline static constexpr uint64_t extend(T low_bits) noexcept {
+ return low_bits >= T() ? uint64_t{0} : ~uint64_t{0};
+ }
+ std::array<uint64_t, 4> little_endian_array_;
+};
+
+ARROW_EXPORT bool operator==(const BasicDecimal256& left, const BasicDecimal256& right);
+ARROW_EXPORT bool operator!=(const BasicDecimal256& left, const BasicDecimal256& right);
+ARROW_EXPORT bool operator<(const BasicDecimal256& left, const BasicDecimal256& right);
+ARROW_EXPORT bool operator<=(const BasicDecimal256& left, const BasicDecimal256& right);
+ARROW_EXPORT bool operator>(const BasicDecimal256& left, const BasicDecimal256& right);
+ARROW_EXPORT bool operator>=(const BasicDecimal256& left, const BasicDecimal256& right);
} // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index 52a1da4..e2f3a07 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -445,6 +445,24 @@ bool ParseDecimalComponents(const char* s, size_t size, DecimalComponents* out)
return pos == size;
}
+inline Status ToArrowStatus(DecimalStatus dstatus, int num_bits) {
+ switch (dstatus) {
+ case DecimalStatus::kSuccess:
+ return Status::OK();
+
+ case DecimalStatus::kDivideByZero:
+ return Status::Invalid("Division by 0 in Decimal", num_bits);
+
+ case DecimalStatus::kOverflow:
+ return Status::Invalid("Overflow occurred during Decimal", num_bits, " operation.");
+
+ case DecimalStatus::kRescaleDataLoss:
+ return Status::Invalid("Rescaling Decimal", num_bits,
+ " value would cause data loss");
+ }
+ return Status::OK();
+}
+
} // namespace
Status Decimal128::FromString(const util::string_view& s, Decimal128* out,
@@ -598,31 +616,109 @@ Result<Decimal128> Decimal128::FromBigEndian(const uint8_t* bytes, int32_t lengt
}
Status Decimal128::ToArrowStatus(DecimalStatus dstatus) const {
- Status status;
+ return arrow::ToArrowStatus(dstatus, 128);
+}
- switch (dstatus) {
- case DecimalStatus::kSuccess:
- status = Status::OK();
- break;
+std::ostream& operator<<(std::ostream& os, const Decimal128& decimal) {
+ os << decimal.ToIntegerString();
+ return os;
+}
- case DecimalStatus::kDivideByZero:
- status = Status::Invalid("Division by 0 in Decimal128");
- break;
+Decimal256::Decimal256(const std::string& str) : Decimal256() {
+ *this = Decimal256::FromString(str).ValueOrDie();
+}
- case DecimalStatus::kOverflow:
- status = Status::Invalid("Overflow occurred during Decimal128 operation.");
- break;
+std::string Decimal256::ToIntegerString() const {
+ std::string result;
+ if (static_cast<int64_t>(little_endian_array()[3]) < 0) {
+ result.push_back('-');
+ Decimal256 abs = *this;
+ abs.Negate();
+ AppendLittleEndianArrayToString(abs.little_endian_array(), &result);
+ } else {
+ AppendLittleEndianArrayToString(little_endian_array(), &result);
+ }
+ return result;
+}
- case DecimalStatus::kRescaleDataLoss:
- status = Status::Invalid("Rescaling decimal value would cause data loss");
- break;
+std::string Decimal256::ToString(int32_t scale) const {
+ std::string str(ToIntegerString());
+ AdjustIntegerStringWithScale(scale, &str);
+ return str;
+}
+
+Status Decimal256::FromString(const util::string_view& s, Decimal256* out,
+ int32_t* precision, int32_t* scale) {
+ if (s.empty()) {
+ return Status::Invalid("Empty string cannot be converted to decimal");
+ }
+
+ DecimalComponents dec;
+ if (!ParseDecimalComponents(s.data(), s.size(), &dec)) {
+ return Status::Invalid("The string '", s, "' is not a valid decimal number");
}
- return status;
+
+ // Count number of significant digits (without leading zeros)
+ size_t first_non_zero = dec.whole_digits.find_first_not_of('0');
+ size_t significant_digits = dec.fractional_digits.size();
+ if (first_non_zero != std::string::npos) {
+ significant_digits += dec.whole_digits.size() - first_non_zero;
+ }
+
+ if (precision != nullptr) {
+ *precision = static_cast<int32_t>(significant_digits);
+ }
+
+ if (scale != nullptr) {
+ if (dec.has_exponent) {
+ auto adjusted_exponent = dec.exponent;
+ auto len = static_cast<int32_t>(significant_digits);
+ *scale = -adjusted_exponent + len - 1;
+ } else {
+ *scale = static_cast<int32_t>(dec.fractional_digits.size());
+ }
+ }
+
+ if (out != nullptr) {
+ std::array<uint64_t, 4> little_endian_array = {0, 0, 0, 0};
+ ShiftAndAdd(dec.whole_digits, little_endian_array.data(), little_endian_array.size());
+ ShiftAndAdd(dec.fractional_digits, little_endian_array.data(),
+ little_endian_array.size());
+ *out = Decimal256(little_endian_array);
+
+ if (dec.sign == '-') {
+ out->Negate();
+ }
+ }
+
+ return Status::OK();
}
-std::ostream& operator<<(std::ostream& os, const Decimal128& decimal) {
- os << decimal.ToIntegerString();
- return os;
+Status Decimal256::FromString(const std::string& s, Decimal256* out, int32_t* precision,
+ int32_t* scale) {
+ return FromString(util::string_view(s), out, precision, scale);
+}
+
+Status Decimal256::FromString(const char* s, Decimal256* out, int32_t* precision,
+ int32_t* scale) {
+ return FromString(util::string_view(s), out, precision, scale);
}
+Result<Decimal256> Decimal256::FromString(const util::string_view& s) {
+ Decimal256 out;
+ RETURN_NOT_OK(FromString(s, &out, nullptr, nullptr));
+ return std::move(out);
+}
+
+Result<Decimal256> Decimal256::FromString(const std::string& s) {
+ return FromString(util::string_view(s));
+}
+
+Result<Decimal256> Decimal256::FromString(const char* s) {
+ return FromString(util::string_view(s));
+}
+
+Status Decimal256::ToArrowStatus(DecimalStatus dstatus) const {
+ return arrow::ToArrowStatus(dstatus, 256);
+}
} // namespace arrow
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 1f72705..a883797 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -172,4 +172,64 @@ struct Decimal128::ToRealConversion<double> {
}
};
+/// Represents a signed 256-bit integer in two's complement.
+/// The max decimal precision that can be safely represented is
+/// 76 significant digits.
+///
+/// The implementation is split into two parts :
+///
+/// 1. BasicDecimal256
+/// - can be safely compiled to IR without references to libstdc++.
+/// 2. Decimal256
+/// - (TODO) has additional functionality on top of BasicDecimal256 to deal with
+/// strings and streams.
+class ARROW_EXPORT Decimal256 : public BasicDecimal256 {
+ public:
+ /// \cond FALSE
+ // (need to avoid a duplicate definition in Sphinx)
+ using BasicDecimal256::BasicDecimal256;
+ /// \endcond
+
+ /// \brief constructor creates a Decimal256 from a BasicDecimal128.
+ constexpr Decimal256(const BasicDecimal256& value) noexcept : BasicDecimal256(value) {}
+
+ /// \brief Parse the number from a base 10 string representation.
+ explicit Decimal256(const std::string& value);
+
+ /// \brief Empty constructor creates a Decimal256 with a value of 0.
+ // This is required on some older compilers.
+ constexpr Decimal256() noexcept : BasicDecimal256() {}
+
+ /// \brief Convert the Decimal256 value to a base 10 decimal string with the given
+ /// scale.
+ std::string ToString(int32_t scale) const;
+
+ /// \brief Convert the value to an integer string
+ std::string ToIntegerString() const;
+
+ /// \brief Convert a decimal string to a Decimal256 value, optionally including
+ /// precision and scale if they're passed in and not null.
+ static Status FromString(const util::string_view& s, Decimal256* out,
+ int32_t* precision, int32_t* scale = NULLPTR);
+ static Status FromString(const std::string& s, Decimal256* out, int32_t* precision,
+ int32_t* scale = NULLPTR);
+ static Status FromString(const char* s, Decimal256* out, int32_t* precision,
+ int32_t* scale = NULLPTR);
+ static Result<Decimal256> FromString(const util::string_view& s);
+ static Result<Decimal256> FromString(const std::string& s);
+ static Result<Decimal256> FromString(const char* s);
+
+ /// \brief Convert Decimal256 from one scale to another
+ Result<Decimal256> Rescale(int32_t original_scale, int32_t new_scale) const {
+ Decimal256 out;
+ auto dstatus = BasicDecimal256::Rescale(original_scale, new_scale, &out);
+ ARROW_RETURN_NOT_OK(ToArrowStatus(dstatus));
+ return std::move(out);
+ }
+
+ private:
+ /// Converts internal error code to Status
+ Status ToArrowStatus(DecimalStatus dstatus) const;
+};
+
} // namespace arrow
diff --git a/cpp/src/arrow/util/decimal_test.cc b/cpp/src/arrow/util/decimal_test.cc
index fc79a56..abd4c2a 100644
--- a/cpp/src/arrow/util/decimal_test.cc
+++ b/cpp/src/arrow/util/decimal_test.cc
@@ -1146,4 +1146,95 @@ TEST(Decimal128Test, FitsInPrecision) {
Decimal128("-100000000000000000000000000000000000000").FitsInPrecision(38));
}
+static constexpr std::array<uint64_t, 4> kSortedDecimal256Bits[] = {
+ {0, 0, 0, 0x8000000000000000ULL}, // min
+ {0xFFFFFFFFFFFFFFFEULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+ 0xFFFFFFFFFFFFFFFFULL}, // -2
+ {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+ 0xFFFFFFFFFFFFFFFFULL}, // -1
+ {0, 0, 0, 0},
+ {1, 0, 0, 0},
+ {2, 0, 0, 0},
+ {0xFFFFFFFFFFFFFFFFULL, 0, 0, 0},
+ {0, 1, 0, 0},
+ {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0, 0},
+ {0, 0, 1, 0},
+ {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0},
+ {0, 0, 0, 1},
+ {0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL, 0xFFFFFFFFFFFFFFFFULL,
+ 0x7FFFFFFFFFFFFFFFULL}, // max
+};
+
+TEST(Decimal256Test, TestComparators) {
+ constexpr size_t num_values =
+ sizeof(kSortedDecimal256Bits) / sizeof(kSortedDecimal256Bits[0]);
+ for (size_t i = 0; i < num_values; ++i) {
+ Decimal256 left(kSortedDecimal256Bits[i]);
+ for (size_t j = 0; j < num_values; ++j) {
+ Decimal256 right(kSortedDecimal256Bits[j]);
+ EXPECT_EQ(i == j, left == right);
+ EXPECT_EQ(i != j, left != right);
+ EXPECT_EQ(i < j, left < right);
+ EXPECT_EQ(i > j, left > right);
+ EXPECT_EQ(i <= j, left <= right);
+ EXPECT_EQ(i >= j, left >= right);
+ }
+ }
+}
+
+TEST(Decimal256Test, TestToBytesRoundTrip) {
+ for (const std::array<uint64_t, 4>& bits : kSortedDecimal256Bits) {
+ Decimal256 decimal(bits);
+ EXPECT_EQ(decimal, Decimal256(decimal.ToBytes().data()));
+ }
+}
+
+template <typename T>
+class Decimal256Test : public ::testing::Test {
+ public:
+ Decimal256Test() {}
+};
+
+using Decimal256Types =
+ ::testing::Types<char, unsigned char, short, unsigned short, // NOLINT
+ int, unsigned int, long, unsigned long, // NOLINT
+ long long, unsigned long long // NOLINT
+ >;
+
+TYPED_TEST_SUITE(Decimal256Test, Decimal256Types);
+
+TYPED_TEST(Decimal256Test, ConstructibleFromAnyIntegerType) {
+ using UInt64Array = std::array<uint64_t, 4>;
+ Decimal256 value(TypeParam{42});
+ EXPECT_EQ(UInt64Array({42, 0, 0, 0}), value.little_endian_array());
+
+ TypeParam max = std::numeric_limits<TypeParam>::max();
+ Decimal256 max_value(max);
+ EXPECT_EQ(UInt64Array({static_cast<uint64_t>(max), 0, 0, 0}),
+ max_value.little_endian_array());
+
+ TypeParam min = std::numeric_limits<TypeParam>::min();
+ Decimal256 min_value(min);
+ uint64_t high_bits = std::is_signed<TypeParam>::value ? ~uint64_t{0} : uint64_t{0};
+ EXPECT_EQ(UInt64Array({static_cast<uint64_t>(min), high_bits, high_bits, high_bits}),
+ min_value.little_endian_array());
+}
+
+TEST(Decimal256Test, ConstructibleFromBool) {
+ EXPECT_EQ(Decimal256(0), Decimal256(false));
+ EXPECT_EQ(Decimal256(1), Decimal256(true));
+}
+
+class Decimal256ToStringTest : public ::testing::TestWithParam<ToStringTestParam> {};
+
+TEST_P(Decimal256ToStringTest, ToString) {
+ const ToStringTestParam& data = GetParam();
+ const Decimal256 value(data.test_value);
+ const std::string printed_value = value.ToString(data.scale);
+ ASSERT_EQ(data.expected_string, printed_value);
+}
+
+INSTANTIATE_TEST_SUITE_P(Decimal256ToStringTest, Decimal256ToStringTest,
+ ::testing::ValuesIn(kToStringTestData));
+
} // namespace arrow
diff --git a/cpp/src/arrow/visitor.cc b/cpp/src/arrow/visitor.cc
index 0a452d5..8517850 100644
--- a/cpp/src/arrow/visitor.cc
+++ b/cpp/src/arrow/visitor.cc
@@ -67,6 +67,7 @@ ARRAY_VISITOR_DEFAULT(SparseUnionArray)
ARRAY_VISITOR_DEFAULT(DenseUnionArray)
ARRAY_VISITOR_DEFAULT(DictionaryArray)
ARRAY_VISITOR_DEFAULT(Decimal128Array)
+ARRAY_VISITOR_DEFAULT(Decimal256Array)
ARRAY_VISITOR_DEFAULT(ExtensionArray)
#undef ARRAY_VISITOR_DEFAULT
@@ -106,6 +107,7 @@ TYPE_VISITOR_DEFAULT(DayTimeIntervalType)
TYPE_VISITOR_DEFAULT(MonthIntervalType)
TYPE_VISITOR_DEFAULT(DurationType)
TYPE_VISITOR_DEFAULT(Decimal128Type)
+TYPE_VISITOR_DEFAULT(Decimal256Type)
TYPE_VISITOR_DEFAULT(ListType)
TYPE_VISITOR_DEFAULT(LargeListType)
TYPE_VISITOR_DEFAULT(MapType)
@@ -154,6 +156,7 @@ SCALAR_VISITOR_DEFAULT(DayTimeIntervalScalar)
SCALAR_VISITOR_DEFAULT(MonthIntervalScalar)
SCALAR_VISITOR_DEFAULT(DurationScalar)
SCALAR_VISITOR_DEFAULT(Decimal128Scalar)
+SCALAR_VISITOR_DEFAULT(Decimal256Scalar)
SCALAR_VISITOR_DEFAULT(ListScalar)
SCALAR_VISITOR_DEFAULT(LargeListScalar)
SCALAR_VISITOR_DEFAULT(MapScalar)
diff --git a/cpp/src/arrow/visitor.h b/cpp/src/arrow/visitor.h
index 7ab136c..0382e46 100644
--- a/cpp/src/arrow/visitor.h
+++ b/cpp/src/arrow/visitor.h
@@ -54,6 +54,7 @@ class ARROW_EXPORT ArrayVisitor {
virtual Status Visit(const MonthIntervalArray& array);
virtual Status Visit(const DurationArray& array);
virtual Status Visit(const Decimal128Array& array);
+ virtual Status Visit(const Decimal256Array& array);
virtual Status Visit(const ListArray& array);
virtual Status Visit(const LargeListArray& array);
virtual Status Visit(const MapArray& array);
@@ -96,6 +97,7 @@ class ARROW_EXPORT TypeVisitor {
virtual Status Visit(const DayTimeIntervalType& type);
virtual Status Visit(const DurationType& type);
virtual Status Visit(const Decimal128Type& type);
+ virtual Status Visit(const Decimal256Type& type);
virtual Status Visit(const ListType& type);
virtual Status Visit(const LargeListType& type);
virtual Status Visit(const MapType& type);
@@ -138,6 +140,7 @@ class ARROW_EXPORT ScalarVisitor {
virtual Status Visit(const MonthIntervalScalar& scalar);
virtual Status Visit(const DurationScalar& scalar);
virtual Status Visit(const Decimal128Scalar& scalar);
+ virtual Status Visit(const Decimal256Scalar& scalar);
virtual Status Visit(const ListScalar& scalar);
virtual Status Visit(const LargeListScalar& scalar);
virtual Status Visit(const MapScalar& scalar);
diff --git a/cpp/src/arrow/visitor_inline.h b/cpp/src/arrow/visitor_inline.h
index bff97fc..45193f2 100644
--- a/cpp/src/arrow/visitor_inline.h
+++ b/cpp/src/arrow/visitor_inline.h
@@ -68,6 +68,7 @@ namespace arrow {
ACTION(MonthInterval); \
ACTION(DayTimeInterval); \
ACTION(Decimal128); \
+ ACTION(Decimal256); \
ACTION(List); \
ACTION(LargeList); \
ACTION(Map); \
diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py
index 9e22cc0..07d0fe6 100644
--- a/python/pyarrow/__init__.py
+++ b/python/pyarrow/__init__.py
@@ -94,7 +94,7 @@ from pyarrow.lib import (null, bool_,
float16, float32, float64,
binary, string, utf8,
large_binary, large_string, large_utf8,
- decimal128,
+ decimal128, decimal256,
list_, large_list, map_, struct, union, dictionary,
field,
type_for_alias,
@@ -102,7 +102,7 @@ from pyarrow.lib import (null, bool_,
ListType, LargeListType, MapType, FixedSizeListType,
UnionType,
TimestampType, Time32Type, Time64Type, DurationType,
- FixedSizeBinaryType, Decimal128Type,
+ FixedSizeBinaryType, Decimal128Type, Decimal256Type,
BaseExtensionType, ExtensionType,
PyExtensionType, UnknownExtensionType,
register_extension_type, unregister_extension_type,
@@ -132,13 +132,13 @@ from pyarrow.lib import (null, bool_,
DictionaryArray,
Date32Array, Date64Array, TimestampArray,
Time32Array, Time64Array, DurationArray,
- Decimal128Array, StructArray, ExtensionArray,
+ Decimal128Array, Decimal256Array, StructArray, ExtensionArray,
scalar, NA, _NULL as NULL, Scalar,
NullScalar, BooleanScalar,
Int8Scalar, Int16Scalar, Int32Scalar, Int64Scalar,
UInt8Scalar, UInt16Scalar, UInt32Scalar, UInt64Scalar,
HalfFloatScalar, FloatScalar, DoubleScalar,
- Decimal128Scalar,
+ Decimal128Scalar, Decimal256Scalar,
ListScalar, LargeListScalar, FixedSizeListScalar,
Date32Scalar, Date64Scalar,
Time32Scalar, Time64Scalar,
@@ -290,6 +290,7 @@ LargeStringValue = _deprecate_scalar("LargeString", LargeStringScalar)
FixedSizeBinaryValue = _deprecate_scalar("FixedSizeBinary",
FixedSizeBinaryScalar)
Decimal128Value = _deprecate_scalar("Decimal128", Decimal128Scalar)
+Decimal256Value = _deprecate_scalar("Decimal256", Decimal256Scalar)
UnionValue = _deprecate_scalar("Union", UnionScalar)
StructValue = _deprecate_scalar("Struct", StructScalar)
DictionaryValue = _deprecate_scalar("Dictionary", DictionaryScalar)
diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi
index a9f1be2..87b1efe 100644
--- a/python/pyarrow/array.pxi
+++ b/python/pyarrow/array.pxi
@@ -1460,6 +1460,12 @@ cdef class Decimal128Array(FixedSizeBinaryArray):
Concrete class for Arrow arrays of decimal128 data type.
"""
+
+cdef class Decimal256Array(FixedSizeBinaryArray):
+ """
+ Concrete class for Arrow arrays of decimal256 data type.
+ """
+
cdef class BaseListArray(Array):
def flatten(self):
@@ -2250,7 +2256,8 @@ cdef dict _array_classes = {
_Type_LARGE_STRING: LargeStringArray,
_Type_DICTIONARY: DictionaryArray,
_Type_FIXED_SIZE_BINARY: FixedSizeBinaryArray,
- _Type_DECIMAL: Decimal128Array,
+ _Type_DECIMAL128: Decimal128Array,
+ _Type_DECIMAL256: Decimal256Array,
_Type_STRUCT: StructArray,
_Type_EXTENSION: ExtensionArray,
}
diff --git a/python/pyarrow/includes/libarrow.pxd b/python/pyarrow/includes/libarrow.pxd
index 9a3451e..fb1c1da 100644
--- a/python/pyarrow/includes/libarrow.pxd
+++ b/python/pyarrow/includes/libarrow.pxd
@@ -49,6 +49,11 @@ cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
c_string ToString(int32_t scale) const
+cdef extern from "arrow/util/decimal.h" namespace "arrow" nogil:
+ cdef cppclass CDecimal256" arrow::Decimal256":
+ c_string ToString(int32_t scale) const
+
+
cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CBuildInfo "arrow::BuildInfo":
@@ -86,7 +91,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
_Type_FLOAT" arrow::Type::FLOAT"
_Type_DOUBLE" arrow::Type::DOUBLE"
- _Type_DECIMAL" arrow::Type::DECIMAL"
+ _Type_DECIMAL128" arrow::Type::DECIMAL128"
+ _Type_DECIMAL256" arrow::Type::DECIMAL256"
_Type_DATE32" arrow::Type::DATE32"
_Type_DATE64" arrow::Type::DATE64"
@@ -356,6 +362,12 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
int precision()
int scale()
+ cdef cppclass CDecimal256Type \
+ " arrow::Decimal256Type"(CFixedSizeBinaryType):
+ CDecimal256Type(int precision, int scale)
+ int precision()
+ int scale()
+
cdef cppclass CField" arrow::Field":
cppclass CMergeOptions "arrow::Field::MergeOptions":
c_bool promote_nullability
@@ -541,6 +553,11 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
):
c_string FormatValue(int i)
+ cdef cppclass CDecimal256Array" arrow::Decimal256Array"(
+ CFixedSizeBinaryArray
+ ):
+ c_string FormatValue(int i)
+
cdef cppclass CListArray" arrow::ListArray"(CArray):
@staticmethod
CResult[shared_ptr[CArray]] FromArrays(
@@ -935,6 +952,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
cdef cppclass CDecimal128Scalar" arrow::Decimal128Scalar"(CScalar):
CDecimal128 value
+ cdef cppclass CDecimal256Scalar" arrow::Decimal256Scalar"(CScalar):
+ CDecimal256 value
+
cdef cppclass CDate32Scalar" arrow::Date32Scalar"(CScalar):
int32_t value
diff --git a/python/pyarrow/lib.pxd b/python/pyarrow/lib.pxd
index 8e06dcd..0073932 100644
--- a/python/pyarrow/lib.pxd
+++ b/python/pyarrow/lib.pxd
@@ -135,6 +135,11 @@ cdef class Decimal128Type(FixedSizeBinaryType):
const CDecimal128Type* decimal128_type
+cdef class Decimal256Type(FixedSizeBinaryType):
+ cdef:
+ const CDecimal256Type* decimal256_type
+
+
cdef class BaseExtensionType(DataType):
cdef:
const CExtensionType* ext_type
@@ -345,6 +350,10 @@ cdef class Decimal128Array(FixedSizeBinaryArray):
pass
+cdef class Decimal256Array(FixedSizeBinaryArray):
+ pass
+
+
cdef class StructArray(Array):
pass
diff --git a/python/pyarrow/lib.pyx b/python/pyarrow/lib.pyx
index 0f57ced..eba0f5a 100644
--- a/python/pyarrow/lib.pyx
+++ b/python/pyarrow/lib.pyx
@@ -73,7 +73,8 @@ Type_INT64 = _Type_INT64
Type_HALF_FLOAT = _Type_HALF_FLOAT
Type_FLOAT = _Type_FLOAT
Type_DOUBLE = _Type_DOUBLE
-Type_DECIMAL = _Type_DECIMAL
+Type_DECIMAL128 = _Type_DECIMAL128
+Type_DECIMAL256 = _Type_DECIMAL256
Type_DATE32 = _Type_DATE32
Type_DATE64 = _Type_DATE64
Type_TIMESTAMP = _Type_TIMESTAMP
diff --git a/python/pyarrow/public-api.pxi b/python/pyarrow/public-api.pxi
index 81c57b1..aa738f9 100644
--- a/python/pyarrow/public-api.pxi
+++ b/python/pyarrow/public-api.pxi
@@ -103,8 +103,10 @@ cdef api object pyarrow_wrap_data_type(
out = DurationType.__new__(DurationType)
elif type.get().id() == _Type_FIXED_SIZE_BINARY:
out = FixedSizeBinaryType.__new__(FixedSizeBinaryType)
- elif type.get().id() == _Type_DECIMAL:
+ elif type.get().id() == _Type_DECIMAL128:
out = Decimal128Type.__new__(Decimal128Type)
+ elif type.get().id() == _Type_DECIMAL256:
+ out = Decimal256Type.__new__(Decimal256Type)
elif type.get().id() == _Type_EXTENSION:
ext_type = <const CExtensionType*> type.get()
cpy_ext_type = dynamic_cast[_CPyExtensionTypePtr](ext_type)
diff --git a/python/pyarrow/scalar.pxi b/python/pyarrow/scalar.pxi
index cc06ca6..b4e6bcc 100644
--- a/python/pyarrow/scalar.pxi
+++ b/python/pyarrow/scalar.pxi
@@ -307,6 +307,26 @@ cdef class Decimal128Scalar(Scalar):
return None
+cdef class Decimal256Scalar(Scalar):
+ """
+ Concrete class for decimal256 scalars.
+ """
+
+ def as_py(self):
+ """
+ Return this value as a Python Decimal.
+ """
+ cdef:
+ CDecimal256Scalar* sp = <CDecimal256Scalar*> self.wrapped.get()
+ CDecimal256Type* dtype = <CDecimal256Type*> sp.type.get()
+ if sp.is_valid:
+ return _pydecimal.Decimal(
+ frombytes(sp.value.ToString(dtype.scale()))
+ )
+ else:
+ return None
+
+
cdef class Date32Scalar(Scalar):
"""
Concrete class for date32 scalars.
@@ -762,7 +782,8 @@ cdef dict _scalar_classes = {
_Type_HALF_FLOAT: HalfFloatScalar,
_Type_FLOAT: FloatScalar,
_Type_DOUBLE: DoubleScalar,
- _Type_DECIMAL: Decimal128Scalar,
+ _Type_DECIMAL128: Decimal128Scalar,
+ _Type_DECIMAL256: Decimal256Scalar,
_Type_DATE32: Date32Scalar,
_Type_DATE64: Date64Scalar,
_Type_TIME32: Time32Scalar,
diff --git a/python/pyarrow/tests/strategies.py b/python/pyarrow/tests/strategies.py
index 088f291..99fac67 100644
--- a/python/pyarrow/tests/strategies.py
+++ b/python/pyarrow/tests/strategies.py
@@ -60,12 +60,18 @@ floating_types = st.sampled_from([
pa.float32(),
pa.float64()
])
-decimal_type = st.builds(
+decimal128_type = st.builds(
pa.decimal128,
precision=st.integers(min_value=1, max_value=38),
scale=st.integers(min_value=1, max_value=38)
)
-numeric_types = st.one_of(integer_types, floating_types, decimal_type)
+decimal256_type = st.builds(
+ pa.decimal256,
+ precision=st.integers(min_value=1, max_value=76),
+ scale=st.integers(min_value=1, max_value=76)
+)
+numeric_types = st.one_of(integer_types, floating_types,
+ decimal128_type, decimal256_type)
date_types = st.sampled_from([
pa.date32(),
diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py
index feb84dd..aee1a93 100644
--- a/python/pyarrow/tests/test_array.py
+++ b/python/pyarrow/tests/test_array.py
@@ -1277,7 +1277,7 @@ def test_decimal_to_int_non_integer():
for case in non_integer_cases:
# test safe casting raises
- msg_regexp = 'Rescaling decimal value would cause data loss'
+ msg_regexp = 'Rescaling Decimal128 value would cause data loss'
with pytest.raises(pa.ArrowInvalid, match=msg_regexp):
_check_cast_case(case)
@@ -1296,8 +1296,8 @@ def test_decimal_to_decimal():
)
assert result.equals(expected)
- with pytest.raises(pa.ArrowInvalid,
- match='Rescaling decimal value would cause data loss'):
+ msg_regexp = 'Rescaling Decimal128 value would cause data loss'
+ with pytest.raises(pa.ArrowInvalid, match=msg_regexp):
result = arr.cast(pa.decimal128(9, 1))
result = arr.cast(pa.decimal128(9, 1), safe=False)
diff --git a/python/pyarrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py
index f62a941..d9deb88 100644
--- a/python/pyarrow/tests/test_convert_builtin.py
+++ b/python/pyarrow/tests/test_convert_builtin.py
@@ -1343,61 +1343,62 @@ def test_sequence_mixed_types_with_specified_type_fails():
def test_sequence_decimal():
data = [decimal.Decimal('1234.183'), decimal.Decimal('8094.234')]
- type = pa.decimal128(precision=7, scale=3)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=7, scale=3))
+ assert arr.to_pylist() == data
def test_sequence_decimal_different_precisions():
data = [
decimal.Decimal('1234234983.183'), decimal.Decimal('80943244.234')
]
- type = pa.decimal128(precision=13, scale=3)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=13, scale=3))
+ assert arr.to_pylist() == data
def test_sequence_decimal_no_scale():
data = [decimal.Decimal('1234234983'), decimal.Decimal('8094324')]
- type = pa.decimal128(precision=10)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=10))
+ assert arr.to_pylist() == data
def test_sequence_decimal_negative():
data = [decimal.Decimal('-1234.234983'), decimal.Decimal('-8.094324')]
- type = pa.decimal128(precision=10, scale=6)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=10, scale=6))
+ assert arr.to_pylist() == data
def test_sequence_decimal_no_whole_part():
data = [decimal.Decimal('-.4234983'), decimal.Decimal('.0103943')]
- type = pa.decimal128(precision=7, scale=7)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=7, scale=7))
+ assert arr.to_pylist() == data
def test_sequence_decimal_large_integer():
data = [decimal.Decimal('-394029506937548693.42983'),
decimal.Decimal('32358695912932.01033')]
- type = pa.decimal128(precision=23, scale=5)
- arr = pa.array(data, type=type)
- assert arr.to_pylist() == data
+ for type in [pa.decimal128, pa.decimal256]:
+ arr = pa.array(data, type=type(precision=23, scale=5))
+ assert arr.to_pylist() == data
def test_sequence_decimal_from_integers():
data = [0, 1, -39402950693754869342983]
expected = [decimal.Decimal(x) for x in data]
+ # TODO: update this test after scaling implementation.
type = pa.decimal128(precision=28, scale=5)
arr = pa.array(data, type=type)
assert arr.to_pylist() == expected
def test_sequence_decimal_too_high_precision():
- # ARROW-6989 python decimal created from float has too high precision
+ # ARROW-6989 python decimal has too high precision
with pytest.raises(ValueError, match="precision out of range"):
- pa.array([decimal.Decimal(123.234)])
+ pa.array([decimal.Decimal('1' * 80)])
def test_range_types():
diff --git a/python/pyarrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py
index 091ae38..8c49570 100644
--- a/python/pyarrow/tests/test_scalars.py
+++ b/python/pyarrow/tests/test_scalars.py
@@ -43,6 +43,8 @@ import pyarrow as pa
(np.float16(1.0), pa.float16(), pa.HalfFloatScalar, pa.HalfFloatValue),
(1.0, pa.float32(), pa.FloatScalar, pa.FloatValue),
(decimal.Decimal("1.123"), None, pa.Decimal128Scalar, pa.Decimal128Value),
+ (decimal.Decimal("1.1234567890123456789012345678901234567890"),
+ None, pa.Decimal256Scalar, pa.Decimal256Value),
("string", None, pa.StringScalar, pa.StringValue),
(b"bytes", None, pa.BinaryScalar, pa.BinaryValue),
("largestring", pa.large_string(), pa.LargeStringScalar,
@@ -176,7 +178,7 @@ def test_numerics():
assert s.as_py() == 0.5
-def test_decimal():
+def test_decimal128():
v = decimal.Decimal("1.123")
s = pa.scalar(v)
assert isinstance(s, pa.Decimal128Scalar)
@@ -194,6 +196,25 @@ def test_decimal():
assert s.as_py() == v
+def test_decimal256():
+ v = decimal.Decimal("1234567890123456789012345678901234567890.123")
+ s = pa.scalar(v)
+ assert isinstance(s, pa.Decimal256Scalar)
+ assert s.as_py() == v
+ assert s.type == pa.decimal256(43, 3)
+
+ v = decimal.Decimal("1.1234")
+ with pytest.raises(pa.ArrowInvalid):
+ pa.scalar(v, type=pa.decimal256(4, scale=3))
+ # TODO: Add the following after implementing Decimal256 scaling.
+ # with pytest.raises(pa.ArrowInvalid):
+ # pa.scalar(v, type=pa.decimal256(5, scale=3))
+
+ s = pa.scalar(v, type=pa.decimal256(5, scale=4))
+ assert isinstance(s, pa.Decimal256Scalar)
+ assert s.as_py() == v
+
+
def test_date():
# ARROW-5125
d1 = datetime.date(3200, 1, 1)
diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py
index a642573..48b342d 100644
--- a/python/pyarrow/tests/test_schema.py
+++ b/python/pyarrow/tests/test_schema.py
@@ -603,6 +603,7 @@ def test_type_schema_pickling():
pa.timestamp('ms'),
pa.timestamp('ns'),
pa.decimal128(12, 2),
+ pa.decimal256(76, 38),
pa.field('a', 'string', metadata={b'foo': b'bar'})
]
diff --git a/python/pyarrow/tests/test_types.py b/python/pyarrow/tests/test_types.py
index c52751e..75d19d2 100644
--- a/python/pyarrow/tests/test_types.py
+++ b/python/pyarrow/tests/test_types.py
@@ -52,6 +52,7 @@ def get_many_types():
pa.float32(),
pa.float64(),
pa.decimal128(19, 4),
+ pa.decimal256(76, 38),
pa.string(),
pa.binary(),
pa.binary(10),
@@ -123,8 +124,21 @@ def test_null_field_may_not_be_non_nullable():
def test_is_decimal():
- assert types.is_decimal(pa.decimal128(19, 4))
- assert not types.is_decimal(pa.int32())
+ decimal128 = pa.decimal128(19, 4)
+ decimal256 = pa.decimal256(76, 38)
+ int32 = pa.int32()
+
+ assert types.is_decimal(decimal128)
+ assert types.is_decimal(decimal256)
+ assert not types.is_decimal(int32)
+
+ assert types.is_decimal128(decimal128)
+ assert not types.is_decimal128(decimal256)
+ assert not types.is_decimal128(int32)
+
+ assert not types.is_decimal256(decimal128)
+ assert types.is_decimal256(decimal256)
+ assert not types.is_decimal256(int32)
def test_is_list():
@@ -684,6 +698,7 @@ def test_bit_width():
(pa.uint32(), 32),
(pa.float16(), 16),
(pa.decimal128(19, 4), 128),
+ (pa.decimal256(76, 38), 256),
(pa.binary(42), 42 * 8)]:
assert ty.bit_width == expected
for ty in [pa.binary(), pa.string(), pa.list_(pa.int16())]:
@@ -701,6 +716,10 @@ def test_decimal_properties():
assert ty.byte_width == 16
assert ty.precision == 19
assert ty.scale == 4
+ ty = pa.decimal256(76, 38)
+ assert ty.byte_width == 32
+ assert ty.precision == 76
+ assert ty.scale == 38
def test_decimal_overflow():
@@ -708,7 +727,13 @@ def test_decimal_overflow():
pa.decimal128(38, 0)
for i in (0, -1, 39):
with pytest.raises(ValueError):
- pa.decimal128(39, 0)
+ pa.decimal128(i, 0)
+
+ pa.decimal256(1, 0)
+ pa.decimal256(76, 0)
+ for i in (0, -1, 77):
+ with pytest.raises(ValueError):
+ pa.decimal256(i, 0)
def test_type_equality_operators():
diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi
index 466e25a..f1333ec 100644
--- a/python/pyarrow/types.pxi
+++ b/python/pyarrow/types.pxi
@@ -46,7 +46,7 @@ cdef dict _pandas_type_map = {
_Type_FIXED_SIZE_BINARY: np.object_,
_Type_STRING: np.object_,
_Type_LIST: np.object_,
- _Type_DECIMAL: np.object_,
+ _Type_DECIMAL128: np.object_,
}
cdef dict _pep3118_type_map = {
@@ -623,6 +623,33 @@ cdef class Decimal128Type(FixedSizeBinaryType):
return self.decimal128_type.scale()
+cdef class Decimal256Type(FixedSizeBinaryType):
+ """
+ Concrete class for Decimal256 data types.
+ """
+
+ cdef void init(self, const shared_ptr[CDataType]& type) except *:
+ FixedSizeBinaryType.init(self, type)
+ self.decimal256_type = <const CDecimal256Type*> type.get()
+
+ def __reduce__(self):
+ return decimal256, (self.precision, self.scale)
+
+ @property
+ def precision(self):
+ """
+ The decimal precision, in number of decimal digits (an integer).
+ """
+ return self.decimal256_type.precision()
+
+ @property
+ def scale(self):
+ """
+ The decimal scale (an integer).
+ """
+ return self.decimal256_type.scale()
+
+
cdef class BaseExtensionType(DataType):
"""
Concrete base class for extension types.
@@ -2087,6 +2114,26 @@ cpdef DataType decimal128(int precision, int scale=0):
return pyarrow_wrap_data_type(decimal_type)
+cpdef DataType decimal256(int precision, int scale=0):
+ """
+ Create decimal type with precision and scale and 256bit width.
+
+ Parameters
+ ----------
+ precision : int
+ scale : int
+
+ Returns
+ -------
+ decimal_type : Decimal256Type
+ """
+ cdef shared_ptr[CDataType] decimal_type
+ if precision < 1 or precision > 76:
+ raise ValueError("precision should be between 1 and 76")
+ decimal_type.reset(new CDecimal256Type(precision, scale))
+ return pyarrow_wrap_data_type(decimal_type)
+
+
def string():
"""
Create UTF8 variable-length string type.
diff --git a/python/pyarrow/types.py b/python/pyarrow/types.py
index 6679154..708e2bc 100644
--- a/python/pyarrow/types.py
+++ b/python/pyarrow/types.py
@@ -31,6 +31,7 @@ _UNSIGNED_INTEGER_TYPES = {lib.Type_UINT8, lib.Type_UINT16, lib.Type_UINT32,
lib.Type_UINT64}
_INTEGER_TYPES = _SIGNED_INTEGER_TYPES | _UNSIGNED_INTEGER_TYPES
_FLOATING_TYPES = {lib.Type_HALF_FLOAT, lib.Type_FLOAT, lib.Type_DOUBLE}
+_DECIMAL_TYPES = {lib.Type_DECIMAL128, lib.Type_DECIMAL256}
_DATE_TYPES = {lib.Type_DATE32, lib.Type_DATE64}
_TIME_TYPES = {lib.Type_TIME32, lib.Type_TIME64}
_TEMPORAL_TYPES = {lib.Type_TIMESTAMP,
@@ -325,7 +326,21 @@ def is_decimal(t):
"""
Return True if value is an instance of a decimal type.
"""
- return t.id == lib.Type_DECIMAL
+ return t.id in _DECIMAL_TYPES
+
+
+def is_decimal128(t):
+ """
+ Return True if value is an instance of a decimal type.
+ """
+ return t.id == lib.Type_DECIMAL128
+
+
+def is_decimal256(t):
+ """
+ Return True if value is an instance of a decimal type.
+ """
+ return t.id == lib.Type_DECIMAL256
def is_dictionary(t):
diff --git a/r/R/enums.R b/r/R/enums.R
index 0590571..14910bc 100644
--- a/r/R/enums.R
+++ b/r/R/enums.R
@@ -66,18 +66,19 @@ Type <- enum("Type::type",
INTERVAL_MONTHS = 21L,
INTERVAL_DAY_TIME = 22L,
DECIMAL = 23L,
- LIST = 24L,
- STRUCT = 25L,
- SPARSE_UNION = 26L,
- DENSE_UNION = 27L,
- DICTIONARY = 28L,
- MAP = 29L,
- EXTENSION = 30L,
- FIXED_SIZE_LIST = 31L,
- DURATION = 32L,
- LARGE_STRING = 33L,
- LARGE_BINARY = 34L,
- LARGE_LIST = 35L
+ DECIMAL256 = 24L,
+ LIST = 25L,
+ STRUCT = 26L,
+ SPARSE_UNION = 27L,
+ DENSE_UNION = 28L,
+ DICTIONARY = 29L,
+ MAP = 30L,
+ EXTENSION = 31L,
+ FIXED_SIZE_LIST = 32L,
+ DURATION = 33L,
+ LARGE_STRING = 34L,
+ LARGE_BINARY = 35L,
+ LARGE_LIST = 36L
)
#' @rdname enums