You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/05 21:21:13 UTC
arrow git commit: ARROW-696: [C++] Support decimals in IPC and JSON
reader/writer to enable integration tests
Repository: arrow
Updated Branches:
refs/heads/master bea5ab037 -> 243328931
ARROW-696: [C++] Support decimals in IPC and JSON reader/writer to enable integration tests
Also adds `arrow::decimal` function and renames `decimal` namespace to `DecimalUtil`. The decimal array tests have been consolidated in array-test.cc. Fixed a signedness bug in HexEncode used in JSON write and pretty printing
Author: Wes McKinney <we...@twosigma.com>
Closes #1049 from wesm/ARROW-696 and squashes the following commits:
73b767cf [Wes McKinney] Support + test decimals in JSON reader/writer
c8ca538e [Wes McKinney] Implement IPC roundtrip for decimals
9f7cef6b [Wes McKinney] Some decimal API refinements. Add failing IPC test for decimals
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/24332893
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/24332893
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/24332893
Branch: refs/heads/master
Commit: 2433289314249ec8ad4b6fb503fde782372ac8df
Parents: bea5ab0
Author: Wes McKinney <we...@twosigma.com>
Authored: Tue Sep 5 17:21:08 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Tue Sep 5 17:21:08 2017 -0400
----------------------------------------------------------------------
cpp/src/arrow/CMakeLists.txt | 1 -
cpp/src/arrow/array-decimal-test.cc | 132 --------------------------
cpp/src/arrow/array-test.cc | 110 +++++++++++++++++++--
cpp/src/arrow/array.cc | 4 +-
cpp/src/arrow/builder.cc | 2 +-
cpp/src/arrow/builder.h | 11 +--
cpp/src/arrow/gpu/cuda_context.cc | 2 +-
cpp/src/arrow/ipc/ipc-json-test.cc | 3 +-
cpp/src/arrow/ipc/ipc-read-write-test.cc | 2 +-
cpp/src/arrow/ipc/json-internal.cc | 42 +++++---
cpp/src/arrow/ipc/metadata-internal.cc | 13 ++-
cpp/src/arrow/ipc/reader.cc | 2 -
cpp/src/arrow/ipc/test-common.h | 35 ++++++-
cpp/src/arrow/ipc/writer.cc | 1 +
cpp/src/arrow/pretty_print.cc | 5 +-
cpp/src/arrow/python/arrow_to_pandas.cc | 4 +-
cpp/src/arrow/python/builtin_convert.cc | 4 +-
cpp/src/arrow/python/helpers.cc | 2 +-
cpp/src/arrow/python/pandas_to_arrow.cc | 4 +-
cpp/src/arrow/status.h | 20 ++--
cpp/src/arrow/type.cc | 2 +-
cpp/src/arrow/type.h | 37 +++++---
cpp/src/arrow/util/bit-util.cc | 4 +-
cpp/src/arrow/util/bit-util.h | 5 +-
cpp/src/arrow/util/decimal-test.cc | 20 ++--
cpp/src/arrow/util/decimal.cc | 4 +-
cpp/src/arrow/util/decimal.h | 6 +-
cpp/src/arrow/util/int128.cc | 2 -
cpp/src/arrow/util/int128.h | 2 -
cpp/src/arrow/util/string.h | 2 +-
integration/integration_test.py | 2 +-
31 files changed, 256 insertions(+), 229 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
index 55fab2d..6963b11 100644
--- a/cpp/src/arrow/CMakeLists.txt
+++ b/cpp/src/arrow/CMakeLists.txt
@@ -50,7 +50,6 @@ install(
ADD_ARROW_TEST(allocator-test)
ADD_ARROW_TEST(array-test)
-ADD_ARROW_TEST(array-decimal-test)
ADD_ARROW_TEST(buffer-test)
ADD_ARROW_TEST(memory_pool-test)
ADD_ARROW_TEST(pretty_print-test)
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array-decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-decimal-test.cc b/cpp/src/arrow/array-decimal-test.cc
deleted file mode 100644
index 5f3a4f1..0000000
--- a/cpp/src/arrow/array-decimal-test.cc
+++ /dev/null
@@ -1,132 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include <cstddef>
-#include <cstdint>
-#include <memory>
-#include <vector>
-
-#include <gtest/gtest.h>
-
-#include "arrow/array.h"
-#include "arrow/builder.h"
-#include "arrow/test-util.h"
-#include "arrow/util/decimal.h"
-#include "arrow/util/int128.h"
-
-using std::size_t;
-
-namespace arrow {
-
-class Buffer;
-
-namespace decimal {
-
-using DecimalVector = std::vector<Int128>;
-
-class DecimalTest : public ::testing::TestWithParam<int> {
- public:
- DecimalTest() {}
-
- template <size_t BYTE_WIDTH = 16>
- void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
- out->reserve(input.size() * BYTE_WIDTH);
-
- std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
-
- for (const auto& value : input) {
- ASSERT_OK(value.ToBytes(&bytes));
- out->insert(out->end(), bytes.cbegin(), bytes.cend());
- }
- }
-
- template <size_t BYTE_WIDTH = 16>
- void TestCreate(int precision, const DecimalVector& draw,
- const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
- auto type = std::make_shared<DecimalType>(precision, 4);
-
- auto builder = std::make_shared<DecimalBuilder>(type);
-
- size_t null_count = 0;
-
- const size_t size = draw.size();
-
- ASSERT_OK(builder->Reserve(size));
-
- for (size_t i = 0; i < size; ++i) {
- if (valid_bytes[i]) {
- ASSERT_OK(builder->Append(draw[i]));
- } else {
- ASSERT_OK(builder->AppendNull());
- ++null_count;
- }
- }
-
- std::shared_ptr<Array> out;
- ASSERT_OK(builder->Finish(&out));
-
- std::vector<uint8_t> raw_bytes;
-
- raw_bytes.reserve(size * BYTE_WIDTH);
- MakeData<BYTE_WIDTH>(draw, &raw_bytes);
-
- auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH);
- std::shared_ptr<Buffer> expected_null_bitmap;
- ASSERT_OK(BitUtil::BytesToBits(valid_bytes, &expected_null_bitmap));
-
- int64_t expected_null_count = test::null_count(valid_bytes);
- auto expected = std::make_shared<DecimalArray>(
- type, size, expected_data, expected_null_bitmap, expected_null_count);
-
- std::shared_ptr<Array> lhs = out->Slice(offset);
- std::shared_ptr<Array> rhs = expected->Slice(offset);
- bool result = lhs->Equals(rhs);
- ASSERT_TRUE(result);
- }
-};
-
-TEST_P(DecimalTest, NoNulls) {
- int precision = GetParam();
- std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
- Int128(-12348)};
- std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
- this->TestCreate(precision, draw, valid_bytes, 0);
- this->TestCreate(precision, draw, valid_bytes, 2);
-}
-
-TEST_P(DecimalTest, WithNulls) {
- int precision = GetParam();
- std::vector<Int128> draw = {Int128(1), Int128(2), Int128(-1), Int128(4),
- Int128(-1), Int128(1), Int128(2)};
- Int128 big;
- ASSERT_OK(FromString("230342903942.234234", &big));
- draw.push_back(big);
-
- Int128 big_negative;
- ASSERT_OK(FromString("-23049302932.235234", &big_negative));
- draw.push_back(big_negative);
-
- std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
- true, true, true, true};
- this->TestCreate(precision, draw, valid_bytes, 0);
- this->TestCreate(precision, draw, valid_bytes, 2);
-}
-
-INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
-
-} // namespace decimal
-} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
index 3a3d28c..a73a73e 100644
--- a/cpp/src/arrow/array-test.cc
+++ b/cpp/src/arrow/array-test.cc
@@ -33,6 +33,8 @@
#include "arrow/test-util.h"
#include "arrow/type.h"
#include "arrow/type_traits.h"
+#include "arrow/util/decimal.h"
+#include "arrow/util/int128.h"
namespace arrow {
@@ -69,7 +71,7 @@ Status MakeArrayFromValidBytes(const vector<uint8_t>& v, MemoryPool* pool,
int64_t null_count = v.size() - std::accumulate(v.begin(), v.end(), 0);
std::shared_ptr<Buffer> null_buf;
- RETURN_NOT_OK(BitUtil::BytesToBits(v, &null_buf));
+ RETURN_NOT_OK(BitUtil::BytesToBits(v, default_memory_pool(), &null_buf));
TypedBufferBuilder<int32_t> value_builder(pool);
for (size_t i = 0; i < v.size(); ++i) {
@@ -157,7 +159,7 @@ TEST_F(TestArray, TestIsNull) {
}
std::shared_ptr<Buffer> null_buf;
- ASSERT_OK(BitUtil::BytesToBits(null_bitmap, &null_buf));
+ ASSERT_OK(BitUtil::BytesToBits(null_bitmap, default_memory_pool(), &null_buf));
std::unique_ptr<Array> arr;
arr.reset(new Int32Array(null_bitmap.size(), nullptr, null_buf, null_count));
@@ -236,7 +238,8 @@ class TestPrimitiveBuilder : public TestBuilder {
int64_t ex_null_count = 0;
if (nullable) {
- ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
+ ASSERT_OK(
+ BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap));
ex_null_count = test::null_count(valid_bytes_);
} else {
ex_null_bitmap = nullptr;
@@ -327,13 +330,13 @@ void TestPrimitiveBuilder<PBoolean>::Check(const std::unique_ptr<BooleanBuilder>
int64_t size = builder->length();
std::shared_ptr<Buffer> ex_data;
- ASSERT_OK(BitUtil::BytesToBits(draws_, &ex_data));
+ ASSERT_OK(BitUtil::BytesToBits(draws_, default_memory_pool(), &ex_data));
std::shared_ptr<Buffer> ex_null_bitmap;
int64_t ex_null_count = 0;
if (nullable) {
- ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &ex_null_bitmap));
+ ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &ex_null_bitmap));
ex_null_count = test::null_count(valid_bytes_);
} else {
ex_null_bitmap = nullptr;
@@ -753,7 +756,7 @@ class TestStringArray : public ::testing::Test {
length_ = static_cast<int64_t>(offsets_.size()) - 1;
value_buf_ = test::GetBufferFromVector(chars_);
offsets_buf_ = test::GetBufferFromVector(offsets_);
- ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
+ ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_));
null_count_ = test::null_count(valid_bytes_);
strings_ = std::make_shared<StringArray>(length_, offsets_buf_, value_buf_,
@@ -967,7 +970,7 @@ class TestBinaryArray : public ::testing::Test {
value_buf_ = test::GetBufferFromVector(chars_);
offsets_buf_ = test::GetBufferFromVector(offsets_);
- ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, &null_bitmap_));
+ ASSERT_OK(BitUtil::BytesToBits(valid_bytes_, default_memory_pool(), &null_bitmap_));
null_count_ = test::null_count(valid_bytes_);
strings_ = std::make_shared<BinaryArray>(length_, offsets_buf_, value_buf_,
@@ -2464,4 +2467,97 @@ TEST(TestUnionArrayAdHoc, TestSliceEquals) {
CheckUnion(batch->column(2));
}
+using DecimalVector = std::vector<Int128>;
+
+class DecimalTest : public ::testing::TestWithParam<int> {
+ public:
+ DecimalTest() {}
+
+ template <size_t BYTE_WIDTH = 16>
+ void MakeData(const DecimalVector& input, std::vector<uint8_t>* out) const {
+ out->reserve(input.size() * BYTE_WIDTH);
+
+ std::array<uint8_t, BYTE_WIDTH> bytes{{0}};
+
+ for (const auto& value : input) {
+ ASSERT_OK(value.ToBytes(&bytes));
+ out->insert(out->end(), bytes.cbegin(), bytes.cend());
+ }
+ }
+
+ template <size_t BYTE_WIDTH = 16>
+ void TestCreate(int32_t precision, const DecimalVector& draw,
+ const std::vector<uint8_t>& valid_bytes, int64_t offset) const {
+ auto type = std::make_shared<DecimalType>(precision, 4);
+
+ auto builder = std::make_shared<DecimalBuilder>(type);
+
+ size_t null_count = 0;
+
+ const size_t size = draw.size();
+
+ ASSERT_OK(builder->Reserve(size));
+
+ for (size_t i = 0; i < size; ++i) {
+ if (valid_bytes[i]) {
+ ASSERT_OK(builder->Append(draw[i]));
+ } else {
+ ASSERT_OK(builder->AppendNull());
+ ++null_count;
+ }
+ }
+
+ std::shared_ptr<Array> out;
+ ASSERT_OK(builder->Finish(&out));
+
+ std::vector<uint8_t> raw_bytes;
+
+ raw_bytes.reserve(size * BYTE_WIDTH);
+ MakeData<BYTE_WIDTH>(draw, &raw_bytes);
+
+ auto expected_data = std::make_shared<Buffer>(raw_bytes.data(), BYTE_WIDTH);
+ std::shared_ptr<Buffer> expected_null_bitmap;
+ ASSERT_OK(
+ BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &expected_null_bitmap));
+
+ int64_t expected_null_count = test::null_count(valid_bytes);
+ auto expected = std::make_shared<DecimalArray>(
+ type, size, expected_data, expected_null_bitmap, expected_null_count);
+
+ std::shared_ptr<Array> lhs = out->Slice(offset);
+ std::shared_ptr<Array> rhs = expected->Slice(offset);
+ bool result = lhs->Equals(rhs);
+ ASSERT_TRUE(result);
+ }
+};
+
+TEST_P(DecimalTest, NoNulls) {
+ int32_t precision = GetParam();
+ std::vector<Int128> draw = {Int128(1), Int128(-2), Int128(2389), Int128(4),
+ Int128(-12348)};
+ std::vector<uint8_t> valid_bytes = {true, true, true, true, true};
+ this->TestCreate(precision, draw, valid_bytes, 0);
+ this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+TEST_P(DecimalTest, WithNulls) {
+ int32_t precision = GetParam();
+ std::vector<Int128> draw = {Int128(1), Int128(2), Int128(-1), Int128(4),
+ Int128(-1), Int128(1), Int128(2)};
+ Int128 big;
+ ASSERT_OK(DecimalUtil::FromString("230342903942.234234", &big));
+ draw.push_back(big);
+
+ Int128 big_negative;
+ ASSERT_OK(DecimalUtil::FromString("-23049302932.235234", &big_negative));
+ draw.push_back(big_negative);
+
+ std::vector<uint8_t> valid_bytes = {true, true, false, true, false,
+ true, true, true, true};
+ this->TestCreate(precision, draw, valid_bytes, 0);
+ this->TestCreate(precision, draw, valid_bytes, 2);
+}
+
+INSTANTIATE_TEST_CASE_P(DecimalTest, DecimalTest, ::testing::Range(1, 38));
+
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/array.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
index babf326..34f0868 100644
--- a/cpp/src/arrow/array.cc
+++ b/cpp/src/arrow/array.cc
@@ -319,8 +319,8 @@ DecimalArray::DecimalArray(const std::shared_ptr<internal::ArrayData>& data)
std::string DecimalArray::FormatValue(int64_t i) const {
const auto& type_ = static_cast<const DecimalType&>(*type());
- decimal::Int128 value(GetValue(i));
- return decimal::ToString(value, type_.precision(), type_.scale());
+ Int128 value(GetValue(i));
+ return DecimalUtil::ToString(value, type_.precision(), type_.scale());
}
// ----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/builder.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc
index 24d26d4..0129dc8 100644
--- a/cpp/src/arrow/builder.cc
+++ b/cpp/src/arrow/builder.cc
@@ -1032,7 +1032,7 @@ DecimalBuilder::DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>
: DecimalBuilder(type, pool) {}
#endif
-Status DecimalBuilder::Append(const decimal::Int128& value) {
+Status DecimalBuilder::Append(const Int128& value) {
RETURN_NOT_OK(FixedSizeBinaryBuilder::Reserve(1));
std::array<uint8_t, 16> bytes;
RETURN_NOT_OK(value.ToBytes(&bytes));
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/builder.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/builder.h b/cpp/src/arrow/builder.h
index a649d62..a99e682 100644
--- a/cpp/src/arrow/builder.h
+++ b/cpp/src/arrow/builder.h
@@ -38,6 +38,7 @@
namespace arrow {
class Array;
+class Int128;
namespace internal {
@@ -45,12 +46,6 @@ struct ArrayData;
} // namespace internal
-namespace decimal {
-
-class Int128;
-
-} // namespace decimal
-
static constexpr int64_t kMinBuilderCapacity = 1 << 5;
/// Base class for all data array builders.
@@ -747,7 +742,9 @@ class ARROW_EXPORT DecimalBuilder : public FixedSizeBinaryBuilder {
explicit DecimalBuilder(MemoryPool* pool, const std::shared_ptr<DataType>& type);
#endif
- Status Append(const decimal::Int128& val);
+ using FixedSizeBinaryBuilder::Append;
+
+ Status Append(const Int128& val);
Status Finish(std::shared_ptr<Array>* out) override;
};
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/gpu/cuda_context.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/gpu/cuda_context.cc b/cpp/src/arrow/gpu/cuda_context.cc
index 430ecab..42315cd 100644
--- a/cpp/src/arrow/gpu/cuda_context.cc
+++ b/cpp/src/arrow/gpu/cuda_context.cc
@@ -185,7 +185,7 @@ class CudaDeviceManager::CudaDeviceManagerImpl {
// device_number -> CudaContext
std::unordered_map<int, std::shared_ptr<CudaContext>> contexts_;
- int host_bytes_allocated_;
+ int64_t host_bytes_allocated_;
};
CudaDeviceManager::CudaDeviceManager() { impl_.reset(new CudaDeviceManagerImpl()); }
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/ipc-json-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-json-test.cc b/cpp/src/arrow/ipc/ipc-json-test.cc
index 1967e24..7855aea 100644
--- a/cpp/src/arrow/ipc/ipc-json-test.cc
+++ b/cpp/src/arrow/ipc/ipc-json-test.cc
@@ -376,7 +376,8 @@ TEST(TestJsonFileReadWrite, MinimalFormatExample) {
::testing::Values(&MakeIntRecordBatch, &MakeListRecordBatch, &MakeNonNullRecordBatch, \
&MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \
&MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, &MakeDates, \
- &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDictionary);
+ &MakeTimestamps, &MakeTimes, &MakeFWBinary, &MakeDecimal, \
+ &MakeDictionary);
class TestJsonRoundTrip : public ::testing::TestWithParam<MakeRecordBatch*> {
public:
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/ipc-read-write-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/ipc-read-write-test.cc b/cpp/src/arrow/ipc/ipc-read-write-test.cc
index 94d8eda..ac4d36e 100644
--- a/cpp/src/arrow/ipc/ipc-read-write-test.cc
+++ b/cpp/src/arrow/ipc/ipc-read-write-test.cc
@@ -121,7 +121,7 @@ TEST_F(TestSchemaMetadata, NestedFields) {
&MakeZeroLengthRecordBatch, &MakeDeeplyNestedList, \
&MakeStringTypesRecordBatch, &MakeStruct, &MakeUnion, \
&MakeDictionary, &MakeDates, &MakeTimestamps, &MakeTimes, \
- &MakeFWBinary, &MakeBooleanBatch);
+ &MakeFWBinary, &MakeDecimal, &MakeBooleanBatch);
static int g_file_number = 0;
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/json-internal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/json-internal.cc b/cpp/src/arrow/ipc/json-internal.cc
index f0fb963..025f6c2 100644
--- a/cpp/src/arrow/ipc/json-internal.cc
+++ b/cpp/src/arrow/ipc/json-internal.cc
@@ -346,6 +346,7 @@ class SchemaWriter {
return WritePrimitive("fixedsizebinary", type);
}
+ Status Visit(const DecimalType& type) { return WritePrimitive("decimal", type); }
Status Visit(const TimestampType& type) { return WritePrimitive("timestamp", type); }
Status Visit(const IntervalType& type) { return WritePrimitive("interval", type); }
@@ -364,8 +365,6 @@ class SchemaWriter {
return Status::OK();
}
- Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
-
Status Visit(const DictionaryType& type) {
return VisitType(*type.dictionary()->type());
}
@@ -437,10 +436,11 @@ class ArrayWriter {
WriteDataValues(const T& arr) {
for (int64_t i = 0; i < arr.length(); ++i) {
int32_t length;
- const char* buf = reinterpret_cast<const char*>(arr.GetValue(i, &length));
+ const uint8_t* buf = arr.GetValue(i, &length);
if (std::is_base_of<StringArray, T>::value) {
- writer_->String(buf, length);
+ // Presumed UTF-8
+ writer_->String(reinterpret_cast<const char*>(buf), length);
} else {
writer_->String(HexEncode(buf, length));
}
@@ -450,8 +450,9 @@ class ArrayWriter {
void WriteDataValues(const FixedSizeBinaryArray& arr) {
int32_t width = arr.byte_width();
for (int64_t i = 0; i < arr.length(); ++i) {
- const char* buf = reinterpret_cast<const char*>(arr.GetValue(i));
- writer_->String(HexEncode(buf, width));
+ const uint8_t* buf = arr.GetValue(i);
+ std::string encoded = HexEncode(buf, width);
+ writer_->String(encoded);
}
}
@@ -535,8 +536,6 @@ class ArrayWriter {
return Status::OK();
}
- Status Visit(const DecimalArray& array) { return Status::NotImplemented("decimal"); }
-
Status Visit(const DictionaryArray& array) {
return VisitArrayValues(*array.indices());
}
@@ -666,6 +665,17 @@ static Status GetFixedSizeBinary(const RjObject& json_type,
return Status::OK();
}
+static Status GetDecimal(const RjObject& json_type, std::shared_ptr<DataType>* type) {
+ const auto& it_precision = json_type.FindMember("precision");
+ const auto& it_scale = json_type.FindMember("scale");
+
+ RETURN_NOT_INT("precision", it_precision, json_type);
+ RETURN_NOT_INT("scale", it_scale, json_type);
+
+ *type = decimal(it_precision->value.GetInt(), it_scale->value.GetInt());
+ return Status::OK();
+}
+
static Status GetDate(const RjObject& json_type, std::shared_ptr<DataType>* type) {
const auto& it_unit = json_type.FindMember("unit");
RETURN_NOT_STRING("unit", it_unit, json_type);
@@ -802,6 +812,8 @@ static Status GetType(const RjObject& json_type,
*type = binary();
} else if (type_name == "fixedsizebinary") {
return GetFixedSizeBinary(json_type, type);
+ } else if (type_name == "decimal") {
+ return GetDecimal(json_type, type);
} else if (type_name == "null") {
*type = null();
} else if (type_name == "date") {
@@ -817,8 +829,12 @@ static Status GetType(const RjObject& json_type,
*type = list(children[0]);
} else if (type_name == "struct") {
*type = struct_(children);
- } else {
+ } else if (type_name == "union") {
return GetUnion(json_type, children, type);
+ } else {
+ std::stringstream ss;
+ ss << "Unrecognized type name: " << type_name;
+ return Status::Invalid(ss.str());
}
return Status::OK();
}
@@ -1039,7 +1055,7 @@ class ArrayReader {
template <typename T>
typename std::enable_if<std::is_base_of<FixedSizeBinaryType, T>::value, Status>::type
Visit(const T& type) {
- FixedSizeBinaryBuilder builder(type_, pool_);
+ typename TypeTraits<T>::BuilderType builder(type_, pool_);
const auto& json_data = obj_->FindMember("DATA");
RETURN_NOT_ARRAY("DATA", json_data, *obj_);
@@ -1063,8 +1079,10 @@ class ArrayReader {
const rj::Value& val = json_data_arr[i];
DCHECK(val.IsString());
std::string hex_string = val.GetString();
- DCHECK_EQ(static_cast<int32_t>(hex_string.size()), byte_width * 2)
- << "Expected size: " << byte_width * 2 << " got: " << hex_string.size();
+ if (static_cast<int32_t>(hex_string.size()) != byte_width * 2) {
+ DCHECK(false) << "Expected size: " << byte_width * 2
+ << " got: " << hex_string.size();
+ }
const char* hex_data = hex_string.c_str();
for (int32_t j = 0; j < byte_width; ++j) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/metadata-internal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/metadata-internal.cc b/cpp/src/arrow/ipc/metadata-internal.cc
index 1f2340d..962fe79 100644
--- a/cpp/src/arrow/ipc/metadata-internal.cc
+++ b/cpp/src/arrow/ipc/metadata-internal.cc
@@ -248,8 +248,11 @@ static Status TypeFromFlatbuffer(flatbuf::Type type, const void* type_data,
case flatbuf::Type_Bool:
*out = boolean();
return Status::OK();
- case flatbuf::Type_Decimal:
- return Status::NotImplemented("Decimal");
+ case flatbuf::Type_Decimal: {
+ auto dec_type = static_cast<const flatbuf::Decimal*>(type_data);
+ *out = decimal(dec_type->precision(), dec_type->scale());
+ return Status::OK();
+ }
case flatbuf::Type_Date: {
auto date_type = static_cast<const flatbuf::Date*>(type_data);
if (date_type->unit() == flatbuf::DateUnit_DAY) {
@@ -420,6 +423,12 @@ static Status TypeToFlatbuffer(FBB& fbb, const DataType& type,
}
*offset = flatbuf::CreateTimestamp(fbb, fb_unit, fb_timezone).Union();
} break;
+ case Type::DECIMAL: {
+ const auto& dec_type = static_cast<const DecimalType&>(*value_type);
+ *out_type = flatbuf::Type_Decimal;
+ *offset =
+ flatbuf::CreateDecimal(fbb, dec_type.precision(), dec_type.scale()).Union();
+ } break;
case Type::LIST:
*out_type = flatbuf::Type_List;
return ListToFlatbuffer(fbb, *value_type, children, dictionary_memo, offset);
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/reader.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/reader.cc b/cpp/src/arrow/ipc/reader.cc
index a4cac0b..919dd7a 100644
--- a/cpp/src/arrow/ipc/reader.cc
+++ b/cpp/src/arrow/ipc/reader.cc
@@ -189,8 +189,6 @@ class ArrayLoader {
Status Visit(const NullType& type) { return Status::NotImplemented("null"); }
- Status Visit(const DecimalType& type) { return Status::NotImplemented("decimal"); }
-
template <typename T>
typename std::enable_if<std::is_base_of<FixedWidthType, T>::value &&
!std::is_base_of<FixedSizeBinaryType, T>::value &&
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/test-common.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/test-common.h b/cpp/src/arrow/ipc/test-common.h
index c1e79d4..ae36b5f 100644
--- a/cpp/src/arrow/ipc/test-common.h
+++ b/cpp/src/arrow/ipc/test-common.h
@@ -161,12 +161,12 @@ Status MakeRandomBooleanArray(const int length, bool include_nulls,
std::vector<uint8_t> values(length);
test::random_null_bytes(length, 0.5, values.data());
std::shared_ptr<Buffer> data;
- RETURN_NOT_OK(BitUtil::BytesToBits(values, &data));
+ RETURN_NOT_OK(BitUtil::BytesToBits(values, default_memory_pool(), &data));
if (include_nulls) {
std::vector<uint8_t> valid_bytes(length);
std::shared_ptr<Buffer> null_bitmap;
- RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, &null_bitmap));
+ RETURN_NOT_OK(BitUtil::BytesToBits(valid_bytes, default_memory_pool(), &null_bitmap));
test::random_null_bytes(length, 0.1, valid_bytes.data());
*out = std::make_shared<BooleanArray>(length, data, null_bitmap, -1);
} else {
@@ -371,7 +371,7 @@ Status MakeStruct(std::shared_ptr<RecordBatch>* out) {
std::vector<uint8_t> null_bytes(list_batch->num_rows(), 1);
null_bytes[0] = 0;
std::shared_ptr<Buffer> null_bitmask;
- RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
+ RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask));
std::shared_ptr<Array> with_nulls(
new StructArray(type, list_batch->num_rows(), columns, null_bitmask, 1));
@@ -431,7 +431,7 @@ Status MakeUnion(std::shared_ptr<RecordBatch>* out) {
std::vector<uint8_t> null_bytes(length, 1);
null_bytes[2] = 0;
std::shared_ptr<Buffer> null_bitmask;
- RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, &null_bitmask));
+ RETURN_NOT_OK(BitUtil::BytesToBits(null_bytes, default_memory_pool(), &null_bitmask));
// construct individual nullable/non-nullable struct arrays
auto sparse_no_nulls =
@@ -670,6 +670,33 @@ Status MakeFWBinary(std::shared_ptr<RecordBatch>* out) {
return Status::OK();
}
+Status MakeDecimal(std::shared_ptr<RecordBatch>* out) {
+ auto f0 = field("f0", decimal(19, 4));
+ auto schema = ::arrow::schema({f0, f0});
+
+ constexpr int kDecimalSize = 16;
+ constexpr int length = 10;
+
+ std::shared_ptr<Buffer> data, is_valid;
+ std::vector<uint8_t> is_valid_bytes(length);
+
+ RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), kDecimalSize * length, &data));
+
+ test::random_bytes(kDecimalSize * length, 0, data->mutable_data());
+ test::random_null_bytes(length, 0.1, is_valid_bytes.data());
+
+ RETURN_NOT_OK(BitUtil::BytesToBits(is_valid_bytes, default_memory_pool(), &is_valid));
+
+ auto a1 = std::make_shared<DecimalArray>(f0->type(), length, data, is_valid,
+ kUnknownNullCount);
+
+ auto a2 = std::make_shared<DecimalArray>(f0->type(), length, data);
+
+ ArrayVector arrays = {a1, a2};
+ *out = std::make_shared<RecordBatch>(schema, a1->length(), arrays);
+ return Status::OK();
+}
+
} // namespace ipc
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/ipc/writer.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/ipc/writer.cc b/cpp/src/arrow/ipc/writer.cc
index 7ac6380..3f3d168 100644
--- a/cpp/src/arrow/ipc/writer.cc
+++ b/cpp/src/arrow/ipc/writer.cc
@@ -341,6 +341,7 @@ class RecordBatchSerializer : public ArrayVisitor {
VISIT_FIXED_WIDTH(Time32Array);
VISIT_FIXED_WIDTH(Time64Array);
VISIT_FIXED_WIDTH(FixedSizeBinaryArray);
+ VISIT_FIXED_WIDTH(DecimalArray);
#undef VISIT_FIXED_WIDTH
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/pretty_print.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/pretty_print.cc b/cpp/src/arrow/pretty_print.cc
index aedad12..7794f8d 100644
--- a/cpp/src/arrow/pretty_print.cc
+++ b/cpp/src/arrow/pretty_print.cc
@@ -99,7 +99,7 @@ class ArrayPrinter {
if (array.IsNull(i)) {
Write("null");
} else {
- const char* buf = reinterpret_cast<const char*>(array.GetValue(i, &length));
+ const uint8_t* buf = array.GetValue(i, &length);
(*sink_) << HexEncode(buf, length);
}
}
@@ -116,8 +116,7 @@ class ArrayPrinter {
if (array.IsNull(i)) {
Write("null");
} else {
- const char* buf = reinterpret_cast<const char*>(array.GetValue(i));
- (*sink_) << HexEncode(buf, width);
+ (*sink_) << HexEncode(array.GetValue(i), width);
}
}
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/arrow_to_pandas.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/arrow_to_pandas.cc b/cpp/src/arrow/python/arrow_to_pandas.cc
index d2d6fb4..769f609 100644
--- a/cpp/src/arrow/python/arrow_to_pandas.cc
+++ b/cpp/src/arrow/python/arrow_to_pandas.cc
@@ -606,8 +606,8 @@ static Status ConvertTimes(PandasOptions options, const ChunkedArray& data,
static Status RawDecimalToString(const uint8_t* bytes, int precision, int scale,
std::string* result) {
DCHECK_NE(result, nullptr);
- decimal::Int128 decimal(bytes);
- *result = decimal::ToString(decimal, precision, scale);
+ Int128 decimal(bytes);
+ *result = DecimalUtil::ToString(decimal, precision, scale);
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/builtin_convert.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/builtin_convert.cc b/cpp/src/arrow/python/builtin_convert.cc
index dca3f45..eb17f4e 100644
--- a/cpp/src/arrow/python/builtin_convert.cc
+++ b/cpp/src/arrow/python/builtin_convert.cc
@@ -589,8 +589,8 @@ class DecimalConverter
std::string string;
RETURN_NOT_OK(PythonDecimalToString(item.obj(), &string));
- decimal::Int128 value;
- RETURN_NOT_OK(decimal::FromString(string, &value));
+ Int128 value;
+ RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
return typed_builder_->Append(value);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/helpers.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
index 67f0655..deda93f 100644
--- a/cpp/src/arrow/python/helpers.cc
+++ b/cpp/src/arrow/python/helpers.cc
@@ -103,7 +103,7 @@ Status InferDecimalPrecisionAndScale(PyObject* python_decimal, int* precision,
auto size = str.size;
std::string c_string(bytes, size);
- return decimal::FromString(c_string, nullptr, precision, scale);
+ return DecimalUtil::FromString(c_string, nullptr, precision, scale);
}
Status DecimalFromString(PyObject* decimal_constructor, const std::string& decimal_string,
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/python/pandas_to_arrow.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/python/pandas_to_arrow.cc b/cpp/src/arrow/python/pandas_to_arrow.cc
index fe8ae80..8410381 100644
--- a/cpp/src/arrow/python/pandas_to_arrow.cc
+++ b/cpp/src/arrow/python/pandas_to_arrow.cc
@@ -634,8 +634,8 @@ Status PandasConverter::ConvertDecimals() {
std::string string;
RETURN_NOT_OK(PythonDecimalToString(object, &string));
- decimal::Int128 value;
- RETURN_NOT_OK(decimal::FromString(string, &value));
+ Int128 value;
+ RETURN_NOT_OK(DecimalUtil::FromString(string, &value));
RETURN_NOT_OK(builder.Append(value));
} else if (PandasObjectIsNull(object)) {
RETURN_NOT_OK(builder.AppendNull());
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/status.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/status.h b/cpp/src/arrow/status.h
index 0cb0fb1..80285da 100644
--- a/cpp/src/arrow/status.h
+++ b/cpp/src/arrow/status.h
@@ -51,16 +51,14 @@ namespace arrow {
#ifdef ARROW_EXTRA_ERROR_CONTEXT
-#define RETURN_NOT_OK(s) \
- do { \
- Status _s = (s); \
- if (ARROW_PREDICT_FALSE(!_s.ok())) { \
- std::stringstream ss; \
- ss << __FILE__ << ":" << __LINE__ \
- << " code: " << #s \
- << "\n" << _s.message(); \
- return Status(_s.code(), ss.str()); \
- } \
+#define RETURN_NOT_OK(s) \
+ do { \
+ Status _s = (s); \
+ if (ARROW_PREDICT_FALSE(!_s.ok())) { \
+ std::stringstream ss; \
+ ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \
+ return Status(_s.code(), ss.str()); \
+ } \
} while (0)
#else
@@ -73,7 +71,7 @@ namespace arrow {
} \
} while (0)
-#endif // ARROW_EXTRA_ERROR_CONTEXT
+#endif // ARROW_EXTRA_ERROR_CONTEXT
#define RETURN_NOT_OK_ELSE(s, else_) \
do { \
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/type.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type.cc b/cpp/src/arrow/type.cc
index b7963b8..87bb73b 100644
--- a/cpp/src/arrow/type.cc
+++ b/cpp/src/arrow/type.cc
@@ -455,7 +455,7 @@ std::shared_ptr<Field> field(const std::string& name,
return std::make_shared<Field>(name, type, nullable, metadata);
}
-std::shared_ptr<DataType> decimal(int precision, int scale) {
+std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale) {
return std::make_shared<DecimalType>(precision, scale);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/type.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/type.h b/cpp/src/arrow/type.h
index 2a89423..d197817 100644
--- a/cpp/src/arrow/type.h
+++ b/cpp/src/arrow/type.h
@@ -461,19 +461,19 @@ class ARROW_EXPORT DecimalType : public FixedSizeBinaryType {
public:
static constexpr Type::type type_id = Type::DECIMAL;
- explicit DecimalType(int precision, int scale)
+ explicit DecimalType(int32_t precision, int32_t scale)
: FixedSizeBinaryType(16, Type::DECIMAL), precision_(precision), scale_(scale) {}
Status Accept(TypeVisitor* visitor) const override;
std::string ToString() const override;
static std::string name() { return "decimal"; }
- int precision() const { return precision_; }
- int scale() const { return scale_; }
+ int32_t precision() const { return precision_; }
+ int32_t scale() const { return scale_; }
private:
- int precision_;
- int scale_;
+ int32_t precision_;
+ int32_t scale_;
};
enum class UnionMode : char { SPARSE, DENSE };
@@ -753,14 +753,29 @@ class ARROW_EXPORT Schema {
// ----------------------------------------------------------------------
// Factory functions
-std::shared_ptr<DataType> ARROW_EXPORT fixed_size_binary(int32_t byte_width);
+/// \brief Make an instance of FixedSizeBinaryType
+ARROW_EXPORT
+std::shared_ptr<DataType> fixed_size_binary(int32_t byte_width);
-std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<Field>& value_type);
-std::shared_ptr<DataType> ARROW_EXPORT list(const std::shared_ptr<DataType>& value_type);
+/// \brief Make an instance of DecimalType
+ARROW_EXPORT
+std::shared_ptr<DataType> decimal(int32_t precision, int32_t scale);
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit);
-std::shared_ptr<DataType> ARROW_EXPORT timestamp(TimeUnit::type unit,
- const std::string& timezone);
+/// \brief Make an instance of ListType
+ARROW_EXPORT
+std::shared_ptr<DataType> list(const std::shared_ptr<Field>& value_type);
+
+/// \brief Make an instance of ListType
+ARROW_EXPORT
+std::shared_ptr<DataType> list(const std::shared_ptr<DataType>& value_type);
+
+/// \brief Make an instance of TimestampType
+ARROW_EXPORT
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit);
+
+/// \brief Make an instance of TimestampType
+ARROW_EXPORT
+std::shared_ptr<DataType> timestamp(TimeUnit::type unit, const std::string& timezone);
/// \brief Create an instance of 32-bit time type
/// Unit can be either SECOND or MILLI
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/bit-util.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/bit-util.cc b/cpp/src/arrow/util/bit-util.cc
index 0984055..15bf359 100644
--- a/cpp/src/arrow/util/bit-util.cc
+++ b/cpp/src/arrow/util/bit-util.cc
@@ -42,12 +42,12 @@ void BitUtil::FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits
}
}
-Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes,
+Status BitUtil::BytesToBits(const std::vector<uint8_t>& bytes, MemoryPool* pool,
std::shared_ptr<Buffer>* out) {
int64_t bit_length = BitUtil::BytesForBits(bytes.size());
std::shared_ptr<Buffer> buffer;
- RETURN_NOT_OK(AllocateBuffer(default_memory_pool(), bit_length, &buffer));
+ RETURN_NOT_OK(AllocateBuffer(pool, bit_length, &buffer));
memset(buffer->mutable_data(), 0, static_cast<size_t>(bit_length));
FillBitsFromBytes(bytes, buffer->mutable_data());
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/bit-util.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/bit-util.h b/cpp/src/arrow/util/bit-util.h
index da2826f..aa83746 100644
--- a/cpp/src/arrow/util/bit-util.h
+++ b/cpp/src/arrow/util/bit-util.h
@@ -380,7 +380,10 @@ static T ShiftRightLogical(T v, int shift) {
}
void FillBitsFromBytes(const std::vector<uint8_t>& bytes, uint8_t* bits);
-ARROW_EXPORT Status BytesToBits(const std::vector<uint8_t>&, std::shared_ptr<Buffer>*);
+
+/// \brief Convert vector of bytes to bitmap buffer
+ARROW_EXPORT
+Status BytesToBits(const std::vector<uint8_t>&, MemoryPool*, std::shared_ptr<Buffer>*);
} // namespace BitUtil
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal-test.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal-test.cc b/cpp/src/arrow/util/decimal-test.cc
index 5162971..97af088 100644
--- a/cpp/src/arrow/util/decimal-test.cc
+++ b/cpp/src/arrow/util/decimal-test.cc
@@ -27,7 +27,7 @@
#include "arrow/util/int128.h"
namespace arrow {
-namespace decimal {
+namespace DecimalUtil {
class DecimalTestFixture : public ::testing::Test {
public:
@@ -163,35 +163,35 @@ TEST(DecimalTest, TestDecimalStringAndBytesRoundTrip) {
TEST(DecimalTest, TestInvalidInputMinus) {
std::string invalid_value("-");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputDot) {
std::string invalid_value("0.0.0");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputEmbeddedMinus) {
std::string invalid_value("0-13-32");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputSingleChar) {
std::string invalid_value("a");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
std::string invalid_value("-23092.235-");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
auto msg = status.message();
ASSERT_RAISES(Invalid, status);
}
@@ -199,21 +199,21 @@ TEST(DecimalTest, TestInvalidInputWithValidSubstring) {
TEST(DecimalTest, TestInvalidInputWithMinusPlus) {
std::string invalid_value("-+23092.235");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputWithPlusMinus) {
std::string invalid_value("+-23092.235");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
TEST(DecimalTest, TestInvalidInputWithLeadingZeros) {
std::string invalid_value("00a");
Int128 out;
- Status status = decimal::FromString(invalid_value, &out);
+ Status status = DecimalUtil::FromString(invalid_value, &out);
ASSERT_RAISES(Invalid, status);
}
@@ -282,5 +282,5 @@ TEST(Int128TestFalse, ConstructibleFromBool) {
ASSERT_EQ(0, value.low_bits());
}
-} // namespace decimal
+} // namespace DecimalUtil
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
index bfd87a5..3e17bf6 100644
--- a/cpp/src/arrow/util/decimal.cc
+++ b/cpp/src/arrow/util/decimal.cc
@@ -26,7 +26,7 @@
#include "arrow/util/logging.h"
namespace arrow {
-namespace decimal {
+namespace DecimalUtil {
void StringToInteger(const std::string& whole, const std::string& fractional, int8_t sign,
Int128* out) {
@@ -206,5 +206,5 @@ std::string ToString(const Int128& decimal_value, int precision, int scale) {
return str;
}
-} // namespace decimal
+} // namespace DecimalUtil
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/decimal.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/decimal.h b/cpp/src/arrow/util/decimal.h
index 4b7cac8..2e94877 100644
--- a/cpp/src/arrow/util/decimal.h
+++ b/cpp/src/arrow/util/decimal.h
@@ -23,10 +23,11 @@
#include "arrow/status.h"
namespace arrow {
-namespace decimal {
class Int128;
+namespace DecimalUtil {
+
ARROW_EXPORT void StringToInteger(const std::string& whole, const std::string& fractional,
int8_t sign, Int128* out);
@@ -35,6 +36,7 @@ ARROW_EXPORT Status FromString(const std::string& string, Int128* out,
ARROW_EXPORT std::string ToString(const Int128& decimal_value, int precision, int scale);
-} // namespace decimal
+} // namespace DecimalUtil
} // namespace arrow
+
#endif // ARROW_DECIMAL_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/int128.cc
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/int128.cc b/cpp/src/arrow/util/int128.cc
index 552c6cc..d5659a8 100644
--- a/cpp/src/arrow/util/int128.cc
+++ b/cpp/src/arrow/util/int128.cc
@@ -29,7 +29,6 @@
#include "arrow/util/logging.h"
namespace arrow {
-namespace decimal {
static constexpr uint64_t kIntMask = 0xFFFFFFFF;
static constexpr auto kCarryBit = static_cast<uint64_t>(1) << static_cast<uint64_t>(32);
@@ -525,5 +524,4 @@ Int128 operator%(const Int128& left, const Int128& right) {
return remainder;
}
-} // namespace decimal
} // namespace arrow
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/int128.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/int128.h b/cpp/src/arrow/util/int128.h
index f6fafd7..dd2ec52 100644
--- a/cpp/src/arrow/util/int128.h
+++ b/cpp/src/arrow/util/int128.h
@@ -27,7 +27,6 @@
#include "arrow/util/visibility.h"
namespace arrow {
-namespace decimal {
/// Represents a signed 128-bit integer in two's complement.
/// Calculations wrap around and overflow is ignored.
@@ -125,7 +124,6 @@ ARROW_EXPORT Int128 operator*(const Int128& left, const Int128& right);
ARROW_EXPORT Int128 operator/(const Int128& left, const Int128& right);
ARROW_EXPORT Int128 operator%(const Int128& left, const Int128& right);
-} // namespace decimal
} // namespace arrow
#endif // ARROW_INT128_H
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/cpp/src/arrow/util/string.h
----------------------------------------------------------------------
diff --git a/cpp/src/arrow/util/string.h b/cpp/src/arrow/util/string.h
index 6e70ddc..a2af87c 100644
--- a/cpp/src/arrow/util/string.h
+++ b/cpp/src/arrow/util/string.h
@@ -27,7 +27,7 @@ namespace arrow {
static const char* kAsciiTable = "0123456789ABCDEF";
-static inline std::string HexEncode(const char* data, int32_t length) {
+static inline std::string HexEncode(const uint8_t* data, int32_t length) {
std::string hex_string;
hex_string.reserve(length * 2);
for (int32_t j = 0; j < length; ++j) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/24332893/integration/integration_test.py
----------------------------------------------------------------------
diff --git a/integration/integration_test.py b/integration/integration_test.py
index 4653948..4c17345 100644
--- a/integration/integration_test.py
+++ b/integration/integration_test.py
@@ -896,7 +896,7 @@ class CPPTester(Tester):
EXE_PATH = os.environ.get(
'ARROW_CPP_EXE_PATH',
- os.path.join(ARROW_HOME, 'cpp/test-build/debug'))
+ os.path.join(ARROW_HOME, 'cpp/build/debug'))
CPP_INTEGRATION_EXE = os.path.join(EXE_PATH, 'json-integration-test')
STREAM_TO_FILE = os.path.join(EXE_PATH, 'stream-to-file')