You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ap...@apache.org on 2020/05/19 11:22:37 UTC
[arrow] branch master updated: ARROW-8841: [C++] Add benchmark and
unittest for encoding::PLAIN spaced
This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new bfb07ad ARROW-8841: [C++] Add benchmark and unittest for encoding::PLAIN spaced
bfb07ad is described below
commit bfb07ad1ed6447d74cfcd00682fed70ec01fbdaf
Author: Frank Du <fr...@intel.com>
AuthorDate: Tue May 19 13:22:08 2020 +0200
ARROW-8841: [C++] Add benchmark and unittest for encoding::PLAIN spaced
Benchmark(1%, 10%, 50%, 90%, 99% null probabilities) and unittest for future SIMD optimizations.
Signed-off-by: Frank Du <fr...@intel.com>
Closes #7213 from jianxind/spaced-unittest-benchmark
Authored-by: Frank Du <fr...@intel.com>
Signed-off-by: Antoine Pitrou <an...@python.org>
---
cpp/src/parquet/encoding_benchmark.cc | 118 ++++++++++++++++++++++++++++++++++
cpp/src/parquet/encoding_test.cc | 76 +++++++++++++++++++++-
2 files changed, 193 insertions(+), 1 deletion(-)
diff --git a/cpp/src/parquet/encoding_benchmark.cc b/cpp/src/parquet/encoding_benchmark.cc
index c8e6322..ef97d10 100644
--- a/cpp/src/parquet/encoding_benchmark.cc
+++ b/cpp/src/parquet/encoding_benchmark.cc
@@ -199,6 +199,124 @@ static void BM_PlainDecodingFloat(benchmark::State& state) {
BENCHMARK(BM_PlainDecodingFloat)->Range(MIN_RANGE, MAX_RANGE);
+template <typename ParquetType>
+struct BM_SpacedEncodingTraits {
+ using ArrowType = typename EncodingTraits<ParquetType>::ArrowType;
+ using ArrayType = typename arrow::TypeTraits<ArrowType>::ArrayType;
+ using CType = typename ParquetType::c_type;
+};
+
+template <>
+struct BM_SpacedEncodingTraits<BooleanType> {
+ // Leverage UInt8 vector array data for Boolean, the input src of PutSpaced is bool*
+ using ArrowType = ::arrow::UInt8Type;
+ using ArrayType = ::arrow::UInt8Array;
+ using CType = bool;
+};
+
+static void BM_PlainSpacedArgs(benchmark::internal::Benchmark* bench) {
+ constexpr auto kPlainSpacedSize = 32 * 1024; // 32k
+
+ bench->Args({/*size*/ kPlainSpacedSize, /*null_percentage=*/1});
+ bench->Args({/*size*/ kPlainSpacedSize, /*null_percentage=*/10});
+ bench->Args({/*size*/ kPlainSpacedSize, /*null_percentage=*/50});
+ bench->Args({/*size*/ kPlainSpacedSize, /*null_percentage=*/90});
+ bench->Args({/*size*/ kPlainSpacedSize, /*null_percentage=*/99});
+}
+
+template <typename ParquetType>
+static void BM_PlainEncodingSpaced(benchmark::State& state) {
+ using ArrowType = typename BM_SpacedEncodingTraits<ParquetType>::ArrowType;
+ using ArrayType = typename BM_SpacedEncodingTraits<ParquetType>::ArrayType;
+ using CType = typename BM_SpacedEncodingTraits<ParquetType>::CType;
+
+ const auto num_values = state.range(0);
+ const auto null_percent = static_cast<double>(state.range(1)) / 100.0;
+
+ auto rand = ::arrow::random::RandomArrayGenerator(1923);
+ const auto array = rand.Numeric<ArrowType>(num_values, -100, 100, null_percent);
+ const auto valid_bits = array->null_bitmap_data();
+ const auto array_actual = arrow::internal::checked_pointer_cast<ArrayType>(array);
+ const auto raw_values = array_actual->raw_values();
+ // Guarantee the type cast between raw_values and input of PutSpaced.
+ static_assert(sizeof(CType) == sizeof(*raw_values));
+ // Cast only happens for BooleanType as it use UInt8 for the array data to match a bool*
+ // input to PutSpaced.
+ const auto src = reinterpret_cast<const CType*>(raw_values);
+
+ auto encoder = MakeTypedEncoder<ParquetType>(Encoding::PLAIN);
+ for (auto _ : state) {
+ encoder->PutSpaced(src, num_values, valid_bits, 0);
+ encoder->FlushValues();
+ }
+ state.SetBytesProcessed(state.iterations() * num_values * sizeof(CType));
+}
+
+static void BM_PlainEncodingSpacedBoolean(benchmark::State& state) {
+ BM_PlainEncodingSpaced<BooleanType>(state);
+}
+BENCHMARK(BM_PlainEncodingSpacedBoolean)->Apply(BM_PlainSpacedArgs);
+
+static void BM_PlainEncodingSpacedFloat(benchmark::State& state) {
+ BM_PlainEncodingSpaced<FloatType>(state);
+}
+BENCHMARK(BM_PlainEncodingSpacedFloat)->Apply(BM_PlainSpacedArgs);
+
+static void BM_PlainEncodingSpacedDouble(benchmark::State& state) {
+ BM_PlainEncodingSpaced<DoubleType>(state);
+}
+BENCHMARK(BM_PlainEncodingSpacedDouble)->Apply(BM_PlainSpacedArgs);
+
+template <typename ParquetType>
+static void BM_PlainDecodingSpaced(benchmark::State& state) {
+ using ArrowType = typename BM_SpacedEncodingTraits<ParquetType>::ArrowType;
+ using ArrayType = typename BM_SpacedEncodingTraits<ParquetType>::ArrayType;
+ using CType = typename BM_SpacedEncodingTraits<ParquetType>::CType;
+
+ const auto num_values = state.range(0);
+ const auto null_percent = static_cast<double>(state.range(1)) / 100.0;
+
+ auto rand = ::arrow::random::RandomArrayGenerator(1923);
+ const auto array = rand.Numeric<ArrowType>(num_values, -100, 100, null_percent);
+ const auto valid_bits = array->null_bitmap_data();
+ const auto null_count = array->null_count();
+ const auto array_actual = arrow::internal::checked_pointer_cast<ArrayType>(array);
+ const auto raw_values = array_actual->raw_values();
+ // Guarantee the type cast between raw_values and input of PutSpaced.
+ static_assert(sizeof(CType) == sizeof(*raw_values));
+ // Cast only happens for BooleanType as it use UInt8 for the array data to match a bool*
+ // input to PutSpaced.
+ const auto src = reinterpret_cast<const CType*>(raw_values);
+
+ auto encoder = MakeTypedEncoder<ParquetType>(Encoding::PLAIN);
+ encoder->PutSpaced(src, num_values, valid_bits, 0);
+ std::shared_ptr<Buffer> buf = encoder->FlushValues();
+
+ auto decoder = MakeTypedDecoder<ParquetType>(Encoding::PLAIN);
+ std::vector<uint8_t> decode_values(num_values * sizeof(CType));
+ auto decode_buf = reinterpret_cast<CType*>(decode_values.data());
+ for (auto _ : state) {
+ decoder->SetData(num_values - null_count, buf->data(), buf->size());
+ decoder->DecodeSpaced(decode_buf, num_values, null_count, valid_bits, 0);
+ }
+ state.SetBytesProcessed(state.iterations() * num_values * sizeof(CType));
+}
+
+static void BM_PlainDecodingSpacedBoolean(benchmark::State& state) {
+ BM_PlainDecodingSpaced<BooleanType>(state);
+}
+BENCHMARK(BM_PlainDecodingSpacedBoolean)->Apply(BM_PlainSpacedArgs);
+
+static void BM_PlainDecodingSpacedFloat(benchmark::State& state) {
+ BM_PlainDecodingSpaced<FloatType>(state);
+}
+BENCHMARK(BM_PlainDecodingSpacedFloat)->Apply(BM_PlainSpacedArgs);
+
+static void BM_PlainDecodingSpacedDouble(benchmark::State& state) {
+ BM_PlainDecodingSpaced<DoubleType>(state);
+}
+BENCHMARK(BM_PlainDecodingSpacedDouble)->Apply(BM_PlainSpacedArgs);
+
template <typename T, typename DecodeFunc>
static void BM_ByteStreamSplitDecode(benchmark::State& state, DecodeFunc&& decode_func) {
std::vector<T> values(state.range(0), 64.0);
diff --git a/cpp/src/parquet/encoding_test.cc b/cpp/src/parquet/encoding_test.cc
index 6acc912..af3b6bc 100644
--- a/cpp/src/parquet/encoding_test.cc
+++ b/cpp/src/parquet/encoding_test.cc
@@ -132,6 +132,16 @@ void VerifyResults(T* result, T* expected, int num_values) {
}
}
+template <typename T>
+void VerifyResultsSpaced(T* result, T* expected, int num_values, uint8_t* valid_bits,
+ int64_t valid_bits_offset) {
+ for (auto i = 0; i < num_values; ++i) {
+ if (BitUtil::GetBit(valid_bits, valid_bits_offset + i)) {
+ ASSERT_EQ(expected[i], result[i]) << i;
+ }
+ }
+}
+
template <>
void VerifyResults<FLBA>(FLBA* result, FLBA* expected, int num_values) {
for (int i = 0; i < num_values; ++i) {
@@ -139,6 +149,16 @@ void VerifyResults<FLBA>(FLBA* result, FLBA* expected, int num_values) {
}
}
+template <>
+void VerifyResultsSpaced<FLBA>(FLBA* result, FLBA* expected, int num_values,
+ uint8_t* valid_bits, int64_t valid_bits_offset) {
+ for (auto i = 0; i < num_values; ++i) {
+ if (BitUtil::GetBit(valid_bits, valid_bits_offset + i)) {
+ ASSERT_EQ(0, memcmp(expected[i].ptr, result[i].ptr, flba_length)) << i;
+ }
+ }
+}
+
// ----------------------------------------------------------------------
// Create some column descriptors
@@ -191,11 +211,24 @@ class TestEncodingBase : public ::testing::Test {
virtual void CheckRoundtrip() = 0;
+ virtual void CheckRoundtripSpaced(int64_t valid_bits_offset) {}
+
void Execute(int nvalues, int repeats) {
InitData(nvalues, repeats);
CheckRoundtrip();
}
+ void ExecuteSpaced(int nvalues, int repeats, int64_t valid_bits_offset) {
+ InitData(nvalues, repeats);
+ auto num_value_valid_bits_buffer =
+ static_cast<int>(arrow::BitUtil::BytesForBits(num_values_ + valid_bits_offset));
+ valid_bits_buffer_.resize(num_value_valid_bits_buffer * sizeof(uint8_t));
+ // Random initialization the valid bits map
+ random_bytes(num_value_valid_bits_buffer, 0, &valid_bits_buffer_);
+ valid_bits_ = valid_bits_buffer_.data();
+ CheckRoundtripSpaced(valid_bits_offset);
+ }
+
protected:
MemoryPool* allocator_;
@@ -207,6 +240,9 @@ class TestEncodingBase : public ::testing::Test {
std::vector<uint8_t> output_bytes_;
std::vector<uint8_t> data_buffer_;
+ uint8_t* valid_bits_; // valid bits map for spaced
+ std::vector<uint8_t> valid_bits_buffer_;
+
std::shared_ptr<Buffer> encode_buffer_;
std::shared_ptr<ColumnDescriptor> descr_;
};
@@ -221,7 +257,8 @@ class TestEncodingBase : public ::testing::Test {
using TestEncodingBase<Type>::data_buffer_; \
using TestEncodingBase<Type>::type_length_; \
using TestEncodingBase<Type>::encode_buffer_; \
- using TestEncodingBase<Type>::decode_buf_
+ using TestEncodingBase<Type>::decode_buf_; \
+ using TestEncodingBase<Type>::valid_bits_
template <typename Type>
class TestPlainEncoding : public TestEncodingBase<Type> {
@@ -242,6 +279,27 @@ class TestPlainEncoding : public TestEncodingBase<Type> {
ASSERT_NO_FATAL_FAILURE(VerifyResults<T>(decode_buf_, draws_, num_values_));
}
+ void CheckRoundtripSpaced(int64_t valid_bits_offset) {
+ auto encoder = MakeTypedEncoder<Type>(Encoding::PLAIN, false, descr_.get());
+ auto decoder = MakeTypedDecoder<Type>(Encoding::PLAIN, descr_.get());
+ int null_count = 0;
+ for (auto i = 0; i < num_values_; i++) {
+ if (!BitUtil::GetBit(valid_bits_, valid_bits_offset + i)) {
+ null_count++;
+ }
+ }
+
+ encoder->PutSpaced(draws_, num_values_, valid_bits_, valid_bits_offset);
+ encode_buffer_ = encoder->FlushValues();
+ decoder->SetData(num_values_ - null_count, encode_buffer_->data(),
+ static_cast<int>(encode_buffer_->size()));
+ auto values_decoded = decoder->DecodeSpaced(decode_buf_, num_values_, null_count,
+ valid_bits_, valid_bits_offset);
+ ASSERT_EQ(num_values_, values_decoded);
+ ASSERT_NO_FATAL_FAILURE(VerifyResultsSpaced<T>(decode_buf_, draws_, num_values_,
+ valid_bits_, valid_bits_offset));
+ }
+
protected:
USING_BASE_MEMBERS();
};
@@ -250,6 +308,22 @@ TYPED_TEST_SUITE(TestPlainEncoding, ParquetTypes);
TYPED_TEST(TestPlainEncoding, BasicRoundTrip) {
ASSERT_NO_FATAL_FAILURE(this->Execute(10000, 1));
+
+ // Spaced test with different sizes and offest to guarantee SIMD implementation
+ constexpr int kAvx512Size = 64; // sizeof(__m512i) for Avx512
+ constexpr int kSimdSize = kAvx512Size; // Current the max is Avx512
+ constexpr int kMultiSimdSize = kSimdSize * 33;
+
+ // Test with both size and offset up to 3 Simd block
+ for (auto i = 1; i < kSimdSize * 3; i++) {
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(i, 1, 0));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(i, 1, i + 1));
+ }
+ // Large block and offset
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(kMultiSimdSize, 1, 0));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(kMultiSimdSize + 33, 1, 0));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(kMultiSimdSize, 1, 33));
+ ASSERT_NO_FATAL_FAILURE(this->ExecuteSpaced(kMultiSimdSize + 33, 1, 33));
}
// ----------------------------------------------------------------------