You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/12 14:53:23 UTC
[doris] 02/14: [Opt](parquet reader) Optimize the performance of reading decimal in parquet reader. (#25012)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 569acb2943b088f5fa841af0e546b454592ab150
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Thu Oct 12 09:53:08 2023 +0800
[Opt](parquet reader) Optimize the performance of reading decimal in parquet reader. (#25012)
Optimize the performance of reading decimal in parquet reader.
- Static dispatch `DecimalScaleParams`.
- Optimize `memcpy`, static dispatch copy size in fixed length cases.
- Use right shift bit operator to convert decimals.
---
be/src/util/bit_util.h | 30 +++
.../exec/format/parquet/byte_array_dict_decoder.h | 45 ++++-
.../exec/format/parquet/byte_array_plain_decoder.h | 45 ++++-
.../format/parquet/fix_length_dict_decoder.hpp | 204 +++++++++++++++++----
.../format/parquet/fix_length_plain_decoder.cpp | 154 ++++++++++++++--
.../exec/format/parquet/fix_length_plain_decoder.h | 12 ++
6 files changed, 423 insertions(+), 67 deletions(-)
diff --git a/be/src/util/bit_util.h b/be/src/util/bit_util.h
index 0a60eb54299..3396b24c9d5 100644
--- a/be/src/util/bit_util.h
+++ b/be/src/util/bit_util.h
@@ -27,6 +27,7 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "gutil/bits.h"
+#include "gutil/endian.h"
#include "util/cpu_info.h"
#include "util/sse_util.hpp"
@@ -185,6 +186,35 @@ public:
static inline uint16_t big_endian(uint16_t val) { return val; }
#endif
+ template <typename T>
+ static T big_endian_to_host(T value) {
+ if constexpr (std::is_same_v<T, __int128>) {
+ return BigEndian::ToHost128(value);
+ } else if constexpr (std::is_same_v<T, unsigned __int128>) {
+ return BigEndian::ToHost128(value);
+ } else if constexpr (std::is_same_v<T, int64_t>) {
+ return BigEndian::ToHost64(value);
+ } else if constexpr (std::is_same_v<T, uint64_t>) {
+ return BigEndian::ToHost64(value);
+ } else if constexpr (std::is_same_v<T, int32_t>) {
+ return BigEndian::ToHost32(value);
+ } else if constexpr (std::is_same_v<T, uint32_t>) {
+ return BigEndian::ToHost32(value);
+ } else if constexpr (std::is_same_v<T, int16_t>) {
+ return BigEndian::ToHost16(value);
+ } else if constexpr (std::is_same_v<T, uint16_t>) {
+ return BigEndian::ToHost16(value);
+ } else if constexpr (std::is_same_v<T, int8_t>) {
+ return value;
+ } else if constexpr (std::is_same_v<T, uint8_t>) {
+ return value;
+ } else {
+ __builtin_unreachable();
+ LOG(FATAL) << "__builtin_unreachable";
+ return value;
+ }
+ }
+
/// Returns the smallest power of two that contains v. If v is a power of two, v is
/// returned. Taken from
/// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
diff --git a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
index aaf5ccf13cd..2f90ada4282 100644
--- a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
+++ b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
@@ -25,7 +25,7 @@
#include <vector>
#include "common/status.h"
-#include "gutil/endian.h"
+#include "util/bit_util.h"
#include "vec/columns/columns_number.h"
#include "vec/common/string_ref.h"
#include "vec/core/types.h"
@@ -75,6 +75,12 @@ protected:
std::vector<uint8_t> _dict_data;
size_t _max_value_length;
std::unordered_map<StringRef, int32_t> _dict_value_to_code;
+
+private:
+ template <typename DecimalPrimitiveType, bool has_filter,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+ ColumnSelectVector& select_vector);
};
template <typename DecimalPrimitiveType, bool has_filter>
@@ -82,6 +88,26 @@ Status ByteArrayDictDecoder::_decode_binary_decimal(MutableColumnPtr& doris_colu
DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::SCALE_UP>(
+ doris_column, data_type, select_vector);
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::SCALE_DOWN>(
+ doris_column, data_type, select_vector);
+ } else {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::NO_SCALE>(
+ doris_column, data_type, select_vector);
+ }
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, DecimalScaleParams::ScaleType ScaleType>
+Status ByteArrayDictDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
auto& column_data =
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
size_t data_index = column_data.size();
@@ -98,14 +124,19 @@ Status ByteArrayDictDecoder::_decode_binary_decimal(MutableColumnPtr& doris_colu
uint32_t length = (uint32_t)slice.size;
// When Decimal in parquet is stored in byte arrays, binary and fixed,
// the unscaled number must be encoded as two's complement using big-endian byte order.
- Int128 value = buf_start[0] & 0x80 ? -1 : 0;
- memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - length, buf_start,
- length);
- value = BigEndian::ToHost128(value);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ DecimalPrimitiveType value = 0;
+ memcpy(reinterpret_cast<char*>(&value), buf_start, length);
+ value = BitUtil::big_endian_to_host(value);
+ value = value >> ((sizeof(value) - length) * 8);
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
}
auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h b/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
index 9427fb215a3..5d5d23db603 100644
--- a/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
+++ b/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
@@ -25,7 +25,7 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
#include "common/status.h"
-#include "gutil/endian.h"
+#include "util/bit_util.h"
#include "util/coding.h"
#include "util/slice.h"
#include "vec/core/types.h"
@@ -61,6 +61,12 @@ protected:
template <typename DecimalPrimitiveType, bool has_filter>
Status _decode_binary_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
ColumnSelectVector& select_vector);
+
+private:
+ template <typename DecimalPrimitiveType, bool has_filter,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+ ColumnSelectVector& select_vector);
};
template <typename DecimalPrimitiveType, bool has_filter>
@@ -68,6 +74,26 @@ Status ByteArrayPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::SCALE_UP>(
+ doris_column, data_type, select_vector);
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::SCALE_DOWN>(
+ doris_column, data_type, select_vector);
+ } else {
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+ DecimalScaleParams::NO_SCALE>(
+ doris_column, data_type, select_vector);
+ }
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, DecimalScaleParams::ScaleType ScaleType>
+Status ByteArrayPlainDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
auto& column_data =
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
size_t data_index = column_data.size();
@@ -88,14 +114,19 @@ Status ByteArrayPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
_offset += length;
// When Decimal in parquet is stored in byte arrays, binary and fixed,
// the unscaled number must be encoded as two's complement using big-endian byte order.
- Int128 value = buf_start[0] & 0x80 ? -1 : 0;
- memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - length, buf_start,
- length);
- value = BigEndian::ToHost128(value);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ DecimalPrimitiveType value = 0;
+ memcpy(reinterpret_cast<char*>(&value), buf_start, length);
+ value = BitUtil::big_endian_to_host(value);
+ value = value >> ((sizeof(value) - length) * 8);
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
}
auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
index 10474148996..a30c2dff3d1 100644
--- a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
+++ b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
@@ -17,6 +17,7 @@
#pragma once
+#include "util/bit_util.h"
#include "vec/columns/column_dictionary.h"
#include "vec/columns/column_nullable.h"
#include "vec/data_types/data_type_nullable.h"
@@ -337,6 +338,65 @@ protected:
Status _decode_primitive_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType) \
+ case FixedTypeLength: \
+ return _decode_primitive_decimal_internal<DecimalPrimitiveType, DecimalPhysicalType, \
+ has_filter, FixedTypeLength, ValueCopyType, \
+ ScaleType>(doris_column, data_type, \
+ select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+ M(1, int64_t, ScaleType) \
+ M(2, int64_t, ScaleType) \
+ M(3, int64_t, ScaleType) \
+ M(4, int64_t, ScaleType) \
+ M(5, int64_t, ScaleType) \
+ M(6, int64_t, ScaleType) \
+ M(7, int64_t, ScaleType) \
+ M(8, int64_t, ScaleType) \
+ M(9, int128_t, ScaleType) \
+ M(10, int128_t, ScaleType) \
+ M(11, int128_t, ScaleType) \
+ M(12, int128_t, ScaleType) \
+ M(13, int128_t, ScaleType) \
+ M(14, int128_t, ScaleType) \
+ M(15, int128_t, ScaleType) \
+ M(16, int128_t, ScaleType)
+
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ }
+ return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+ }
+
+ template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+ int fixed_type_length, typename ValueCopyType,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_primitive_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
auto& column_data =
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
.get_data();
@@ -350,12 +410,16 @@ protected:
switch (read_type) {
case ColumnSelectVector::CONTENT: {
for (size_t i = 0; i < run_length; ++i) {
- // we should use decimal128 to scale up/down
- Int128 value = static_cast<Int128>(_dict_items[_indexes[dict_index++]]);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ ValueCopyType value = static_cast<T>(_dict_items[_indexes[dict_index++]]);
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
}
auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
v = (DecimalPrimitiveType)value;
@@ -523,38 +587,75 @@ protected:
Status _decode_binary_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
- auto& column_data =
- static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
- .get_data();
- size_t data_index = column_data.size();
- column_data.resize(data_index + select_vector.num_values() - select_vector.num_filtered());
- size_t dict_index = 0;
DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType) \
+ case FixedTypeLength: \
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter, FixedTypeLength, \
+ ValueCopyType, ScaleType>(doris_column, data_type, \
+ select_vector);
+#define APPLY_FOR_DECIMALS(ScaleType) \
+ M(1, int64_t, ScaleType) \
+ M(2, int64_t, ScaleType) \
+ M(3, int64_t, ScaleType) \
+ M(4, int64_t, ScaleType) \
+ M(5, int64_t, ScaleType) \
+ M(6, int64_t, ScaleType) \
+ M(7, int64_t, ScaleType) \
+ M(8, int64_t, ScaleType) \
+ M(9, int128_t, ScaleType) \
+ M(10, int128_t, ScaleType) \
+ M(11, int128_t, ScaleType) \
+ M(12, int128_t, ScaleType) \
+ M(13, int128_t, ScaleType) \
+ M(14, int128_t, ScaleType) \
+ M(15, int128_t, ScaleType) \
+ M(16, int128_t, ScaleType)
+
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ }
+ return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+ }
+
+ template <bool has_filter>
+ Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector) {
+ size_t dict_index = 0;
ColumnSelectVector::DataReadType read_type;
while (size_t run_length = select_vector.get_next_run<has_filter>(&read_type)) {
switch (read_type) {
case ColumnSelectVector::CONTENT: {
+ std::vector<StringRef> string_values;
+ string_values.reserve(run_length);
for (size_t i = 0; i < run_length; ++i) {
- char* buf_start = _dict_items[_indexes[dict_index++]];
- // When Decimal in parquet is stored in byte arrays, binary and fixed,
- // the unscaled number must be encoded as two's complement using big-endian byte order.
- Int128 value = buf_start[0] & 0x80 ? -1 : 0;
- memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - _type_length,
- buf_start, _type_length);
- value = BigEndian::ToHost128(value);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
- value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
- value /= scale_params.scale_factor;
- }
- auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
- v = (DecimalPrimitiveType)value;
+ string_values.emplace_back(_dict_items[_indexes[dict_index++]], _type_length);
}
+ doris_column->insert_many_strings(&string_values[0], run_length);
break;
}
case ColumnSelectVector::NULL_DATA: {
- data_index += run_length;
+ doris_column->insert_many_defaults(run_length);
break;
}
case ColumnSelectVector::FILTERED_CONTENT: {
@@ -570,23 +671,56 @@ protected:
return Status::OK();
}
- template <bool has_filter>
- Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector) {
+ tparquet::Type::type _physical_type;
+
+ // For dictionary encoding
+ std::vector<char*> _dict_items;
+ std::unordered_map<StringRef, int32_t> _dict_value_to_code;
+
+private:
+ template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+ typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
+ auto& column_data =
+ static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
+ .get_data();
+ size_t data_index = column_data.size();
+ column_data.resize(data_index + select_vector.num_values() - select_vector.num_filtered());
size_t dict_index = 0;
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+
ColumnSelectVector::DataReadType read_type;
while (size_t run_length = select_vector.get_next_run<has_filter>(&read_type)) {
switch (read_type) {
case ColumnSelectVector::CONTENT: {
- std::vector<StringRef> string_values;
- string_values.reserve(run_length);
for (size_t i = 0; i < run_length; ++i) {
- string_values.emplace_back(_dict_items[_indexes[dict_index++]], _type_length);
+ char* buf_start = _dict_items[_indexes[dict_index++]];
+ // When Decimal in parquet is stored in byte arrays, binary and fixed,
+ // the unscaled number must be encoded as two's complement using big-endian byte order.
+ DecimalPrimitiveType result_value = 0;
+ ValueCopyType value = 0;
+ memcpy(reinterpret_cast<char*>(&value), buf_start, fixed_type_length);
+ value = BitUtil::big_endian_to_host(value);
+ value = value >> ((sizeof(value) - fixed_type_length) * 8);
+ result_value = value;
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
+ result_value *= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
+ result_value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
+ v = (DecimalPrimitiveType)result_value;
}
- doris_column->insert_many_strings(&string_values[0], run_length);
break;
}
case ColumnSelectVector::NULL_DATA: {
- doris_column->insert_many_defaults(run_length);
+ data_index += run_length;
break;
}
case ColumnSelectVector::FILTERED_CONTENT: {
@@ -601,12 +735,6 @@ protected:
}
return Status::OK();
}
-
- tparquet::Type::type _physical_type;
-
- // For dictionary encoding
- std::vector<char*> _dict_items;
- std::unordered_map<StringRef, int32_t> _dict_value_to_code;
};
} // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
index e94948ad402..af464c15545 100644
--- a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
+++ b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
@@ -26,7 +26,7 @@
// IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
#include "common/compiler_util.h" // IWYU pragma: keep
-#include "gutil/endian.h"
+#include "util/bit_util.h"
#include "util/slice.h"
#include "vec/columns/column.h"
#include "vec/common/string_ref.h"
@@ -389,6 +389,63 @@ Status FixLengthPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType) \
+ case FixedTypeLength: \
+ return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter, FixedTypeLength, \
+ ValueCopyType, ScaleType>(doris_column, data_type, \
+ select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+ M(1, int64_t, ScaleType) \
+ M(2, int64_t, ScaleType) \
+ M(3, int64_t, ScaleType) \
+ M(4, int64_t, ScaleType) \
+ M(5, int64_t, ScaleType) \
+ M(6, int64_t, ScaleType) \
+ M(7, int64_t, ScaleType) \
+ M(8, int64_t, ScaleType) \
+ M(9, int128_t, ScaleType) \
+ M(10, int128_t, ScaleType) \
+ M(11, int128_t, ScaleType) \
+ M(12, int128_t, ScaleType) \
+ M(13, int128_t, ScaleType) \
+ M(14, int128_t, ScaleType) \
+ M(15, int128_t, ScaleType) \
+ M(16, int128_t, ScaleType)
+
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ }
+ return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+ typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+Status FixLengthPlainDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector& select_vector) {
auto& column_data =
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
size_t data_index = column_data.size();
@@ -403,18 +460,25 @@ Status FixLengthPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
char* buf_start = _data->data + _offset;
// When Decimal in parquet is stored in byte arrays, binary and fixed,
// the unscaled number must be encoded as two's complement using big-endian byte order.
- Int128 value = buf_start[0] & 0x80 ? -1 : 0;
- memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - _type_length, buf_start,
- _type_length);
- value = BigEndian::ToHost128(value);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
- value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
- value /= scale_params.scale_factor;
+ DecimalPrimitiveType result_value = 0;
+ ValueCopyType value = 0;
+ memcpy(reinterpret_cast<char*>(&value), buf_start, fixed_type_length);
+ value = BitUtil::big_endian_to_host(value);
+ value = value >> ((sizeof(value) - fixed_type_length) * 8);
+ result_value = value;
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
+ result_value *= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
+ result_value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
}
auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
- v = (DecimalPrimitiveType)value;
- _offset += _type_length;
+ v = (DecimalPrimitiveType)result_value;
+ _offset += fixed_type_length;
}
break;
}
@@ -440,6 +504,62 @@ Status FixLengthPlainDecoder::_decode_primitive_decimal(MutableColumnPtr& doris_
DataTypePtr& data_type,
ColumnSelectVector& select_vector) {
init_decimal_converter<DecimalPrimitiveType>(data_type);
+ DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, T, ScaleType) \
+ case FixedTypeLength: \
+ return _decode_primitive_decimal_internal<DecimalPrimitiveType, DecimalPhysicalType, \
+ has_filter, FixedTypeLength, T, ScaleType>( \
+ doris_column, data_type, select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+ M(1, int64_t, ScaleType) \
+ M(2, int64_t, ScaleType) \
+ M(3, int64_t, ScaleType) \
+ M(4, int64_t, ScaleType) \
+ M(5, int64_t, ScaleType) \
+ M(6, int64_t, ScaleType) \
+ M(7, int64_t, ScaleType) \
+ M(8, int64_t, ScaleType) \
+ M(9, int128_t, ScaleType) \
+ M(10, int128_t, ScaleType) \
+ M(11, int128_t, ScaleType) \
+ M(12, int128_t, ScaleType) \
+ M(13, int128_t, ScaleType) \
+ M(14, int128_t, ScaleType) \
+ M(15, int128_t, ScaleType) \
+ M(16, int128_t, ScaleType)
+
+ if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ } else {
+ switch (_type_length) {
+ APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+ default:
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
+ }
+ }
+ return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+}
+
+template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+ int fixed_type_length, typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+Status FixLengthPlainDecoder::_decode_primitive_decimal_internal(
+ MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector) {
auto& column_data =
static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
size_t data_index = column_data.size();
@@ -452,12 +572,16 @@ Status FixLengthPlainDecoder::_decode_primitive_decimal(MutableColumnPtr& doris_
case ColumnSelectVector::CONTENT: {
for (size_t i = 0; i < run_length; ++i) {
char* buf_start = _data->data + _offset;
- // we should use decimal128 to scale up/down
- Int128 value = *reinterpret_cast<DecimalPhysicalType*>(buf_start);
- if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+ ValueCopyType value = *reinterpret_cast<DecimalPhysicalType*>(buf_start);
+ if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
value *= scale_params.scale_factor;
- } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+ } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
value /= scale_params.scale_factor;
+ } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+ // do nothing
+ } else {
+ LOG(FATAL) << "__builtin_unreachable";
+ __builtin_unreachable();
}
auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
index 0b3a6e19453..96d674e2588 100644
--- a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
+++ b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
@@ -71,5 +71,17 @@ protected:
Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector);
tparquet::Type::type _physical_type;
+
+private:
+ template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+ typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+ ColumnSelectVector& select_vector);
+ template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+ int fixed_type_length, typename ValueCopyType,
+ DecimalScaleParams::ScaleType ScaleType>
+ Status _decode_primitive_decimal_internal(MutableColumnPtr& doris_column,
+ DataTypePtr& data_type,
+ ColumnSelectVector& select_vector);
};
} // namespace doris::vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org