You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/12 14:53:23 UTC

[doris] 02/14: [Opt](parquet reader) Optimize the performance of reading decimal in parquet reader. (#25012)

This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 569acb2943b088f5fa841af0e546b454592ab150
Author: Qi Chen <ka...@gmail.com>
AuthorDate: Thu Oct 12 09:53:08 2023 +0800

    [Opt](parquet reader) Optimize the performance of reading decimal in parquet reader. (#25012)
    
    Optimize the performance of reading decimal in parquet reader.
    
    - Static dispatch `DecimalScaleParams`.
    - Optimize `memcpy`, static dispatch copy size in fixed length cases.
    - Use right shift bit operator to convert decimals.
---
 be/src/util/bit_util.h                             |  30 +++
 .../exec/format/parquet/byte_array_dict_decoder.h  |  45 ++++-
 .../exec/format/parquet/byte_array_plain_decoder.h |  45 ++++-
 .../format/parquet/fix_length_dict_decoder.hpp     | 204 +++++++++++++++++----
 .../format/parquet/fix_length_plain_decoder.cpp    | 154 ++++++++++++++--
 .../exec/format/parquet/fix_length_plain_decoder.h |  12 ++
 6 files changed, 423 insertions(+), 67 deletions(-)

diff --git a/be/src/util/bit_util.h b/be/src/util/bit_util.h
index 0a60eb54299..3396b24c9d5 100644
--- a/be/src/util/bit_util.h
+++ b/be/src/util/bit_util.h
@@ -27,6 +27,7 @@
 // IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "gutil/bits.h"
+#include "gutil/endian.h"
 #include "util/cpu_info.h"
 #include "util/sse_util.hpp"
 
@@ -185,6 +186,35 @@ public:
     static inline uint16_t big_endian(uint16_t val) { return val; }
 #endif
 
+    template <typename T>
+    static T big_endian_to_host(T value) {
+        if constexpr (std::is_same_v<T, __int128>) {
+            return BigEndian::ToHost128(value);
+        } else if constexpr (std::is_same_v<T, unsigned __int128>) {
+            return BigEndian::ToHost128(value);
+        } else if constexpr (std::is_same_v<T, int64_t>) {
+            return BigEndian::ToHost64(value);
+        } else if constexpr (std::is_same_v<T, uint64_t>) {
+            return BigEndian::ToHost64(value);
+        } else if constexpr (std::is_same_v<T, int32_t>) {
+            return BigEndian::ToHost32(value);
+        } else if constexpr (std::is_same_v<T, uint32_t>) {
+            return BigEndian::ToHost32(value);
+        } else if constexpr (std::is_same_v<T, int16_t>) {
+            return BigEndian::ToHost16(value);
+        } else if constexpr (std::is_same_v<T, uint16_t>) {
+            return BigEndian::ToHost16(value);
+        } else if constexpr (std::is_same_v<T, int8_t>) {
+            return value;
+        } else if constexpr (std::is_same_v<T, uint8_t>) {
+            return value;
+        } else {
+            __builtin_unreachable();
+            LOG(FATAL) << "__builtin_unreachable";
+            return value;
+        }
+    }
+
     /// Returns the smallest power of two that contains v. If v is a power of two, v is
     /// returned. Taken from
     /// http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
diff --git a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
index aaf5ccf13cd..2f90ada4282 100644
--- a/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
+++ b/be/src/vec/exec/format/parquet/byte_array_dict_decoder.h
@@ -25,7 +25,7 @@
 #include <vector>
 
 #include "common/status.h"
-#include "gutil/endian.h"
+#include "util/bit_util.h"
 #include "vec/columns/columns_number.h"
 #include "vec/common/string_ref.h"
 #include "vec/core/types.h"
@@ -75,6 +75,12 @@ protected:
     std::vector<uint8_t> _dict_data;
     size_t _max_value_length;
     std::unordered_map<StringRef, int32_t> _dict_value_to_code;
+
+private:
+    template <typename DecimalPrimitiveType, bool has_filter,
+              DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+                                           ColumnSelectVector& select_vector);
 };
 
 template <typename DecimalPrimitiveType, bool has_filter>
@@ -82,6 +88,26 @@ Status ByteArrayDictDecoder::_decode_binary_decimal(MutableColumnPtr& doris_colu
                                                     DataTypePtr& data_type,
                                                     ColumnSelectVector& select_vector) {
     init_decimal_converter<DecimalPrimitiveType>(data_type);
+    DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::SCALE_UP>(
+                doris_column, data_type, select_vector);
+    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::SCALE_DOWN>(
+                doris_column, data_type, select_vector);
+    } else {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::NO_SCALE>(
+                doris_column, data_type, select_vector);
+    }
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, DecimalScaleParams::ScaleType ScaleType>
+Status ByteArrayDictDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+                                                             DataTypePtr& data_type,
+                                                             ColumnSelectVector& select_vector) {
     auto& column_data =
             static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
     size_t data_index = column_data.size();
@@ -98,14 +124,19 @@ Status ByteArrayDictDecoder::_decode_binary_decimal(MutableColumnPtr& doris_colu
                 uint32_t length = (uint32_t)slice.size;
                 // When Decimal in parquet is stored in byte arrays, binary and fixed,
                 // the unscaled number must be encoded as two's complement using big-endian byte order.
-                Int128 value = buf_start[0] & 0x80 ? -1 : 0;
-                memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - length, buf_start,
-                       length);
-                value = BigEndian::ToHost128(value);
-                if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+                DecimalPrimitiveType value = 0;
+                memcpy(reinterpret_cast<char*>(&value), buf_start, length);
+                value = BitUtil::big_endian_to_host(value);
+                value = value >> ((sizeof(value) - length) * 8);
+                if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
                     value *= scale_params.scale_factor;
-                } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+                } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
                     value /= scale_params.scale_factor;
+                } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                    // do nothing
+                } else {
+                    LOG(FATAL) << "__builtin_unreachable";
+                    __builtin_unreachable();
                 }
                 auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
                 v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h b/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
index 9427fb215a3..5d5d23db603 100644
--- a/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
+++ b/be/src/vec/exec/format/parquet/byte_array_plain_decoder.h
@@ -25,7 +25,7 @@
 // IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "common/status.h"
-#include "gutil/endian.h"
+#include "util/bit_util.h"
 #include "util/coding.h"
 #include "util/slice.h"
 #include "vec/core/types.h"
@@ -61,6 +61,12 @@ protected:
     template <typename DecimalPrimitiveType, bool has_filter>
     Status _decode_binary_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
                                   ColumnSelectVector& select_vector);
+
+private:
+    template <typename DecimalPrimitiveType, bool has_filter,
+              DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+                                           ColumnSelectVector& select_vector);
 };
 
 template <typename DecimalPrimitiveType, bool has_filter>
@@ -68,6 +74,26 @@ Status ByteArrayPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
                                                      DataTypePtr& data_type,
                                                      ColumnSelectVector& select_vector) {
     init_decimal_converter<DecimalPrimitiveType>(data_type);
+    DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::SCALE_UP>(
+                doris_column, data_type, select_vector);
+    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::SCALE_DOWN>(
+                doris_column, data_type, select_vector);
+    } else {
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter,
+                                               DecimalScaleParams::NO_SCALE>(
+                doris_column, data_type, select_vector);
+    }
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, DecimalScaleParams::ScaleType ScaleType>
+Status ByteArrayPlainDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+                                                              DataTypePtr& data_type,
+                                                              ColumnSelectVector& select_vector) {
     auto& column_data =
             static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
     size_t data_index = column_data.size();
@@ -88,14 +114,19 @@ Status ByteArrayPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
                 _offset += length;
                 // When Decimal in parquet is stored in byte arrays, binary and fixed,
                 // the unscaled number must be encoded as two's complement using big-endian byte order.
-                Int128 value = buf_start[0] & 0x80 ? -1 : 0;
-                memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - length, buf_start,
-                       length);
-                value = BigEndian::ToHost128(value);
-                if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+                DecimalPrimitiveType value = 0;
+                memcpy(reinterpret_cast<char*>(&value), buf_start, length);
+                value = BitUtil::big_endian_to_host(value);
+                value = value >> ((sizeof(value) - length) * 8);
+                if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
                     value *= scale_params.scale_factor;
-                } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+                } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
                     value /= scale_params.scale_factor;
+                } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                    // do nothing
+                } else {
+                    LOG(FATAL) << "__builtin_unreachable";
+                    __builtin_unreachable();
                 }
                 auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
                 v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
index 10474148996..a30c2dff3d1 100644
--- a/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
+++ b/be/src/vec/exec/format/parquet/fix_length_dict_decoder.hpp
@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include "util/bit_util.h"
 #include "vec/columns/column_dictionary.h"
 #include "vec/columns/column_nullable.h"
 #include "vec/data_types/data_type_nullable.h"
@@ -337,6 +338,65 @@ protected:
     Status _decode_primitive_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
                                      ColumnSelectVector& select_vector) {
         init_decimal_converter<DecimalPrimitiveType>(data_type);
+        DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType)                                          \
+    case FixedTypeLength:                                                                     \
+        return _decode_primitive_decimal_internal<DecimalPrimitiveType, DecimalPhysicalType,  \
+                                                  has_filter, FixedTypeLength, ValueCopyType, \
+                                                  ScaleType>(doris_column, data_type,         \
+                                                             select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+    M(1, int64_t, ScaleType)          \
+    M(2, int64_t, ScaleType)          \
+    M(3, int64_t, ScaleType)          \
+    M(4, int64_t, ScaleType)          \
+    M(5, int64_t, ScaleType)          \
+    M(6, int64_t, ScaleType)          \
+    M(7, int64_t, ScaleType)          \
+    M(8, int64_t, ScaleType)          \
+    M(9, int128_t, ScaleType)         \
+    M(10, int128_t, ScaleType)        \
+    M(11, int128_t, ScaleType)        \
+    M(12, int128_t, ScaleType)        \
+    M(13, int128_t, ScaleType)        \
+    M(14, int128_t, ScaleType)        \
+    M(15, int128_t, ScaleType)        \
+    M(16, int128_t, ScaleType)
+
+        if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        } else {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        }
+        return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+    }
+
+    template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+              int fixed_type_length, typename ValueCopyType,
+              DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_primitive_decimal_internal(MutableColumnPtr& doris_column,
+                                              DataTypePtr& data_type,
+                                              ColumnSelectVector& select_vector) {
         auto& column_data =
                 static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
                         .get_data();
@@ -350,12 +410,16 @@ protected:
             switch (read_type) {
             case ColumnSelectVector::CONTENT: {
                 for (size_t i = 0; i < run_length; ++i) {
-                    // we should use decimal128 to scale up/down
-                    Int128 value = static_cast<Int128>(_dict_items[_indexes[dict_index++]]);
-                    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+                    ValueCopyType value = static_cast<T>(_dict_items[_indexes[dict_index++]]);
+                    if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
                         value *= scale_params.scale_factor;
-                    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+                    } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
                         value /= scale_params.scale_factor;
+                    } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                        // do nothing
+                    } else {
+                        LOG(FATAL) << "__builtin_unreachable";
+                        __builtin_unreachable();
                     }
                     auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
                     v = (DecimalPrimitiveType)value;
@@ -523,38 +587,75 @@ protected:
     Status _decode_binary_decimal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
                                   ColumnSelectVector& select_vector) {
         init_decimal_converter<DecimalPrimitiveType>(data_type);
-        auto& column_data =
-                static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
-                        .get_data();
-        size_t data_index = column_data.size();
-        column_data.resize(data_index + select_vector.num_values() - select_vector.num_filtered());
-        size_t dict_index = 0;
         DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType)                                              \
+    case FixedTypeLength:                                                                         \
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter, FixedTypeLength, \
+                                               ValueCopyType, ScaleType>(doris_column, data_type, \
+                                                                         select_vector);
 
+#define APPLY_FOR_DECIMALS(ScaleType) \
+    M(1, int64_t, ScaleType)          \
+    M(2, int64_t, ScaleType)          \
+    M(3, int64_t, ScaleType)          \
+    M(4, int64_t, ScaleType)          \
+    M(5, int64_t, ScaleType)          \
+    M(6, int64_t, ScaleType)          \
+    M(7, int64_t, ScaleType)          \
+    M(8, int64_t, ScaleType)          \
+    M(9, int128_t, ScaleType)         \
+    M(10, int128_t, ScaleType)        \
+    M(11, int128_t, ScaleType)        \
+    M(12, int128_t, ScaleType)        \
+    M(13, int128_t, ScaleType)        \
+    M(14, int128_t, ScaleType)        \
+    M(15, int128_t, ScaleType)        \
+    M(16, int128_t, ScaleType)
+
+        if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        } else {
+            switch (_type_length) {
+                APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+            default:
+                LOG(FATAL) << "__builtin_unreachable";
+                __builtin_unreachable();
+            }
+        }
+        return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+    }
+
+    template <bool has_filter>
+    Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector) {
+        size_t dict_index = 0;
         ColumnSelectVector::DataReadType read_type;
         while (size_t run_length = select_vector.get_next_run<has_filter>(&read_type)) {
             switch (read_type) {
             case ColumnSelectVector::CONTENT: {
+                std::vector<StringRef> string_values;
+                string_values.reserve(run_length);
                 for (size_t i = 0; i < run_length; ++i) {
-                    char* buf_start = _dict_items[_indexes[dict_index++]];
-                    // When Decimal in parquet is stored in byte arrays, binary and fixed,
-                    // the unscaled number must be encoded as two's complement using big-endian byte order.
-                    Int128 value = buf_start[0] & 0x80 ? -1 : 0;
-                    memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - _type_length,
-                           buf_start, _type_length);
-                    value = BigEndian::ToHost128(value);
-                    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
-                        value *= scale_params.scale_factor;
-                    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
-                        value /= scale_params.scale_factor;
-                    }
-                    auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
-                    v = (DecimalPrimitiveType)value;
+                    string_values.emplace_back(_dict_items[_indexes[dict_index++]], _type_length);
                 }
+                doris_column->insert_many_strings(&string_values[0], run_length);
                 break;
             }
             case ColumnSelectVector::NULL_DATA: {
-                data_index += run_length;
+                doris_column->insert_many_defaults(run_length);
                 break;
             }
             case ColumnSelectVector::FILTERED_CONTENT: {
@@ -570,23 +671,56 @@ protected:
         return Status::OK();
     }
 
-    template <bool has_filter>
-    Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector) {
+    tparquet::Type::type _physical_type;
+
+    // For dictionary encoding
+    std::vector<char*> _dict_items;
+    std::unordered_map<StringRef, int32_t> _dict_value_to_code;
+
+private:
+    template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+              typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+                                           ColumnSelectVector& select_vector) {
+        auto& column_data =
+                static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column)
+                        .get_data();
+        size_t data_index = column_data.size();
+        column_data.resize(data_index + select_vector.num_values() - select_vector.num_filtered());
         size_t dict_index = 0;
+        DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+
         ColumnSelectVector::DataReadType read_type;
         while (size_t run_length = select_vector.get_next_run<has_filter>(&read_type)) {
             switch (read_type) {
             case ColumnSelectVector::CONTENT: {
-                std::vector<StringRef> string_values;
-                string_values.reserve(run_length);
                 for (size_t i = 0; i < run_length; ++i) {
-                    string_values.emplace_back(_dict_items[_indexes[dict_index++]], _type_length);
+                    char* buf_start = _dict_items[_indexes[dict_index++]];
+                    // When Decimal in parquet is stored in byte arrays, binary and fixed,
+                    // the unscaled number must be encoded as two's complement using big-endian byte order.
+                    DecimalPrimitiveType result_value = 0;
+                    ValueCopyType value = 0;
+                    memcpy(reinterpret_cast<char*>(&value), buf_start, fixed_type_length);
+                    value = BitUtil::big_endian_to_host(value);
+                    value = value >> ((sizeof(value) - fixed_type_length) * 8);
+                    result_value = value;
+                    if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
+                        result_value *= scale_params.scale_factor;
+                    } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
+                        result_value /= scale_params.scale_factor;
+                    } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                        // do nothing
+                    } else {
+                        LOG(FATAL) << "__builtin_unreachable";
+                        __builtin_unreachable();
+                    }
+                    auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
+                    v = (DecimalPrimitiveType)result_value;
                 }
-                doris_column->insert_many_strings(&string_values[0], run_length);
                 break;
             }
             case ColumnSelectVector::NULL_DATA: {
-                doris_column->insert_many_defaults(run_length);
+                data_index += run_length;
                 break;
             }
             case ColumnSelectVector::FILTERED_CONTENT: {
@@ -601,12 +735,6 @@ protected:
         }
         return Status::OK();
     }
-
-    tparquet::Type::type _physical_type;
-
-    // For dictionary encoding
-    std::vector<char*> _dict_items;
-    std::unordered_map<StringRef, int32_t> _dict_value_to_code;
 };
 
 } // namespace doris::vectorized
diff --git a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
index e94948ad402..af464c15545 100644
--- a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
+++ b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.cpp
@@ -26,7 +26,7 @@
 
 // IWYU pragma: no_include <opentelemetry/common/threadlocal.h>
 #include "common/compiler_util.h" // IWYU pragma: keep
-#include "gutil/endian.h"
+#include "util/bit_util.h"
 #include "util/slice.h"
 #include "vec/columns/column.h"
 #include "vec/common/string_ref.h"
@@ -389,6 +389,63 @@ Status FixLengthPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
                                                      DataTypePtr& data_type,
                                                      ColumnSelectVector& select_vector) {
     init_decimal_converter<DecimalPrimitiveType>(data_type);
+    DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, ValueCopyType, ScaleType)                                              \
+    case FixedTypeLength:                                                                         \
+        return _decode_binary_decimal_internal<DecimalPrimitiveType, has_filter, FixedTypeLength, \
+                                               ValueCopyType, ScaleType>(doris_column, data_type, \
+                                                                         select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+    M(1, int64_t, ScaleType)          \
+    M(2, int64_t, ScaleType)          \
+    M(3, int64_t, ScaleType)          \
+    M(4, int64_t, ScaleType)          \
+    M(5, int64_t, ScaleType)          \
+    M(6, int64_t, ScaleType)          \
+    M(7, int64_t, ScaleType)          \
+    M(8, int64_t, ScaleType)          \
+    M(9, int128_t, ScaleType)         \
+    M(10, int128_t, ScaleType)        \
+    M(11, int128_t, ScaleType)        \
+    M(12, int128_t, ScaleType)        \
+    M(13, int128_t, ScaleType)        \
+    M(14, int128_t, ScaleType)        \
+    M(15, int128_t, ScaleType)        \
+    M(16, int128_t, ScaleType)
+
+    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    } else {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    }
+    return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+}
+
+template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+          typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+Status FixLengthPlainDecoder::_decode_binary_decimal_internal(MutableColumnPtr& doris_column,
+                                                              DataTypePtr& data_type,
+                                                              ColumnSelectVector& select_vector) {
     auto& column_data =
             static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
     size_t data_index = column_data.size();
@@ -403,18 +460,25 @@ Status FixLengthPlainDecoder::_decode_binary_decimal(MutableColumnPtr& doris_col
                 char* buf_start = _data->data + _offset;
                 // When Decimal in parquet is stored in byte arrays, binary and fixed,
                 // the unscaled number must be encoded as two's complement using big-endian byte order.
-                Int128 value = buf_start[0] & 0x80 ? -1 : 0;
-                memcpy(reinterpret_cast<char*>(&value) + sizeof(Int128) - _type_length, buf_start,
-                       _type_length);
-                value = BigEndian::ToHost128(value);
-                if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
-                    value *= scale_params.scale_factor;
-                } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
-                    value /= scale_params.scale_factor;
+                DecimalPrimitiveType result_value = 0;
+                ValueCopyType value = 0;
+                memcpy(reinterpret_cast<char*>(&value), buf_start, fixed_type_length);
+                value = BitUtil::big_endian_to_host(value);
+                value = value >> ((sizeof(value) - fixed_type_length) * 8);
+                result_value = value;
+                if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
+                    result_value *= scale_params.scale_factor;
+                } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
+                    result_value /= scale_params.scale_factor;
+                } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                    // do nothing
+                } else {
+                    LOG(FATAL) << "__builtin_unreachable";
+                    __builtin_unreachable();
                 }
                 auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
-                v = (DecimalPrimitiveType)value;
-                _offset += _type_length;
+                v = (DecimalPrimitiveType)result_value;
+                _offset += fixed_type_length;
             }
             break;
         }
@@ -440,6 +504,62 @@ Status FixLengthPlainDecoder::_decode_primitive_decimal(MutableColumnPtr& doris_
                                                         DataTypePtr& data_type,
                                                         ColumnSelectVector& select_vector) {
     init_decimal_converter<DecimalPrimitiveType>(data_type);
+    DecimalScaleParams& scale_params = _decode_params->decimal_scale;
+#define M(FixedTypeLength, T, ScaleType)                                                      \
+    case FixedTypeLength:                                                                     \
+        return _decode_primitive_decimal_internal<DecimalPrimitiveType, DecimalPhysicalType,  \
+                                                  has_filter, FixedTypeLength, T, ScaleType>( \
+                doris_column, data_type, select_vector);
+
+#define APPLY_FOR_DECIMALS(ScaleType) \
+    M(1, int64_t, ScaleType)          \
+    M(2, int64_t, ScaleType)          \
+    M(3, int64_t, ScaleType)          \
+    M(4, int64_t, ScaleType)          \
+    M(5, int64_t, ScaleType)          \
+    M(6, int64_t, ScaleType)          \
+    M(7, int64_t, ScaleType)          \
+    M(8, int64_t, ScaleType)          \
+    M(9, int128_t, ScaleType)         \
+    M(10, int128_t, ScaleType)        \
+    M(11, int128_t, ScaleType)        \
+    M(12, int128_t, ScaleType)        \
+    M(13, int128_t, ScaleType)        \
+    M(14, int128_t, ScaleType)        \
+    M(15, int128_t, ScaleType)        \
+    M(16, int128_t, ScaleType)
+
+    if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_UP)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::SCALE_DOWN)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    } else {
+        switch (_type_length) {
+            APPLY_FOR_DECIMALS(DecimalScaleParams::NO_SCALE)
+        default:
+            LOG(FATAL) << "__builtin_unreachable";
+            __builtin_unreachable();
+        }
+    }
+    return Status::OK();
+#undef APPLY_FOR_DECIMALS
+#undef M
+}
+
+template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+          int fixed_type_length, typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+Status FixLengthPlainDecoder::_decode_primitive_decimal_internal(
+        MutableColumnPtr& doris_column, DataTypePtr& data_type, ColumnSelectVector& select_vector) {
     auto& column_data =
             static_cast<ColumnDecimal<Decimal<DecimalPrimitiveType>>&>(*doris_column).get_data();
     size_t data_index = column_data.size();
@@ -452,12 +572,16 @@ Status FixLengthPlainDecoder::_decode_primitive_decimal(MutableColumnPtr& doris_
         case ColumnSelectVector::CONTENT: {
             for (size_t i = 0; i < run_length; ++i) {
                 char* buf_start = _data->data + _offset;
-                // we should use decimal128 to scale up/down
-                Int128 value = *reinterpret_cast<DecimalPhysicalType*>(buf_start);
-                if (scale_params.scale_type == DecimalScaleParams::SCALE_UP) {
+                ValueCopyType value = *reinterpret_cast<DecimalPhysicalType*>(buf_start);
+                if constexpr (ScaleType == DecimalScaleParams::SCALE_UP) {
                     value *= scale_params.scale_factor;
-                } else if (scale_params.scale_type == DecimalScaleParams::SCALE_DOWN) {
+                } else if constexpr (ScaleType == DecimalScaleParams::SCALE_DOWN) {
                     value /= scale_params.scale_factor;
+                } else if constexpr (ScaleType == DecimalScaleParams::NO_SCALE) {
+                    // do nothing
+                } else {
+                    LOG(FATAL) << "__builtin_unreachable";
+                    __builtin_unreachable();
                 }
                 auto& v = reinterpret_cast<DecimalPrimitiveType&>(column_data[data_index++]);
                 v = (DecimalPrimitiveType)value;
diff --git a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
index 0b3a6e19453..96d674e2588 100644
--- a/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
+++ b/be/src/vec/exec/format/parquet/fix_length_plain_decoder.h
@@ -71,5 +71,17 @@ protected:
     Status _decode_string(MutableColumnPtr& doris_column, ColumnSelectVector& select_vector);
 
     tparquet::Type::type _physical_type;
+
+private:
+    template <typename DecimalPrimitiveType, bool has_filter, int fixed_type_length,
+              typename ValueCopyType, DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_binary_decimal_internal(MutableColumnPtr& doris_column, DataTypePtr& data_type,
+                                           ColumnSelectVector& select_vector);
+    template <typename DecimalPrimitiveType, typename DecimalPhysicalType, bool has_filter,
+              int fixed_type_length, typename ValueCopyType,
+              DecimalScaleParams::ScaleType ScaleType>
+    Status _decode_primitive_decimal_internal(MutableColumnPtr& doris_column,
+                                              DataTypePtr& data_type,
+                                              ColumnSelectVector& select_vector);
 };
 } // namespace doris::vectorized


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org