You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/10/12 01:43:55 UTC
[GitHub] [doris] morningman commented on a diff in pull request #13184: [feature-wip](parquet-reader) fix string test
morningman commented on code in PR #13184:
URL: https://github.com/apache/doris/pull/13184#discussion_r992901758
##########
be/src/vec/exec/format/parquet/parquet_pred_cmp.h:
##########
@@ -76,40 +56,85 @@ namespace doris::vectorized {
return true; \
}
-static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values,
- const char* min_bytes, const char* max_bytes) {
- switch (conjunct_type) {
+struct ColumnMinMaxParams {
+ PrimitiveType conjunct_type;
+ tparquet::Type::type parquet_type;
+ void* value;
+ // Use for decimal type
+ int32_t parquet_precision;
+ int32_t parquet_scale;
+ int32_t parquet_type_length;
+ // Use for in predicate
+ std::vector<void*> in_pred_values;
+ const char* min_bytes;
+ const char* max_bytes;
+};
+
+template <typename T>
+static void _align_decimal_v2_scale(T* conjunct_value, int32_t value_scale, T* parquet_value,
+ int32_t parquet_scale) {
+ if (value_scale > parquet_scale) {
+ *parquet_value = *parquet_value * common::exp10_i32(value_scale - parquet_scale);
+ } else if (value_scale < parquet_scale) {
+ *conjunct_value = *conjunct_value * common::exp10_i32(parquet_scale - value_scale);
+ }
+}
+
+template <typename T>
+static void _decode_decimal_v2_to_primary(ColumnMinMaxParams& params, const char* raw_parquet_val,
Review Comment:
```suggestion
static void _decode_decimal_v2_to_primary(const ColumnMinMaxParams& params, const char* raw_parquet_val,
```
##########
be/src/vec/exec/format/parquet/parquet_pred_cmp.h:
##########
@@ -120,34 +145,74 @@ static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred
return false;
}
-static bool _eval_eq(PrimitiveType conjunct_type, void* value, const char* min_bytes,
- const char* max_bytes) {
- switch (conjunct_type) {
+static bool _eval_eq(ColumnMinMaxParams& params) {
Review Comment:
```suggestion
static bool _eval_eq(const ColumnMinMaxParams& params) {
```
Same as all others
##########
be/src/vec/exec/format/parquet/parquet_pred_cmp.h:
##########
@@ -76,40 +56,85 @@ namespace doris::vectorized {
return true; \
}
-static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values,
- const char* min_bytes, const char* max_bytes) {
- switch (conjunct_type) {
+struct ColumnMinMaxParams {
+ PrimitiveType conjunct_type;
+ tparquet::Type::type parquet_type;
+ void* value;
+ // Use for decimal type
+ int32_t parquet_precision;
+ int32_t parquet_scale;
+ int32_t parquet_type_length;
+ // Use for in predicate
+ std::vector<void*> in_pred_values;
+ const char* min_bytes;
+ const char* max_bytes;
+};
+
+template <typename T>
+static void _align_decimal_v2_scale(T* conjunct_value, int32_t value_scale, T* parquet_value,
+ int32_t parquet_scale) {
+ if (value_scale > parquet_scale) {
+ *parquet_value = *parquet_value * common::exp10_i32(value_scale - parquet_scale);
+ } else if (value_scale < parquet_scale) {
+ *conjunct_value = *conjunct_value * common::exp10_i32(parquet_scale - value_scale);
+ }
+}
+
+template <typename T>
+static void _decode_decimal_v2_to_primary(ColumnMinMaxParams& params, const char* raw_parquet_val,
+ T* out_value, T* parquet_val) {
+ *parquet_val = reinterpret_cast<const T*>(raw_parquet_val)[0];
+ DecimalV2Value conjunct_value = *((DecimalV2Value*)params.value);
+ *out_value = conjunct_value.value();
+ _align_decimal_v2_scale(out_value, conjunct_value.scale(), parquet_val, params.parquet_scale);
+}
+
+static Int128 _decode_value_to_int128(ColumnMinMaxParams& params, const char* raw_parquet_val) {
Review Comment:
```suggestion
static Int128 _decode_value_to_int128(const ColumnMinMaxParams& params, const char* raw_parquet_val) {
```
##########
be/src/vec/exec/format/parquet/parquet_pred_cmp.h:
##########
@@ -120,34 +145,74 @@ static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred
return false;
}
-static bool _eval_eq(PrimitiveType conjunct_type, void* value, const char* min_bytes,
- const char* max_bytes) {
- switch (conjunct_type) {
+static bool _eval_eq(ColumnMinMaxParams& params) {
+ switch (params.conjunct_type) {
case TYPE_TINYINT: {
- _PLAIN_DECODE(int16_t, value, min_bytes, max_bytes, conjunct_value, min, max)
+ _PLAIN_DECODE(int16_t, params.value, params.min_bytes, params.max_bytes, conjunct_value,
+ min, max)
_FILTER_GROUP_BY_EQ_PRED(conjunct_value, min, max)
break;
}
case TYPE_SMALLINT: {
- _PLAIN_DECODE(int16_t, value, min_bytes, max_bytes, conjunct_value, min, max)
+ _PLAIN_DECODE(int16_t, params.value, params.min_bytes, params.max_bytes, conjunct_value,
+ min, max)
_FILTER_GROUP_BY_EQ_PRED(conjunct_value, min, max)
break;
}
+ case TYPE_DECIMAL32:
case TYPE_INT: {
- _PLAIN_DECODE(int32_t, value, min_bytes, max_bytes, conjunct_value, min, max)
+ _PLAIN_DECODE(int32_t, params.value, params.min_bytes, params.max_bytes, conjunct_value,
+ min, max)
_FILTER_GROUP_BY_EQ_PRED(conjunct_value, min, max)
break;
}
+ case TYPE_DECIMAL64:
case TYPE_BIGINT: {
- _PLAIN_DECODE(int64_t, value, min_bytes, max_bytes, conjunct_value, min, max)
+ _PLAIN_DECODE(int64_t, params.value, params.min_bytes, params.max_bytes, conjunct_value,
+ min, max)
_FILTER_GROUP_BY_EQ_PRED(conjunct_value, min, max)
break;
}
+ case TYPE_DECIMALV2: {
+ if (params.parquet_type == tparquet::Type::INT32) {
+ int32_t min_value = reinterpret_cast<const int32_t*>(params.min_bytes)[0];
+ int32_t max_value = reinterpret_cast<const int32_t*>(params.max_bytes)[0];
+ DecimalV2Value conjunct_value = *((DecimalV2Value*)params.value);
Review Comment:
Same as all others
##########
be/src/vec/exec/format/parquet/parquet_pred_cmp.h:
##########
@@ -76,40 +56,85 @@ namespace doris::vectorized {
return true; \
}
-static bool _eval_in_val(PrimitiveType conjunct_type, std::vector<void*> in_pred_values,
- const char* min_bytes, const char* max_bytes) {
- switch (conjunct_type) {
+struct ColumnMinMaxParams {
+ PrimitiveType conjunct_type;
+ tparquet::Type::type parquet_type;
+ void* value;
+ // Use for decimal type
+ int32_t parquet_precision;
+ int32_t parquet_scale;
+ int32_t parquet_type_length;
+ // Use for in predicate
+ std::vector<void*> in_pred_values;
+ const char* min_bytes;
+ const char* max_bytes;
+};
+
+template <typename T>
+static void _align_decimal_v2_scale(T* conjunct_value, int32_t value_scale, T* parquet_value,
+ int32_t parquet_scale) {
+ if (value_scale > parquet_scale) {
+ *parquet_value = *parquet_value * common::exp10_i32(value_scale - parquet_scale);
+ } else if (value_scale < parquet_scale) {
+ *conjunct_value = *conjunct_value * common::exp10_i32(parquet_scale - value_scale);
+ }
+}
+
+template <typename T>
+static void _decode_decimal_v2_to_primary(ColumnMinMaxParams& params, const char* raw_parquet_val,
+ T* out_value, T* parquet_val) {
+ *parquet_val = reinterpret_cast<const T*>(raw_parquet_val)[0];
+ DecimalV2Value conjunct_value = *((DecimalV2Value*)params.value);
+ *out_value = conjunct_value.value();
+ _align_decimal_v2_scale(out_value, conjunct_value.scale(), parquet_val, params.parquet_scale);
+}
+
+static Int128 _decode_value_to_int128(ColumnMinMaxParams& params, const char* raw_parquet_val) {
+ const uint8_t* buf = reinterpret_cast<const uint8_t*>(raw_parquet_val);
+ int32_t length = params.parquet_type_length;
+ Int128 value = buf[0] & 0x80 ? -1 : 0;
+ memcpy(reinterpret_cast<uint8_t*>(&value) + sizeof(value) - length, buf, length);
+ return BigEndian::ToHost128(value);
+}
+
+static bool _eval_in_val(ColumnMinMaxParams& params) {
Review Comment:
```suggestion
static bool _eval_in_val(const ColumnMinMaxParams& params) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org