You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/07/04 01:22:33 UTC
[doris] branch master updated: [Enhancement][Storage] refactor Comparison Predicates (#10380)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0b251481d5 [Enhancement][Storage] refactor Comparison Predicates (#10380)
0b251481d5 is described below
commit 0b251481d519ce10b5235f22d1d859b3ba75fbea
Author: Pxl <px...@qq.com>
AuthorDate: Mon Jul 4 09:22:27 2022 +0800
[Enhancement][Storage] refactor Comparison Predicates (#10380)
---
be/src/olap/CMakeLists.txt | 1 -
be/src/olap/block_column_predicate.cpp | 8 +
be/src/olap/bloom_filter_predicate.h | 27 +-
be/src/olap/column_predicate.h | 17 +-
be/src/olap/comparison_predicate.cpp | 750 ---------------------
be/src/olap/comparison_predicate.h | 498 +++++++++++++-
be/src/olap/in_list_predicate.h | 7 +-
be/src/olap/null_predicate.cpp | 12 +-
be/src/olap/null_predicate.h | 17 +-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 14 +-
be/src/vec/columns/column.h | 3 +
be/src/vec/columns/column_dictionary.h | 2 +-
be/src/vec/columns/column_nullable.h | 4 +
13 files changed, 532 insertions(+), 828 deletions(-)
diff --git a/be/src/olap/CMakeLists.txt b/be/src/olap/CMakeLists.txt
index c003c2bdc4..805cc9458f 100644
--- a/be/src/olap/CMakeLists.txt
+++ b/be/src/olap/CMakeLists.txt
@@ -36,7 +36,6 @@ add_library(Olap STATIC
collect_iterator.cpp
compaction.cpp
compaction_permit_limiter.cpp
- comparison_predicate.cpp
compress.cpp
convert_rowset.cpp
cumulative_compaction.cpp
diff --git a/be/src/olap/block_column_predicate.cpp b/be/src/olap/block_column_predicate.cpp
index fc35bd8de9..251448ca15 100644
--- a/be/src/olap/block_column_predicate.cpp
+++ b/be/src/olap/block_column_predicate.cpp
@@ -66,6 +66,14 @@ void SingleColumnBlockPredicate::evaluate_vec(vectorized::MutableColumns& block,
bool* flags) const {
auto column_id = _predicate->column_id();
auto& column = block[column_id];
+
+ // Dictionary column should do something to initial.
+ if (PredicateTypeTraits::is_range(_predicate->type())) {
+ column->convert_dict_codes_if_necessary();
+ } else if (PredicateTypeTraits::is_bloom_filter(_predicate->type())) {
+ column->generate_hash_values_for_runtime_filter();
+ }
+
_predicate->evaluate_vec(*column, size, flags);
}
diff --git a/be/src/olap/bloom_filter_predicate.h b/be/src/olap/bloom_filter_predicate.h
index 52ec6c920c..23a2e4a473 100644
--- a/be/src/olap/bloom_filter_predicate.h
+++ b/be/src/olap/bloom_filter_predicate.h
@@ -23,14 +23,11 @@
#include "exprs/bloomfilter_predicate.h"
#include "olap/column_predicate.h"
-#include "olap/field.h"
-#include "runtime/string_value.hpp"
#include "runtime/vectorized_row_batch.h"
#include "vec/columns/column_dictionary.h"
#include "vec/columns/column_nullable.h"
#include "vec/columns/column_vector.h"
#include "vec/columns/predicate_column.h"
-#include "vec/utils/util.hpp"
namespace doris {
@@ -65,18 +62,20 @@ public:
return Status::OK();
}
- uint16_t evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const override;
+ uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const override;
private:
template <bool is_nullable, typename file_type = void>
- uint16_t evaluate(vectorized::IColumn& column, uint8_t* null_map, uint16_t* sel,
+ uint16_t evaluate(const vectorized::IColumn& column, const uint8_t* null_map, uint16_t* sel,
uint16_t size) const {
- if constexpr (is_nullable) DCHECK(null_map);
+ if constexpr (is_nullable) {
+ DCHECK(null_map);
+ }
uint16_t new_size = 0;
if (column.is_column_dictionary()) {
- auto* dict_col = reinterpret_cast<vectorized::ColumnDictI32*>(&column);
- dict_col->generate_hash_values_for_runtime_filter();
+ auto* dict_col = reinterpret_cast<const vectorized::ColumnDictI32*>(&column);
for (uint16_t i = 0; i < size; i++) {
uint16_t idx = sel[i];
sel[new_size] = idx;
@@ -89,13 +88,15 @@ private:
}
} else {
uint24_t tmp_uint24_value;
- auto get_column_data = [](vectorized::IColumn& column) {
+ auto get_column_data = [](const vectorized::IColumn& column) {
if constexpr (std::is_same_v<file_type, uint24_t>) {
- return reinterpret_cast<vectorized::PredicateColumnType<uint32_t>*>(&column)
+ return reinterpret_cast<const vectorized::PredicateColumnType<uint32_t>*>(
+ &column)
->get_data()
.data();
} else {
- return reinterpret_cast<vectorized::PredicateColumnType<file_type>*>(&column)
+ return reinterpret_cast<const vectorized::PredicateColumnType<file_type>*>(
+ &column)
->get_data()
.data();
}
@@ -170,7 +171,7 @@ void BloomFilterColumnPredicate<T>::evaluate(ColumnBlock* block, uint16_t* sel,
}
template <PrimitiveType T>
-uint16_t BloomFilterColumnPredicate<T>::evaluate(vectorized::IColumn& column, uint16_t* sel,
+uint16_t BloomFilterColumnPredicate<T>::evaluate(const vectorized::IColumn& column, uint16_t* sel,
uint16_t size) const {
uint16_t new_size = 0;
using FT = typename PredicatePrimitiveTypeTraits<T>::PredicateFieldType;
@@ -178,7 +179,7 @@ uint16_t BloomFilterColumnPredicate<T>::evaluate(vectorized::IColumn& column, ui
return size;
}
if (column.is_nullable()) {
- auto* nullable_col = reinterpret_cast<vectorized::ColumnNullable*>(&column);
+ auto* nullable_col = reinterpret_cast<const vectorized::ColumnNullable*>(&column);
auto& null_map_data = nullable_col->get_null_map_column().get_data();
new_size = evaluate<true, FT>(nullable_col->get_nested_column(), null_map_data.data(), sel,
size);
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 86580875e6..b02cdc0146 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -47,6 +47,14 @@ enum class PredicateType {
BF = 11, // BloomFilter
};
+struct PredicateTypeTraits {
+ static constexpr bool is_range(PredicateType type) {
+ return (type == PredicateType::LT || type == PredicateType::LE ||
+ type == PredicateType::GT || type == PredicateType::GE);
+ }
+ static constexpr bool is_bloom_filter(PredicateType type) { return type == PredicateType::BF; }
+};
+
class ColumnPredicate {
public:
explicit ColumnPredicate(uint32_t column_id, bool opposite = false)
@@ -73,18 +81,19 @@ public:
// evaluate predicate on IColumn
// a short circuit eval way
- virtual uint16_t evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const {
+ virtual uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const {
return size;
};
- virtual void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ virtual void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const {};
- virtual void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ virtual void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const {};
// used to evaluate pre read column in lazy matertialization
// now only support integer/float
// a vectorized eval way
- virtual void evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const {
+ virtual void evaluate_vec(const vectorized::IColumn& column, uint16_t size, bool* flags) const {
DCHECK(false) << "should not reach here";
}
diff --git a/be/src/olap/comparison_predicate.cpp b/be/src/olap/comparison_predicate.cpp
deleted file mode 100644
index 870be42185..0000000000
--- a/be/src/olap/comparison_predicate.cpp
+++ /dev/null
@@ -1,750 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#include "olap/comparison_predicate.h"
-
-#include "common/logging.h"
-#include "olap/schema.h"
-#include "runtime/string_value.hpp"
-#include "runtime/vectorized_row_batch.h"
-#include "vec/columns/column_dictionary.h"
-#include "vec/columns/column_nullable.h"
-#include "vec/columns/column_vector.h"
-#include "vec/columns/predicate_column.h"
-
-namespace doris {
-
-#define COMPARISON_PRED_CONSTRUCTOR(CLASS) \
- template <class T> \
- CLASS<T>::CLASS(uint32_t column_id, const T& value, bool opposite) \
- : ColumnPredicate(column_id, opposite), _value(value) {}
-
-COMPARISON_PRED_CONSTRUCTOR(EqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR(NotEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR(LessPredicate)
-COMPARISON_PRED_CONSTRUCTOR(LessEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR(GreaterPredicate)
-COMPARISON_PRED_CONSTRUCTOR(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_CONSTRUCTOR_STRING(CLASS) \
- template <> \
- CLASS<StringValue>::CLASS(uint32_t column_id, const StringValue& value, bool opposite) \
- : ColumnPredicate(column_id, opposite) { \
- _value.len = value.len; \
- _value.ptr = value.ptr; \
- }
-
-COMPARISON_PRED_CONSTRUCTOR_STRING(EqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_STRING(NotEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_STRING(LessPredicate)
-COMPARISON_PRED_CONSTRUCTOR_STRING(LessEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_STRING(GreaterPredicate)
-COMPARISON_PRED_CONSTRUCTOR_STRING(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_EVALUATE(CLASS, OP) \
- template <class T> \
- void CLASS<T>::evaluate(VectorizedRowBatch* batch) const { \
- uint16_t n = batch->size(); \
- if (n == 0) { \
- return; \
- } \
- uint16_t* sel = batch->selected(); \
- const T* col_vector = reinterpret_cast<const T*>(batch->column(_column_id)->col_data()); \
- uint16_t new_size = 0; \
- if (batch->column(_column_id)->no_nulls()) { \
- if (batch->selected_in_use()) { \
- for (uint16_t j = 0; j != n; ++j) { \
- uint16_t i = sel[j]; \
- sel[new_size] = i; \
- new_size += (col_vector[i] OP _value); \
- } \
- batch->set_size(new_size); \
- } else { \
- for (uint16_t i = 0; i != n; ++i) { \
- sel[new_size] = i; \
- new_size += (col_vector[i] OP _value); \
- } \
- if (new_size < n) { \
- batch->set_size(new_size); \
- batch->set_selected_in_use(true); \
- } \
- } \
- } else { \
- bool* is_null = batch->column(_column_id)->is_null(); \
- if (batch->selected_in_use()) { \
- for (uint16_t j = 0; j != n; ++j) { \
- uint16_t i = sel[j]; \
- sel[new_size] = i; \
- new_size += (!is_null[i] && (col_vector[i] OP _value)); \
- } \
- batch->set_size(new_size); \
- } else { \
- for (uint16_t i = 0; i != n; ++i) { \
- sel[new_size] = i; \
- new_size += (!is_null[i] && (col_vector[i] OP _value)); \
- } \
- if (new_size < n) { \
- batch->set_size(new_size); \
- batch->set_selected_in_use(true); \
- } \
- } \
- } \
- }
-
-COMPARISON_PRED_EVALUATE(EqualPredicate, ==)
-COMPARISON_PRED_EVALUATE(NotEqualPredicate, !=)
-COMPARISON_PRED_EVALUATE(LessPredicate, <)
-COMPARISON_PRED_EVALUATE(LessEqualPredicate, <=)
-COMPARISON_PRED_EVALUATE(GreaterPredicate, >)
-COMPARISON_PRED_EVALUATE(GreaterEqualPredicate, >=)
-
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(CLASS, OP) \
- template <class T> \
- void CLASS<T>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const { \
- uint16_t new_size = 0; \
- if (block->is_nullable()) { \
- for (uint16_t i = 0; i < *size; ++i) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr()); \
- auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
- new_size += _opposite ? !result : result; \
- } \
- } else { \
- for (uint16_t i = 0; i < *size; ++i) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr()); \
- auto result = (*cell_value OP _value); \
- new_size += _opposite ? !result : result; \
- } \
- } \
- *size = new_size; \
- }
-
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(EqualPredicate, ==)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(NotEqualPredicate, !=)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(LessPredicate, <)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(LessEqualPredicate, <=)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterPredicate, >)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE(GreaterEqualPredicate, >=)
-
-// todo(zeno) define interface in IColumn to simplify code
-// If 1 OP 0 returns true, it means the predicate is > or >=
-// If 1 OP 1 returns true, it means the predicate is >= or <=
-// by this way, avoid redundant code
-#define COMPARISON_PRED_COLUMN_EVALUATE(CLASS, OP, IS_RANGE) \
- template <class T> \
- uint16_t CLASS<T>::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const { \
- uint16_t new_size = 0; \
- if (column.is_nullable()) { \
- auto* nullable_col = \
- vectorized::check_and_get_column<vectorized::ColumnNullable>(column); \
- auto& null_bitmap = reinterpret_cast<const vectorized::ColumnUInt8&>( \
- nullable_col->get_null_map_column()) \
- .get_data(); \
- auto& nested_col = nullable_col->get_nested_column(); \
- if (nested_col.is_column_dictionary()) { \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto* nested_col_ptr = vectorized::check_and_get_column< \
- vectorized::ColumnDictionary<vectorized::Int32>>(nested_col); \
- auto& data_array = nested_col_ptr->get_data(); \
- auto dict_code = \
- IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : nested_col_ptr->find_code(_value); \
- for (uint16_t i = 0; i < size; i++) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const auto& cell_value = data_array[idx]; \
- bool ret = !null_bitmap[idx] && (cell_value OP dict_code); \
- new_size += _opposite ? !ret : ret; \
- } \
- } \
- } else { \
- auto* nested_col_ptr = \
- vectorized::check_and_get_column<vectorized::PredicateColumnType<T>>( \
- nested_col); \
- auto& data_array = nested_col_ptr->get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const auto& cell_value = reinterpret_cast<const T&>(data_array[idx]); \
- bool ret = !null_bitmap[idx] && (cell_value OP _value); \
- new_size += _opposite ? !ret : ret; \
- } \
- } \
- } else if (column.is_column_dictionary()) { \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto& dict_col = \
- reinterpret_cast<vectorized::ColumnDictionary<vectorized::Int32>&>( \
- column); \
- auto& data_array = dict_col.get_data(); \
- auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : dict_col.find_code(_value); \
- for (uint16_t i = 0; i < size; ++i) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const auto& cell_value = data_array[idx]; \
- bool ret = cell_value OP dict_code; \
- new_size += _opposite ? !ret : ret; \
- } \
- } \
- } else { \
- auto& pred_column_ref = reinterpret_cast<vectorized::PredicateColumnType<T>&>(column); \
- auto& data_array = pred_column_ref.get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- uint16_t idx = sel[i]; \
- sel[new_size] = idx; \
- const auto& cell_value = reinterpret_cast<const T&>(data_array[idx]); \
- auto ret = cell_value OP _value; \
- new_size += _opposite ? !ret : ret; \
- } \
- } \
- return new_size; \
- }
-
-COMPARISON_PRED_COLUMN_EVALUATE(EqualPredicate, ==, false)
-COMPARISON_PRED_COLUMN_EVALUATE(NotEqualPredicate, !=, false)
-COMPARISON_PRED_COLUMN_EVALUATE(LessPredicate, <, true)
-COMPARISON_PRED_COLUMN_EVALUATE(LessEqualPredicate, <=, true)
-COMPARISON_PRED_COLUMN_EVALUATE(GreaterPredicate, >, true)
-COMPARISON_PRED_COLUMN_EVALUATE(GreaterEqualPredicate, >=, true)
-
-// todo(wb) for date type we use uint32_t to save it but using Predicate<uint24> to evaluate it.
-// This is done for compatibility with Row Version predicate.
-// We can use Predicate<uint32_t> for date after Row Version is removed.
-#define COMPARISON_PRED_COLUMN_EVALUATE_VEC(CLASS, OP, IS_RANGE) \
- template <class T> \
- void CLASS<T>::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const { \
- if (column.is_nullable()) { \
- auto* nullable_column = \
- vectorized::check_and_get_column<vectorized::ColumnNullable>(column); \
- auto& null_bitmap = reinterpret_cast<const vectorized::ColumnVector<uint8_t>&>( \
- *(nullable_column->get_null_map_column_ptr())) \
- .get_data(); \
- auto& nested_col = nullable_column->get_nested_column(); \
- if constexpr (std::is_same_v<T, uint24_t>) { \
- auto& predicate_column = \
- reinterpret_cast<const vectorized::PredicateColumnType<uint32_t>&>( \
- nested_col); \
- uint32_t int32_val = 0; \
- char* int32_val_ptr = (char*)&int32_val; \
- memory_copy(int32_val_ptr, _value.get_data(), sizeof(uint24_t)); \
- auto& data_array_uint32_t = predicate_column.get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = (data_array_uint32_t[i] OP int32_val) && (!null_bitmap[i]); \
- } \
- } else { \
- if (nested_col.is_column_dictionary()) { \
- if constexpr (IS_RANGE) column.convert_dict_codes_if_necessary(); \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto dict_col = vectorized::check_and_get_column< \
- vectorized::ColumnDictionary<vectorized::Int32>>(nested_col); \
- auto dict_code = \
- IS_RANGE ? dict_col->find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : dict_col->find_code(_value); \
- auto& data_array = dict_col->get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = (data_array[i] OP dict_code) && (!null_bitmap[i]); \
- } \
- } \
- } else { \
- auto& data_array = \
- reinterpret_cast<const vectorized::PredicateColumnType<T>&>( \
- nested_col) \
- .get_data(); \
- \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = (data_array[i] OP _value) && (!null_bitmap[i]); \
- } \
- } \
- } \
- } else { \
- if constexpr (std::is_same_v<T, uint24_t>) { \
- auto& predicate_column = \
- reinterpret_cast<vectorized::PredicateColumnType<uint32_t>&>(column); \
- uint32_t int32_val = 0; \
- char* int32_val_ptr = (char*)&int32_val; \
- memory_copy(int32_val_ptr, _value.get_data(), sizeof(uint24_t)); \
- auto& data_array = predicate_column.get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = data_array[i] OP int32_val; \
- } \
- } else { \
- if (column.is_column_dictionary()) { \
- if constexpr (IS_RANGE) column.convert_dict_codes_if_necessary(); \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto& dict_col = reinterpret_cast< \
- vectorized::ColumnDictionary<vectorized::Int32>&>(column); \
- auto dict_code = \
- IS_RANGE ? dict_col.find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : dict_col.find_code(_value); \
- auto& data_array = dict_col.get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = data_array[i] OP dict_code; \
- } \
- } \
- } else { \
- auto& predicate_column = \
- reinterpret_cast<vectorized::PredicateColumnType<T>&>(column); \
- auto& data_array = predicate_column.get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = data_array[i] OP _value; \
- } \
- } \
- } \
- } \
- if (_opposite) { \
- for (uint16_t i = 0; i < size; i++) { \
- flags[i] = !flags[i]; \
- } \
- } \
- }
-
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(EqualPredicate, ==, false)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(NotEqualPredicate, !=, false)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(LessPredicate, <, true)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(LessEqualPredicate, <=, true)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(GreaterPredicate, >, true)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC(GreaterEqualPredicate, >=, true)
-
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(CLASS, OP, BOOL_NAME, BOOL_OP, SHORT_OP) \
- template <class T> \
- void CLASS<T>::evaluate_##BOOL_NAME(ColumnBlock* block, uint16_t* sel, uint16_t size, \
- bool* flags) const { \
- if (block->is_nullable()) { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (SHORT_OP(flags[i])) continue; \
- uint16_t idx = sel[i]; \
- const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr()); \
- auto result = (!block->cell(idx).is_null() && (*cell_value OP _value)); \
- flags[i] = flags[i] BOOL_OP(_opposite ? !result : result); \
- } \
- } else { \
- for (uint16_t i = 0; i < size; ++i) { \
- if (flags[i]) continue; \
- uint16_t idx = sel[i]; \
- const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr()); \
- auto result = (*cell_value OP _value); \
- flags[i] = flags[i] BOOL_OP(_opposite ? !result : result); \
- } \
- } \
- }
-
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL2(BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(EqualPredicate, ==, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(NotEqualPredicate, !=, BOOL_NAME, BOOL_OP, \
- SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(LessPredicate, <, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(LessEqualPredicate, <=, BOOL_NAME, BOOL_OP, \
- SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(GreaterPredicate, >, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL(GreaterEqualPredicate, >=, BOOL_NAME, BOOL_OP, \
- SHORT_OP)
-
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL2(or, |, )
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_BOOL2(and, &, !)
-
-#define COMPARISON_PRED_COLUMN_EVALUATE_BOOL(CLASS, OP, IS_RANGE, BOOL_NAME, BOOL_OP, SHORT_OP) \
- template <class T> \
- void CLASS<T>::evaluate_##BOOL_NAME(vectorized::IColumn& column, uint16_t* sel, uint16_t size, \
- bool* flags) const { \
- if (column.is_nullable()) { \
- auto* nullable_column = \
- vectorized::check_and_get_column<vectorized::ColumnNullable>(column); \
- auto& nested_col = nullable_column->get_nested_column(); \
- auto& null_bitmap = reinterpret_cast<const vectorized::ColumnVector<uint8_t>&>( \
- *(nullable_column->get_null_map_column_ptr())) \
- .get_data(); \
- if (nested_col.is_column_dictionary()) { \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto* nested_col_ptr = vectorized::check_and_get_column< \
- vectorized::ColumnDictionary<vectorized::Int32>>(nested_col); \
- auto& data_array = nested_col_ptr->get_data(); \
- auto dict_code = \
- IS_RANGE ? nested_col_ptr->find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : nested_col_ptr->find_code(_value); \
- for (uint16_t i = 0; i < size; i++) { \
- if (SHORT_OP(flags[i])) continue; \
- uint16_t idx = sel[i]; \
- bool ret = !null_bitmap[idx] && (data_array[idx] OP dict_code); \
- flags[i] = flags[i] BOOL_OP(_opposite ? !ret : ret); \
- } \
- } \
- } else { \
- auto& data_array = \
- reinterpret_cast<const vectorized::PredicateColumnType<T>&>(nested_col) \
- .get_data(); \
- for (uint16_t i = 0; i < size; i++) { \
- if (SHORT_OP(flags[i])) continue; \
- uint16_t idx = sel[i]; \
- bool ret = !null_bitmap[idx] && (data_array[idx] OP _value); \
- flags[i] = flags[i] BOOL_OP(_opposite ? !ret : ret); \
- } \
- } \
- } else if (column.is_column_dictionary()) { \
- if constexpr (std::is_same_v<T, StringValue>) { \
- auto& dict_col = \
- reinterpret_cast<vectorized::ColumnDictionary<vectorized::Int32>&>( \
- column); \
- auto& data_array = dict_col.get_data(); \
- auto dict_code = IS_RANGE ? dict_col.find_code_by_bound(_value, 1 OP 0, 1 OP 1) \
- : dict_col.find_code(_value); \
- for (uint16_t i = 0; i < size; i++) { \
- if (SHORT_OP(flags[i])) continue; \
- uint16_t idx = sel[i]; \
- bool ret = data_array[idx] OP dict_code; \
- flags[i] = flags[i] BOOL_OP(_opposite ? !ret : ret); \
- } \
- } \
- } else { \
- auto& predicate_column = \
- reinterpret_cast<vectorized::PredicateColumnType<T>&>(column); \
- auto& data_array = predicate_column.get_data(); \
- for (uint16_t i = 0; i < size; ++i) { \
- if (SHORT_OP(flags[i])) continue; \
- uint16_t idx = sel[i]; \
- bool ret = data_array[idx] OP _value; \
- flags[i] = flags[i] BOOL_OP(_opposite ? !ret : ret); \
- } \
- } \
- }
-
-#define COMPARISON_PRED_COLUMN_EVALUATE_BOOL2(BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(EqualPredicate, ==, false, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(NotEqualPredicate, !=, false, BOOL_NAME, BOOL_OP, \
- SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(LessPredicate, <, true, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(LessEqualPredicate, <=, true, BOOL_NAME, BOOL_OP, \
- SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(GreaterPredicate, >, true, BOOL_NAME, BOOL_OP, SHORT_OP) \
- COMPARISON_PRED_COLUMN_EVALUATE_BOOL(GreaterEqualPredicate, >=, true, BOOL_NAME, BOOL_OP, \
- SHORT_OP)
-
-COMPARISON_PRED_COLUMN_EVALUATE_BOOL2(or, |, )
-COMPARISON_PRED_COLUMN_EVALUATE_BOOL2(and, &, !)
-
-#define BITMAP_COMPARE_EqualPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, roaring) \
- do { \
- if (!s.is_not_found()) { \
- if (!s.ok()) { \
- return s; \
- } \
- if (exact_match) { \
- RETURN_IF_ERROR(iterator->read_bitmap(seeked_ordinal, &roaring)); \
- } \
- } \
- } while (0)
-
-#define BITMAP_COMPARE_NotEqualPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, \
- roaring) \
- do { \
- if (s.is_not_found()) { \
- return Status::OK(); \
- } \
- if (!s.ok()) { \
- return s; \
- } \
- if (!exact_match) { \
- return Status::OK(); \
- } \
- RETURN_IF_ERROR(iterator->read_bitmap(seeked_ordinal, &roaring)); \
- *bitmap -= roaring; \
- return Status::OK(); \
- } while (0)
-
-#define BITMAP_COMPARE_LessPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, roaring) \
- do { \
- if (s.is_not_found()) { \
- return Status::OK(); \
- } \
- if (!s.ok()) { \
- return s; \
- } \
- RETURN_IF_ERROR(iterator->read_union_bitmap(0, seeked_ordinal, &roaring)); \
- } while (0)
-
-#define BITMAP_COMPARE_LessEqualPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, \
- roaring) \
- do { \
- if (s.is_not_found()) { \
- return Status::OK(); \
- } \
- if (!s.ok()) { \
- return s; \
- } \
- if (exact_match) { \
- seeked_ordinal++; \
- } \
- RETURN_IF_ERROR(iterator->read_union_bitmap(0, seeked_ordinal, &roaring)); \
- } while (0)
-
-#define BITMAP_COMPARE_GreaterPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, roaring) \
- do { \
- if (!s.is_not_found()) { \
- if (!s.ok()) { \
- return s; \
- } \
- if (exact_match) { \
- seeked_ordinal++; \
- } \
- RETURN_IF_ERROR(iterator->read_union_bitmap(seeked_ordinal, ordinal_limit, &roaring)); \
- } \
- } while (0)
-
-#define BITMAP_COMPARE_GreaterEqualPredicate(s, exact_match, seeked_ordinal, iterator, bitmap, \
- roaring) \
- do { \
- if (!s.is_not_found()) { \
- if (!s.ok()) { \
- return s; \
- } \
- RETURN_IF_ERROR(iterator->read_union_bitmap(seeked_ordinal, ordinal_limit, &roaring)); \
- } \
- } while (0)
-
-#define BITMAP_COMPARE(CLASS, s, exact_match, seeked_ordinal, iterator, bitmap, roaring) \
- BITMAP_COMPARE_##CLASS(s, exact_match, seeked_ordinal, iterator, bitmap, roaring)
-
-#define COMPARISON_PRED_BITMAP_EVALUATE(CLASS, OP) \
- template <class T> \
- Status CLASS<T>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const { \
- BitmapIndexIterator* iterator = iterators[_column_id]; \
- if (iterator == nullptr) { \
- return Status::OK(); \
- } \
- rowid_t ordinal_limit = iterator->bitmap_nums(); \
- if (iterator->has_null_bitmap()) { \
- ordinal_limit--; \
- roaring::Roaring null_bitmap; \
- RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap)); \
- *bitmap -= null_bitmap; \
- } \
- roaring::Roaring roaring; \
- bool exact_match; \
- Status s = iterator->seek_dictionary(&_value, &exact_match); \
- rowid_t seeked_ordinal = iterator->current_ordinal(); \
- BITMAP_COMPARE(CLASS, s, exact_match, seeked_ordinal, iterator, bitmap, roaring); \
- *bitmap &= roaring; \
- return Status::OK(); \
- }
-
-COMPARISON_PRED_BITMAP_EVALUATE(EqualPredicate, ==)
-COMPARISON_PRED_BITMAP_EVALUATE(NotEqualPredicate, !=)
-COMPARISON_PRED_BITMAP_EVALUATE(LessPredicate, <)
-COMPARISON_PRED_BITMAP_EVALUATE(LessEqualPredicate, <=)
-COMPARISON_PRED_BITMAP_EVALUATE(GreaterPredicate, >)
-COMPARISON_PRED_BITMAP_EVALUATE(GreaterEqualPredicate, >=)
-
-#define COMPARISON_PRED_CONSTRUCTOR_DECLARATION(CLASS) \
- template CLASS<int8_t>::CLASS(uint32_t column_id, const int8_t& value, bool opposite); \
- template CLASS<int16_t>::CLASS(uint32_t column_id, const int16_t& value, bool opposite); \
- template CLASS<int32_t>::CLASS(uint32_t column_id, const int32_t& value, bool opposite); \
- template CLASS<int64_t>::CLASS(uint32_t column_id, const int64_t& value, bool opposite); \
- template CLASS<int128_t>::CLASS(uint32_t column_id, const int128_t& value, bool opposite); \
- template CLASS<float>::CLASS(uint32_t column_id, const float& value, bool opposite); \
- template CLASS<double>::CLASS(uint32_t column_id, const double& value, bool opposite); \
- template CLASS<decimal12_t>::CLASS(uint32_t column_id, const decimal12_t& value, \
- bool opposite); \
- template CLASS<uint24_t>::CLASS(uint32_t column_id, const uint24_t& value, bool opposite); \
- template CLASS<uint32_t>::CLASS(uint32_t column_id, const uint32_t& value, bool opposite); \
- template CLASS<uint64_t>::CLASS(uint32_t column_id, const uint64_t& value, bool opposite); \
- template CLASS<bool>::CLASS(uint32_t column_id, const bool& value, bool opposite);
-
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(EqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(LessPredicate)
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_CONSTRUCTOR_DECLARATION(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_EVALUATE_DECLARATION(CLASS) \
- template void CLASS<int8_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<int16_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<int32_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<int64_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<int128_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<float>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<double>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<decimal12_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<StringValue>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<uint24_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<uint32_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<uint64_t>::evaluate(VectorizedRowBatch* batch) const; \
- template void CLASS<bool>::evaluate(VectorizedRowBatch* batch) const;
-
-COMPARISON_PRED_EVALUATE_DECLARATION(EqualPredicate)
-COMPARISON_PRED_EVALUATE_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_EVALUATE_DECLARATION(LessPredicate)
-COMPARISON_PRED_EVALUATE_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_EVALUATE_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_EVALUATE_DECLARATION(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(CLASS) \
- template void CLASS<int8_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<int16_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<int32_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<int64_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<int128_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<float>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const; \
- template void CLASS<double>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<decimal12_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<StringValue>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<uint24_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<uint32_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<uint64_t>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) \
- const; \
- template void CLASS<bool>::evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const;
-
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(EqualPredicate)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(LessPredicate)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_COLUMN_BLOCK_EVALUATE_DECLARATION(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(CLASS) \
- template Status CLASS<int8_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<int16_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<int32_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<int64_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<int128_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<float>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<double>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<decimal12_t>::evaluate( \
- const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<StringValue>::evaluate( \
- const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<uint24_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<uint32_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<uint64_t>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const; \
- template Status CLASS<bool>::evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* bitmap) const;
-
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(EqualPredicate)
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(LessPredicate)
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_BITMAP_EVALUATE_DECLARATION(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(CLASS) \
- template uint16_t CLASS<int8_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<int16_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<int32_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<int64_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<int128_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<float>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<double>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<decimal12_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<StringValue>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<uint24_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<uint32_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<uint64_t>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const; \
- template uint16_t CLASS<bool>::evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const;
-
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(EqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_DECLARATION(GreaterEqualPredicate)
-
-#define COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(CLASS) \
- template void CLASS<int8_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<int16_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<int32_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<int64_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<int128_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<float>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<double>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<decimal12_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<StringValue>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<uint24_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<uint32_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<uint64_t>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const; \
- template void CLASS<bool>::evaluate_vec(vectorized::IColumn& column, uint16_t size, \
- bool* flags) const;
-
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(EqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(NotEqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(LessPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(LessEqualPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(GreaterPredicate)
-COMPARISON_PRED_COLUMN_EVALUATE_VEC_DECLARATION(GreaterEqualPredicate)
-
-} //namespace doris
diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h
index 40b93fb888..d1fdfcbb81 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -17,46 +17,472 @@
#pragma once
-#include <stdint.h>
+#include <cstdint>
#include "olap/column_predicate.h"
+#include "runtime/vectorized_row_batch.h"
+#include "vec/columns/column_dictionary.h"
namespace doris {
-class VectorizedRowBatch;
-
-#define COMPARISON_PRED_CLASS_DEFINE(CLASS, PT) \
- template <class T> \
- class CLASS : public ColumnPredicate { \
- public: \
- CLASS(uint32_t column_id, const T& value, bool opposite = false); \
- PredicateType type() const override { return PredicateType::PT; } \
- virtual void evaluate(VectorizedRowBatch* batch) const override; \
- void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override; \
- void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- virtual Status evaluate(const Schema& schema, \
- const std::vector<BitmapIndexIterator*>& iterators, \
- uint32_t num_rows, roaring::Roaring* roaring) const override; \
- uint16_t evaluate(vectorized::IColumn& column, uint16_t* sel, \
- uint16_t size) const override; \
- void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size, \
- bool* flags) const override; \
- void evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const override; \
- \
- private: \
- T _value; \
- };
-
-COMPARISON_PRED_CLASS_DEFINE(EqualPredicate, EQ)
-COMPARISON_PRED_CLASS_DEFINE(NotEqualPredicate, NE)
-COMPARISON_PRED_CLASS_DEFINE(LessPredicate, LT)
-COMPARISON_PRED_CLASS_DEFINE(LessEqualPredicate, LE)
-COMPARISON_PRED_CLASS_DEFINE(GreaterPredicate, GT)
-COMPARISON_PRED_CLASS_DEFINE(GreaterEqualPredicate, GE)
+template <class T, PredicateType PT>
+class ComparisonPredicateBase : public ColumnPredicate {
+public:
+ ComparisonPredicateBase(uint32_t column_id, const T& value, bool opposite = false)
+ : ColumnPredicate(column_id, opposite), _value(value) {
+ if constexpr (std::is_same_v<T, uint24_t>) {
+ _value_real = 0;
+ memory_copy(&_value_real, _value.get_data(), sizeof(T));
+ } else {
+ _value_real = _value;
+ }
+ }
+
+ PredicateType type() const override { return PT; }
+
+ void evaluate(VectorizedRowBatch* batch) const override {
+ uint16_t n = batch->size();
+ if (n == 0) {
+ return;
+ }
+ uint16_t* sel = batch->selected();
+ const T* col_vector = reinterpret_cast<const T*>(batch->column(_column_id)->col_data());
+ uint16_t new_size = 0;
+ if (batch->column(_column_id)->no_nulls()) {
+ if (batch->selected_in_use()) {
+ for (uint16_t j = 0; j != n; ++j) {
+ uint16_t i = sel[j];
+ sel[new_size] = i;
+ new_size += _operator(col_vector[i], _value);
+ }
+ batch->set_size(new_size);
+ } else {
+ for (uint16_t i = 0; i != n; ++i) {
+ sel[new_size] = i;
+ new_size += _operator(col_vector[i], _value);
+ }
+ if (new_size < n) {
+ batch->set_size(new_size);
+ batch->set_selected_in_use(true);
+ }
+ }
+ } else {
+ bool* is_null = batch->column(_column_id)->is_null();
+ if (batch->selected_in_use()) {
+ for (uint16_t j = 0; j != n; ++j) {
+ uint16_t i = sel[j];
+ sel[new_size] = i;
+ new_size += (!is_null[i] && _operator(col_vector[i], _value));
+ }
+ batch->set_size(new_size);
+ } else {
+ for (uint16_t i = 0; i != n; ++i) {
+ sel[new_size] = i;
+ new_size += (!is_null[i] && _operator(col_vector[i], _value));
+ }
+ if (new_size < n) {
+ batch->set_size(new_size);
+ batch->set_selected_in_use(true);
+ }
+ }
+ }
+ }
+
+ void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override {
+ uint16_t new_size = 0;
+ if (block->is_nullable()) {
+ for (uint16_t i = 0; i < *size; ++i) {
+ uint16_t idx = sel[i];
+ sel[new_size] = idx;
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = (!block->cell(idx).is_null() && _operator(*cell_value, _value));
+ new_size += _opposite ? !result : result;
+ }
+ } else {
+ for (uint16_t i = 0; i < *size; ++i) {
+ uint16_t idx = sel[i];
+ sel[new_size] = idx;
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = _operator(*cell_value, _value);
+ new_size += _opposite ? !result : result;
+ }
+ }
+ *size = new_size;
+ }
+
+ void evaluate_or(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override {
+ if (block->is_nullable()) {
+ for (uint16_t i = 0; i < size; ++i) {
+ if (flags[i]) {
+ continue;
+ }
+ uint16_t idx = sel[i];
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = (!block->cell(idx).is_null() && _operator(*cell_value, _value));
+ flags[i] = flags[i] | (_opposite ? !result : result);
+ }
+ } else {
+ for (uint16_t i = 0; i < size; ++i) {
+ if (flags[i]) {
+ continue;
+ }
+ uint16_t idx = sel[i];
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = _operator(*cell_value, _value);
+ flags[i] = flags[i] | (_opposite ? !result : result);
+ }
+ }
+ }
+
+ void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size,
+ bool* flags) const override {
+ if (block->is_nullable()) {
+ for (uint16_t i = 0; i < size; ++i) {
+ if (!flags[i]) {
+ continue;
+ }
+ uint16_t idx = sel[i];
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = (!block->cell(idx).is_null() && _operator(*cell_value, _value));
+ flags[i] = flags[i] & (_opposite ? !result : result);
+ }
+ } else {
+ for (uint16_t i = 0; i < size; ++i) {
+ if (flags[i]) {
+ continue;
+ }
+ uint16_t idx = sel[i];
+ const T* cell_value = reinterpret_cast<const T*>(block->cell(idx).cell_ptr());
+ auto result = _operator(*cell_value, _value);
+ flags[i] = flags[i] & (_opposite ? !result : result);
+ }
+ }
+ }
+
+ Status evaluate(const Schema& schema, const std::vector<BitmapIndexIterator*>& iterators,
+ uint32_t num_rows, roaring::Roaring* bitmap) const override {
+ BitmapIndexIterator* iterator = iterators[_column_id];
+ if (iterator == nullptr) {
+ return Status::OK();
+ }
+
+ rowid_t ordinal_limit = iterator->bitmap_nums();
+ if (iterator->has_null_bitmap()) {
+ ordinal_limit--;
+ roaring::Roaring null_bitmap;
+ RETURN_IF_ERROR(iterator->read_null_bitmap(&null_bitmap));
+ *bitmap -= null_bitmap;
+ }
+
+ roaring::Roaring roaring;
+ bool exact_match;
+ Status status = iterator->seek_dictionary(&_value, &exact_match);
+ rowid_t seeked_ordinal = iterator->current_ordinal();
+
+ return _bitmap_compare(status, exact_match, ordinal_limit, seeked_ordinal, iterator,
+ bitmap);
+ }
+
+ uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const override {
+ if (column.is_nullable()) {
+ auto* nullable_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnNullable>(column);
+ auto& nested_column = nullable_column_ptr->get_nested_column();
+ auto& null_map = reinterpret_cast<const vectorized::ColumnUInt8&>(
+ nullable_column_ptr->get_null_map_column())
+ .get_data();
+
+ return _base_evaluate<true>(&nested_column, null_map.data(), sel, size);
+ } else {
+ return _base_evaluate<false>(&column, nullptr, sel, size);
+ }
+ }
+
+ void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
+ bool* flags) const override {
+ _evaluate_bit<true>(column, sel, size, flags);
+ }
+
+ void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
+ bool* flags) const override {
+ _evaluate_bit<false>(column, sel, size, flags);
+ }
+
+ void evaluate_vec(const vectorized::IColumn& column, uint16_t size,
+ bool* flags) const override {
+ if (column.is_nullable()) {
+ auto* nullable_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnNullable>(column);
+ auto& nested_column = nullable_column_ptr->get_nested_column();
+ auto& null_map = reinterpret_cast<const vectorized::ColumnUInt8&>(
+ nullable_column_ptr->get_null_map_column())
+ .get_data();
+
+ if (nested_column.is_column_dictionary()) {
+ if constexpr (std::is_same_v<T, StringValue>) {
+ auto* dict_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnDictI32>(
+ nested_column);
+ auto dict_code = _is_range() ? dict_column_ptr->find_code_by_bound(
+ _value, _is_greater(), _is_eq())
+ : dict_column_ptr->find_code(_value);
+ auto* data_array = dict_column_ptr->get_data().data();
+
+ _base_loop_vec<true>(size, flags, null_map.data(), data_array, dict_code);
+ } else {
+ LOG(FATAL) << "column_dictionary must use StringValue predicate.";
+ }
+ } else {
+ auto* data_array = reinterpret_cast<const vectorized::PredicateColumnType<TReal>&>(
+ nested_column)
+ .get_data()
+ .data();
+
+ _base_loop_vec<true>(size, flags, null_map.data(), data_array, _value_real);
+ }
+ } else {
+ if (column.is_column_dictionary()) {
+ if constexpr (std::is_same_v<T, StringValue>) {
+ auto* dict_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnDictI32>(column);
+ auto dict_code = _is_range() ? dict_column_ptr->find_code_by_bound(
+ _value, _is_greater(), _is_eq())
+ : dict_column_ptr->find_code(_value);
+ auto* data_array = dict_column_ptr->get_data().data();
+
+ _base_loop_vec<false>(size, flags, nullptr, data_array, dict_code);
+ } else {
+ LOG(FATAL) << "column_dictionary must use StringValue predicate.";
+ }
+ } else {
+ auto* data_array =
+ vectorized::check_and_get_column<vectorized::PredicateColumnType<TReal>>(
+ column)
+ ->get_data()
+ .data();
+
+ _base_loop_vec<false>(size, flags, nullptr, data_array, _value_real);
+ }
+ }
+
+ if (_opposite) {
+ for (uint16_t i = 0; i < size; i++) {
+ flags[i] = !flags[i];
+ }
+ }
+ }
+
+private:
+ using TReal = std::conditional_t<std::is_same_v<T, uint24_t>, uint32_t, T>;
+
+ template <typename LeftT, typename RightT>
+ bool _operator(const LeftT& lhs, const RightT& rhs) const {
+ if constexpr (PT == PredicateType::EQ) {
+ return lhs == rhs;
+ } else if constexpr (PT == PredicateType::NE) {
+ return lhs != rhs;
+ } else if constexpr (PT == PredicateType::LT) {
+ return lhs < rhs;
+ } else if constexpr (PT == PredicateType::LE) {
+ return lhs <= rhs;
+ } else if constexpr (PT == PredicateType::GT) {
+ return lhs > rhs;
+ } else if constexpr (PT == PredicateType::GE) {
+ return lhs >= rhs;
+ }
+ }
+
+ constexpr bool _is_range() const { return PredicateTypeTraits::is_range(PT); }
+
+ constexpr bool _is_greater() const { return _operator(1, 0); }
+
+ constexpr bool _is_eq() const { return _operator(1, 1); }
+
+ Status _bitmap_compare(Status status, bool exact_match, rowid_t ordinal_limit,
+ rowid_t& seeked_ordinal, BitmapIndexIterator* iterator,
+ roaring::Roaring* bitmap) const {
+ roaring::Roaring roaring;
+
+ if (status.is_not_found()) {
+ if constexpr (PT == PredicateType::EQ || PT == PredicateType::GT ||
+ PT == PredicateType::GE) {
+ *bitmap &= roaring; // set bitmap to empty
+ }
+ return Status::OK();
+ }
+
+ if (!status.ok()) {
+ return status;
+ }
+
+ if constexpr (PT == PredicateType::EQ || PT == PredicateType::NE) {
+ if (exact_match) {
+ RETURN_IF_ERROR(iterator->read_bitmap(seeked_ordinal, &roaring));
+ }
+ } else if constexpr (PredicateTypeTraits::is_range(PT)) {
+ rowid_t from = 0;
+ rowid_t to = ordinal_limit;
+ if constexpr (PT == PredicateType::LT) {
+ to = seeked_ordinal;
+ } else if constexpr (PT == PredicateType::LE) {
+ to = seeked_ordinal + exact_match;
+ } else if constexpr (PT == PredicateType::GT) {
+ from = seeked_ordinal + exact_match;
+ } else if constexpr (PT == PredicateType::GE) {
+ from = seeked_ordinal;
+ }
+
+ RETURN_IF_ERROR(iterator->read_union_bitmap(from, to, &roaring));
+ }
+
+ if constexpr (PT == PredicateType::NE) {
+ *bitmap -= roaring;
+ } else {
+ *bitmap &= roaring;
+ }
+
+ return Status::OK();
+ }
+
+ template <bool is_and>
+ void _evaluate_bit(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
+ bool* flags) const {
+ if (column.is_nullable()) {
+ auto* nullable_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnNullable>(column);
+ auto& nested_column = nullable_column_ptr->get_nested_column();
+ auto& null_map = reinterpret_cast<const vectorized::ColumnUInt8&>(
+ nullable_column_ptr->get_null_map_column())
+ .get_data();
+
+ _base_evaluate_bit<true, is_and>(&nested_column, null_map.data(), sel, size, flags);
+ } else {
+ _base_evaluate_bit<false, is_and>(&column, nullptr, sel, size, flags);
+ }
+ }
+
+ template <bool is_nullable, typename TArray, typename TValue>
+ void _base_loop_vec(uint16_t size, bool* __restrict flags, const uint8_t* __restrict null_map,
+ const TArray* __restrict data_array, const TValue& value) const {
+ for (uint16_t i = 0; i < size; i++) {
+ if constexpr (is_nullable) {
+ flags[i] = !null_map[i] && _operator(data_array[i], value);
+ } else {
+ flags[i] = _operator(data_array[i], value);
+ }
+ }
+ }
+
+ template <bool is_nullable, bool is_and, typename TArray, typename TValue>
+ void _base_loop_bit(const uint16_t* sel, uint16_t size, bool* flags,
+ const uint8_t* __restrict null_map, const TArray* __restrict data_array,
+ const TValue& value) const {
+ for (uint16_t i = 0; i < size; i++) {
+ if (is_and ^ flags[i]) {
+ continue;
+ }
+ if constexpr (is_nullable) {
+ if (_opposite ^ is_and ^
+ (!null_map[sel[i]] && _operator(data_array[sel[i]], value))) {
+ flags[i] = !is_and;
+ }
+ } else {
+ if (_opposite ^ is_and ^ _operator(data_array[sel[i]], value)) {
+ flags[i] = !is_and;
+ }
+ }
+ }
+ }
+
+ template <bool is_nullable, bool is_and>
+ void _base_evaluate_bit(const vectorized::IColumn* column, const uint8_t* null_map,
+ const uint16_t* sel, uint16_t size, bool* flags) const {
+ if (column->is_column_dictionary()) {
+ if constexpr (std::is_same_v<T, StringValue>) {
+ auto* dict_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnDictI32>(column);
+ auto* data_array = dict_column_ptr->get_data().data();
+ auto dict_code = _is_range() ? dict_column_ptr->find_code_by_bound(
+ _value, _operator(1, 0), _operator(1, 1))
+ : dict_column_ptr->find_code(_value);
+ _base_loop_bit<is_nullable, is_and>(sel, size, flags, null_map, data_array,
+ dict_code);
+ } else {
+ LOG(FATAL) << "column_dictionary must use StringValue predicate.";
+ }
+ } else {
+ auto* data_array =
+ vectorized::check_and_get_column<vectorized::PredicateColumnType<TReal>>(column)
+ ->get_data()
+ .data();
+
+ _base_loop_bit<is_nullable, is_and>(sel, size, flags, null_map, data_array,
+ _value_real);
+ }
+ }
+
+ template <bool is_nullable, typename TArray, typename TValue>
+ uint16_t _base_loop(uint16_t* sel, uint16_t size, const uint8_t* __restrict null_map,
+ const TArray* __restrict data_array, const TValue& value) const {
+ uint16_t new_size = 0;
+ for (uint16_t i = 0; i < size; ++i) {
+ uint16_t idx = sel[i];
+ if constexpr (is_nullable) {
+ if (_opposite ^ (!null_map[i] && _operator(data_array[idx], value))) {
+ sel[new_size++] = idx;
+ }
+ } else {
+ if (_opposite ^ _operator(data_array[idx], value)) {
+ sel[new_size++] = idx;
+ }
+ }
+ }
+ return new_size;
+ }
+
+ template <bool is_nullable>
+ uint16_t _base_evaluate(const vectorized::IColumn* column, const uint8_t* null_map,
+ uint16_t* sel, uint16_t size) const {
+ if (column->is_column_dictionary()) {
+ if constexpr (std::is_same_v<T, StringValue>) {
+ auto* dict_column_ptr =
+ vectorized::check_and_get_column<vectorized::ColumnDictI32>(column);
+ auto* data_array = dict_column_ptr->get_data().data();
+ auto dict_code = _is_range() ? dict_column_ptr->find_code_by_bound(
+ _value, _is_greater(), _is_eq())
+ : dict_column_ptr->find_code(_value);
+
+ return _base_loop<is_nullable>(sel, size, null_map, data_array, dict_code);
+ } else {
+ LOG(FATAL) << "column_dictionary must use StringValue predicate.";
+ return 0;
+ }
+ } else {
+ auto* data_array =
+ vectorized::check_and_get_column<vectorized::PredicateColumnType<TReal>>(column)
+ ->get_data()
+ .data();
+
+ return _base_loop<is_nullable>(sel, size, null_map, data_array, _value_real);
+ }
+ }
+
+ T _value;
+ TReal _value_real;
+};
+
+template <class T>
+using EqualPredicate = ComparisonPredicateBase<T, PredicateType::EQ>;
+template <class T>
+using NotEqualPredicate = ComparisonPredicateBase<T, PredicateType::NE>;
+template <class T>
+using LessPredicate = ComparisonPredicateBase<T, PredicateType::LT>;
+template <class T>
+using LessEqualPredicate = ComparisonPredicateBase<T, PredicateType::LE>;
+template <class T>
+using GreaterPredicate = ComparisonPredicateBase<T, PredicateType::GT>;
+template <class T>
+using GreaterEqualPredicate = ComparisonPredicateBase<T, PredicateType::GE>;
} //namespace doris
diff --git a/be/src/olap/in_list_predicate.h b/be/src/olap/in_list_predicate.h
index 04ec211568..2a0fcb0901 100644
--- a/be/src/olap/in_list_predicate.h
+++ b/be/src/olap/in_list_predicate.h
@@ -201,7 +201,8 @@ public:
return Status::OK();
}
- uint16_t evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const override {
+ uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const override {
if (column.is_nullable()) {
auto* nullable_col =
vectorized::check_and_get_column<vectorized::ColumnNullable>(column);
@@ -225,11 +226,11 @@ public:
}
// todo(wb) support evaluate_and,evaluate_or
- void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const override {
LOG(FATAL) << "IColumn not support in_list_predicate.evaluate_and now.";
}
- void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const override {
LOG(FATAL) << "IColumn not support in_list_predicate.evaluate_or now.";
}
diff --git a/be/src/olap/null_predicate.cpp b/be/src/olap/null_predicate.cpp
index 3b94cf6b8f..bdafda6ee7 100644
--- a/be/src/olap/null_predicate.cpp
+++ b/be/src/olap/null_predicate.cpp
@@ -125,7 +125,8 @@ Status NullPredicate::evaluate(const Schema& schema,
return Status::OK();
}
-uint16_t NullPredicate::evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const {
+uint16_t NullPredicate::evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const {
uint16_t new_size = 0;
if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
auto& null_map = nullable->get_null_map_data();
@@ -141,7 +142,8 @@ uint16_t NullPredicate::evaluate(vectorized::IColumn& column, uint16_t* sel, uin
return size;
}
-void NullPredicate::evaluate_or(IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const {
+void NullPredicate::evaluate_or(const IColumn& column, const uint16_t* sel, uint16_t size,
+ bool* flags) const {
if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
auto& null_map = nullable->get_null_map_data();
for (uint16_t i = 0; i < size; ++i) {
@@ -154,7 +156,8 @@ void NullPredicate::evaluate_or(IColumn& column, uint16_t* sel, uint16_t size, b
}
}
-void NullPredicate::evaluate_and(IColumn& column, uint16_t* sel, uint16_t size, bool* flags) const {
+void NullPredicate::evaluate_and(const IColumn& column, const uint16_t* sel, uint16_t size,
+ bool* flags) const {
if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
auto& null_map = nullable->get_null_map_data();
for (uint16_t i = 0; i < size; ++i) {
@@ -167,7 +170,8 @@ void NullPredicate::evaluate_and(IColumn& column, uint16_t* sel, uint16_t size,
}
}
-void NullPredicate::evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const {
+void NullPredicate::evaluate_vec(const vectorized::IColumn& column, uint16_t size,
+ bool* flags) const {
if (auto* nullable = check_and_get_column<ColumnNullable>(column)) {
auto& null_map = nullable->get_null_map_data();
for (uint16_t i = 0; i < size; ++i) {
diff --git a/be/src/olap/null_predicate.h b/be/src/olap/null_predicate.h
index 294ce339b5..c1eee63268 100644
--- a/be/src/olap/null_predicate.h
+++ b/be/src/olap/null_predicate.h
@@ -31,9 +31,9 @@ class NullPredicate : public ColumnPredicate {
public:
NullPredicate(uint32_t column_id, bool is_null, bool opposite = false);
- virtual PredicateType type() const override;
+ PredicateType type() const override;
- virtual void evaluate(VectorizedRowBatch* batch) const override;
+ void evaluate(VectorizedRowBatch* batch) const override;
void evaluate(ColumnBlock* block, uint16_t* sel, uint16_t* size) const override;
@@ -41,18 +41,19 @@ public:
void evaluate_and(ColumnBlock* block, uint16_t* sel, uint16_t size, bool* flags) const override;
- virtual Status evaluate(const Schema& schema, const vector<BitmapIndexIterator*>& iterators,
- uint32_t num_rows, roaring::Roaring* roaring) const override;
+ Status evaluate(const Schema& schema, const vector<BitmapIndexIterator*>& iterators,
+ uint32_t num_rows, roaring::Roaring* roaring) const override;
- uint16_t evaluate(vectorized::IColumn& column, uint16_t* sel, uint16_t size) const override;
+ uint16_t evaluate(const vectorized::IColumn& column, uint16_t* sel,
+ uint16_t size) const override;
- void evaluate_or(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ void evaluate_or(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const override;
- void evaluate_and(vectorized::IColumn& column, uint16_t* sel, uint16_t size,
+ void evaluate_and(const vectorized::IColumn& column, const uint16_t* sel, uint16_t size,
bool* flags) const override;
- void evaluate_vec(vectorized::IColumn& column, uint16_t size, bool* flags) const override;
+ void evaluate_vec(const vectorized::IColumn& column, uint16_t size, bool* flags) const override;
private:
bool _is_null; //true for null, false for not null
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index d00d1856bd..1bacc3cecf 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -24,9 +24,7 @@
#include "gutil/strings/substitute.h"
#include "olap/column_predicate.h"
#include "olap/fs/fs_util.h"
-#include "olap/in_list_predicate.h"
#include "olap/olap_common.h"
-#include "olap/row.h"
#include "olap/row_block2.h"
#include "olap/row_cursor.h"
#include "olap/rowset/segment_v2/column_reader.h"
@@ -34,9 +32,6 @@
#include "olap/short_key_index.h"
#include "util/doris_metrics.h"
#include "util/simd/bits.h"
-#include "vec/columns/column_dictionary.h"
-
-using strings::Substitute;
namespace doris {
namespace segment_v2 {
@@ -933,11 +928,14 @@ uint16_t SegmentIterator::_evaluate_short_circuit_predicate(uint16_t* vec_sel_ro
auto column_id = predicate->column_id();
auto& short_cir_column = _current_return_columns[column_id];
auto* col_ptr = short_cir_column.get();
- // range comparison predicate needs to sort the dict and convert the encoding
- if (predicate->type() == PredicateType::LT || predicate->type() == PredicateType::LE ||
- predicate->type() == PredicateType::GT || predicate->type() == PredicateType::GE) {
+
+ // Dictionary column should do something to initial.
+ if (PredicateTypeTraits::is_range(predicate->type())) {
col_ptr->convert_dict_codes_if_necessary();
+ } else if (PredicateTypeTraits::is_bloom_filter(predicate->type())) {
+ col_ptr->generate_hash_values_for_runtime_filter();
}
+
selected_size = predicate->evaluate(*short_cir_column, vec_sel_rowid_idx, selected_size);
}
_opts.stats->rows_vec_cond_filtered += original_size - selected_size;
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
index 35035d419f..b0c5bee895 100644
--- a/be/src/vec/columns/column.h
+++ b/be/src/vec/columns/column.h
@@ -71,6 +71,9 @@ public:
/// If column is ColumnDictionary, and is a range comparison predicate, convert dict encoding
virtual void convert_dict_codes_if_necessary() {}
+ /// If column is ColumnDictionary, and is a bloom filter predicate, generate_hash_values
+ virtual void generate_hash_values_for_runtime_filter() {}
+
/// Creates empty column with the same type.
virtual MutablePtr clone_empty() const { return clone_resized(0); }
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
index d38bc4e049..bb8ec43762 100644
--- a/be/src/vec/columns/column_dictionary.h
+++ b/be/src/vec/columns/column_dictionary.h
@@ -241,7 +241,7 @@ public:
return _dict.find_code_by_bound(value, greater, eq);
}
- void generate_hash_values_for_runtime_filter() {
+ void generate_hash_values_for_runtime_filter() override {
_dict.generate_hash_values_for_runtime_filter(_type);
}
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
index 31e1f2b80e..1361711d8c 100644
--- a/be/src/vec/columns/column_nullable.h
+++ b/be/src/vec/columns/column_nullable.h
@@ -282,6 +282,10 @@ public:
get_nested_column().convert_dict_codes_if_necessary();
}
+ void generate_hash_values_for_runtime_filter() override {
+ get_nested_column().generate_hash_values_for_runtime_filter();
+ }
+
private:
WrappedPtr nested_column;
WrappedPtr null_map;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org