You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ji...@apache.org on 2024/07/10 11:47:12 UTC
(doris) branch master updated: [Refact](inverted index) refact no need read data opt logic (#36969)
This is an automated email from the ASF dual-hosted git repository.
jianliangqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d1185a94f35 [Refact](inverted index) refact no need read data opt logic (#36969)
d1185a94f35 is described below
commit d1185a94f35dfd5a1be10eba735be6d053f8af89
Author: airborne12 <ai...@gmail.com>
AuthorDate: Wed Jul 10 19:47:05 2024 +0800
[Refact](inverted index) refact no need read data opt logic (#36969)
Refact no need read data for column where all predicates have passed the
inverted index, make it clear
---
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 119 +++++++++------------
be/src/olap/rowset/segment_v2/segment_iterator.h | 7 +-
2 files changed, 57 insertions(+), 69 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 82a3c8ba509..09366120a9f 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -361,42 +361,36 @@ Status SegmentIterator::_init_impl(const StorageReadOptions& opts) {
}
}
- // find columns that definitely require reading data, such as functions that are not pushed down.
- {
- std::set<std::string> push_down_preds;
- for (auto* pred : _col_predicates) {
- if (!_check_apply_by_inverted_index(pred)) {
- //column predicate, like column predicate etc. always need read data
- auto cid = pred->column_id();
- _need_read_data_indices[cid] = true;
- continue;
- }
- push_down_preds.insert(_gen_predicate_result_sign(pred));
- }
- for (auto* pred : _col_preds_except_leafnode_of_andnode) {
- if (!_check_apply_by_inverted_index(pred)) {
- //column predicate, like column predicate etc. always need read data
- auto cid = pred->column_id();
- _need_read_data_indices[cid] = true;
- continue;
- }
- push_down_preds.insert(_gen_predicate_result_sign(pred));
- }
- for (auto& preds_in_remaining_vconjuct : _column_pred_in_remaining_vconjunct) {
- const auto& column_name = preds_in_remaining_vconjuct.first;
- for (auto& pred_info : preds_in_remaining_vconjuct.second) {
- auto column_sign = _gen_predicate_result_sign(&pred_info);
- if (!push_down_preds.contains(column_sign)) {
- auto cid = _opts.tablet_schema->field_index(column_name);
- _need_read_data_indices[cid] = true;
- }
- }
- }
- }
+ _initialize_predicate_results();
return Status::OK();
}
+void SegmentIterator::_initialize_predicate_results() {
+ // Initialize from _col_predicates
+ for (auto* pred : _col_predicates) {
+ int cid = pred->column_id();
+ std::string pred_sign = _gen_predicate_result_sign(pred);
+ _column_predicate_inverted_index_status[cid][pred_sign] = false;
+ }
+
+ // Initialize from _col_preds_except_leafnode_of_andnode
+ for (auto* pred : _col_preds_except_leafnode_of_andnode) {
+ int cid = pred->column_id();
+ std::string pred_sign = _gen_predicate_result_sign(pred);
+ _column_predicate_inverted_index_status[cid][pred_sign] = false;
+ }
+
+ // Initialize from _column_pred_in_remaining_vconjunct
+ for (auto& preds_in_remaining_vconjuct : _column_pred_in_remaining_vconjunct) {
+ for (auto& pred_info : preds_in_remaining_vconjuct.second) {
+ int cid = _schema->column_id(pred_info.column_id);
+ std::string pred_sign = _gen_predicate_result_sign(&pred_info);
+ _column_predicate_inverted_index_status[cid][pred_sign] = false;
+ }
+ }
+}
+
Status SegmentIterator::init_iterators() {
RETURN_IF_ERROR(_init_return_column_iterators());
RETURN_IF_ERROR(_init_bitmap_index_iterators());
@@ -579,7 +573,13 @@ Status SegmentIterator::_get_row_ranges_by_column_conditions() {
RETURN_IF_ERROR(_apply_bitmap_index());
RETURN_IF_ERROR(_apply_inverted_index());
+ for (auto cid : _schema->column_ids()) {
+ bool result_true = _check_all_predicates_passed_inverted_index_for_column(cid);
+ if (result_true) {
+ _need_read_data_indices[cid] = false;
+ }
+ }
if (!_row_bitmap.isEmpty() &&
(!_opts.topn_filter_source_node_ids.empty() || !_opts.col_id_to_predicates.empty() ||
_opts.delete_condition_predicates->num_of_column_predicate() > 0)) {
@@ -734,13 +734,6 @@ Status SegmentIterator::_apply_bitmap_index() {
} else {
RETURN_IF_ERROR(pred->evaluate(_bitmap_index_iterators[cid].get(), _segment->num_rows(),
&_row_bitmap));
-
- auto column_name = _schema->column(pred->column_id())->name();
- if (_check_column_pred_all_push_down(column_name) &&
- !pred->predicate_params()->marked_by_runtime_filter) {
- _need_read_data_indices[cid] = false;
- }
-
if (_row_bitmap.isEmpty()) {
break; // all rows have been pruned, no need to process further predicates
}
@@ -966,7 +959,6 @@ Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
pred_type == PredicateType::IN_LIST ||
pred_type == PredicateType::NOT_IN_LIST;
if (!is_support) {
- _need_read_data_indices[column_id] = true;
continue;
}
@@ -976,7 +968,6 @@ Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
if (can_apply_by_inverted_index) {
res = _apply_inverted_index_except_leafnode_of_andnode(pred, &bitmap);
} else {
- _need_read_data_indices[column_id] = true;
continue;
}
@@ -986,7 +977,6 @@ Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
// downgrade without index query
_not_apply_index_pred.insert(column_id);
- _need_read_data_indices[column_id] = true;
continue;
}
LOG(WARNING) << "failed to evaluate index"
@@ -997,11 +987,8 @@ Status SegmentIterator::_apply_index_except_leafnode_of_andnode() {
std::string pred_result_sign = _gen_predicate_result_sign(pred);
_rowid_result_for_index.emplace(pred_result_sign, std::make_pair(true, std::move(bitmap)));
-
if (!pred->predicate_params()->marked_by_runtime_filter) {
- if (!_need_read_data_indices.contains(column_id)) {
- _need_read_data_indices[column_id] = false;
- }
+ _column_predicate_inverted_index_status[column_id][pred_result_sign] = true;
}
}
@@ -1090,11 +1077,6 @@ inline bool SegmentIterator::_inverted_index_not_support_pred_type(const Predica
std::all_of(predicate_set.begin(), predicate_set.end(), \
[](const ColumnPredicate* p) { return PredicateTypeTraits::is_range(p->type()); })
-#define all_predicates_are_marked_by_runtime_filter(predicate_set) \
- std::all_of(predicate_set.begin(), predicate_set.end(), [](const ColumnPredicate* p) { \
- return const_cast<ColumnPredicate*>(p)->predicate_params()->marked_by_runtime_filter; \
- })
-
Status SegmentIterator::_apply_inverted_index_on_column_predicate(
ColumnPredicate* pred, std::vector<ColumnPredicate*>& remaining_predicates,
bool* continue_apply) {
@@ -1109,7 +1091,6 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate(
if (!res.ok()) {
if (_downgrade_without_index(res, need_remaining_after_evaluate)) {
remaining_predicates.emplace_back(pred);
- _need_read_data_indices[pred->column_id()] = true;
return Status::OK();
}
LOG(WARNING) << "failed to evaluate index"
@@ -1131,18 +1112,11 @@ Status SegmentIterator::_apply_inverted_index_on_column_predicate(
if (need_remaining_after_evaluate) {
remaining_predicates.emplace_back(pred);
- _need_read_data_indices[pred->column_id()] = true;
return Status::OK();
}
-
- auto column_name = _schema->column(pred->column_id())->name();
- if (_check_column_pred_all_push_down(column_name, false,
- pred->type() == PredicateType::MATCH) &&
- !pred->predicate_params()->marked_by_runtime_filter) {
- // if column's need_read_data already set true, we can not set it to false now.
- if (_need_read_data_indices.find(pred->column_id()) == _need_read_data_indices.end()) {
- _need_read_data_indices[pred->column_id()] = false;
- }
+ if (!pred->predicate_params()->marked_by_runtime_filter) {
+ std::string pred_result_sign = _gen_predicate_result_sign(pred);
+ _column_predicate_inverted_index_status[pred->column_id()][pred_result_sign] = true;
}
}
return Status::OK();
@@ -1172,12 +1146,6 @@ Status SegmentIterator::_apply_inverted_index_on_block_column_predicate(
num_rows(), &_row_bitmap);
if (res.ok()) {
- if (_check_column_pred_all_push_down(column_name) &&
- !all_predicates_are_marked_by_runtime_filter(predicate_set)) {
- if (_need_read_data_indices.find(column_id) == _need_read_data_indices.end()) {
- _need_read_data_indices[column_id] = false;
- }
- }
no_need_to_pass_column_predicate_set.insert(predicate_set.begin(), predicate_set.end());
if (_row_bitmap.isEmpty()) {
// all rows have been pruned, no need to process further predicates
@@ -1377,6 +1345,21 @@ Status SegmentIterator::_apply_inverted_index() {
return Status::OK();
}
+bool SegmentIterator::_check_all_predicates_passed_inverted_index_for_column(ColumnId cid) {
+ auto it = _column_predicate_inverted_index_status.find(cid);
+ if (it != _column_predicate_inverted_index_status.end()) {
+ const auto& pred_map = it->second;
+
+ bool all_true = std::all_of(pred_map.begin(), pred_map.end(),
+ [](const auto& pred_entry) { return pred_entry.second; });
+
+ if (all_true) {
+ return true;
+ }
+ }
+ return false;
+}
+
Status SegmentIterator::_init_return_column_iterators() {
if (_cur_rowid >= num_rows()) {
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h
index 5b89d1932fa..ae865ddc456 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.h
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.h
@@ -164,7 +164,6 @@ private:
[[nodiscard]] Status _init_return_column_iterators();
[[nodiscard]] Status _init_bitmap_index_iterators();
[[nodiscard]] Status _init_inverted_index_iterators();
-
// calculate row ranges that fall into requested key ranges using short key index
[[nodiscard]] Status _get_row_ranges_by_keys();
[[nodiscard]] Status _prepare_seek(const StorageReadOptions::KeyRange& key_range);
@@ -389,6 +388,9 @@ private:
bool _can_opt_topn_reads() const;
+ void _initialize_predicate_results();
+ bool _check_all_predicates_passed_inverted_index_for_column(ColumnId cid);
+
class BitmapRangeIterator;
class BackwardBitmapRangeIterator;
@@ -498,6 +500,9 @@ private:
std::unique_ptr<HierarchicalDataReader> _path_reader;
std::vector<uint8_t> _ret_flags;
+
+ std::unordered_map<int, std::unordered_map<std::string, bool>>
+ _column_predicate_inverted_index_status;
};
} // namespace segment_v2
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org