You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/14 09:29:39 UTC
[doris] branch revert-25427-pick_prune_predicates created (now bb764f8a6f)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a change to branch revert-25427-pick_prune_predicates
in repository https://gitbox.apache.org/repos/asf/doris.git
at bb764f8a6f Revert "[improvement](scanner) Remove the predicate that is always true for the segment (#25366) (#25427)"
This branch includes the following new commits:
new bb764f8a6f Revert "[improvement](scanner) Remove the predicate that is always true for the segment (#25366) (#25427)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/01: Revert "[improvement](scanner) Remove the predicate that is always true for the segment (#25366) (#25427)"
Posted by kx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch revert-25427-pick_prune_predicates
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bb764f8a6f0c86a7db02bacabc65a1141d8af2cc
Author: Kang <kx...@gmail.com>
AuthorDate: Sat Oct 14 17:29:29 2023 +0800
Revert "[improvement](scanner) Remove the predicate that is always true for the segment (#25366) (#25427)"
This reverts commit 52ee055f3b5026053f2d5ecac66a5b10384b143f.
---
be/src/common/config.cpp | 2 -
be/src/common/config.h | 3 -
be/src/olap/column_predicate.h | 4 --
be/src/olap/comparison_predicate.h | 25 --------
be/src/olap/rowset/segment_v2/column_reader.cpp | 26 ---------
be/src/olap/rowset/segment_v2/column_reader.h | 3 -
be/src/olap/rowset/segment_v2/segment.cpp | 20 +------
.../query_p0/test_select_with_predicate_prune.out | 25 --------
.../test_select_with_predicate_prune.groovy | 67 ----------------------
9 files changed, 1 insertion(+), 174 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index f2c726a33b..63ba1a1288 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1084,8 +1084,6 @@ DEFINE_mInt32(tablet_schema_cache_recycle_interval, "86400");
DEFINE_Bool(exit_on_exception, "false")
-DEFINE_Bool(ignore_always_true_predicate_for_segment, "true");
-
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 06f9a18fcd..7c4ab39a40 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1141,9 +1141,6 @@ DECLARE_mInt32(tablet_schema_cache_recycle_interval);
// Use `LOG(FATAL)` to replace `throw` when true
DECLARE_mBool(exit_on_exception);
-// Remove predicate that is always true for a segment.
-DECLARE_Bool(ignore_always_true_predicate_for_segment);
-
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 05e84999a8..b98156f5fb 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -173,10 +173,6 @@ public:
return true;
}
- virtual bool is_always_true(const std::pair<WrapperField*, WrapperField*>& statistic) const {
- return false;
- }
-
virtual bool evaluate_del(const std::pair<WrapperField*, WrapperField*>& statistic) const {
return false;
}
diff --git a/be/src/olap/comparison_predicate.h b/be/src/olap/comparison_predicate.h
index fddc554f51..04dfd5dc5c 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -158,8 +158,6 @@ public:
return _operator(*reinterpret_cast<const T*>(statistic.ELE->cell_ptr()), _value); \
}
- using WarpperFieldType = std::conditional_t<Type == TYPE_DATE, uint24_t, T>;
-
bool evaluate_and(const std::pair<WrapperField*, WrapperField*>& statistic) const override {
if (statistic.first->is_null()) {
return true;
@@ -204,29 +202,6 @@ public:
}
}
- bool is_always_true(const std::pair<WrapperField*, WrapperField*>& statistic) const override {
- if (statistic.first->is_null() || statistic.second->is_null()) {
- return false;
- }
-
- T tmp_min_value {};
- T tmp_max_value {};
- memcpy((char*)(&tmp_min_value), statistic.first->cell_ptr(), sizeof(WarpperFieldType));
- memcpy((char*)(&tmp_max_value), statistic.second->cell_ptr(), sizeof(WarpperFieldType));
-
- if constexpr (PT == PredicateType::LT) {
- return _value > tmp_max_value;
- } else if constexpr (PT == PredicateType::LE) {
- return _value >= tmp_max_value;
- } else if constexpr (PT == PredicateType::GT) {
- return _value < tmp_min_value;
- } else if constexpr (PT == PredicateType::GE) {
- return _value <= tmp_min_value;
- }
-
- return false;
- }
-
bool evaluate_del(const std::pair<WrapperField*, WrapperField*>& statistic) const override {
if (statistic.first->is_null() || statistic.second->is_null()) {
return false;
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index d9a074e290..b1b817f545 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -30,7 +30,6 @@
#include "io/fs/file_reader.h"
#include "olap/block_column_predicate.h"
#include "olap/column_predicate.h"
-#include "olap/comparison_predicate.h"
#include "olap/decimal12.h"
#include "olap/inverted_index_parser.h"
#include "olap/iterators.h"
@@ -340,31 +339,6 @@ bool ColumnReader::match_condition(const AndBlockColumnPredicate* col_predicates
col_predicates);
}
-bool ColumnReader::prune_predicates_by_zone_map(std::vector<ColumnPredicate*>& predicates,
- const int column_id) const {
- if (_zone_map_index == nullptr) {
- return false;
- }
-
- FieldType type = _type_info->type();
- std::unique_ptr<WrapperField> min_value(WrapperField::create_by_type(type, _meta_length));
- std::unique_ptr<WrapperField> max_value(WrapperField::create_by_type(type, _meta_length));
- _parse_zone_map(*_segment_zone_map, min_value.get(), max_value.get());
-
- auto pruned = false;
- for (auto it = predicates.begin(); it != predicates.end();) {
- auto predicate = *it;
- if (predicate->column_id() == column_id &&
- predicate->is_always_true({min_value.get(), max_value.get()})) {
- pruned = true;
- it = predicates.erase(it);
- } else {
- ++it;
- }
- }
- return pruned;
-}
-
void ColumnReader::_parse_zone_map(const ZoneMapPB& zone_map, WrapperField* min_value_container,
WrapperField* max_value_container) const {
// min value and max value are valid if has_not_null is true
diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h
index 7964555ade..174aabdefa 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.h
+++ b/be/src/olap/rowset/segment_v2/column_reader.h
@@ -162,9 +162,6 @@ public:
bool is_empty() const { return _num_rows == 0; }
- bool prune_predicates_by_zone_map(std::vector<ColumnPredicate*>& predicates,
- const int column_id) const;
-
CompressionTypePB get_compression() const { return _meta_compression; }
uint64_t num_rows() const { return _num_rows; }
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp b/be/src/olap/rowset/segment_v2/segment.cpp
index ddce80bcc3..153ed92517 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -147,6 +147,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
return Status::OK();
}
}
+
if (read_options.use_topn_opt) {
auto query_ctx = read_options.runtime_state->get_query_ctx();
auto runtime_predicate = query_ctx->get_runtime_predicate().get_predictate();
@@ -174,25 +175,6 @@ Status Segment::new_iterator(SchemaSPtr schema, const StorageReadOptions& read_o
iter->reset(new SegmentIterator(this->shared_from_this(), schema));
}
- if (config::ignore_always_true_predicate_for_segment &&
- read_options.io_ctx.reader_type == ReaderType::READER_QUERY &&
- !read_options.column_predicates.empty()) {
- auto pruned_predicates = read_options.column_predicates;
- auto pruned = false;
- for (auto& it : _column_readers) {
- if (it.second->prune_predicates_by_zone_map(pruned_predicates, it.first)) {
- pruned = true;
- }
- }
-
- if (pruned) {
- auto options_with_pruned_predicates = read_options;
- options_with_pruned_predicates.column_predicates = pruned_predicates;
- LOG(INFO) << "column_predicates pruned from " << read_options.column_predicates.size()
- << " to " << pruned_predicates.size();
- return iter->get()->init(options_with_pruned_predicates);
- }
- }
return iter->get()->init(read_options);
}
diff --git a/regression-test/data/query_p0/test_select_with_predicate_prune.out b/regression-test/data/query_p0/test_select_with_predicate_prune.out
deleted file mode 100644
index 2e1fad8749..0000000000
--- a/regression-test/data/query_p0/test_select_with_predicate_prune.out
+++ /dev/null
@@ -1,25 +0,0 @@
--- This file is automatically generated. You should know what you did if you want to edit this
--- !select1 --
-1 jerry 2020-10-01
-2 tom 2020-10-02
-3 jack 2020-10-01
-4 tony 2020-10-02
-
--- !select2 --
-1 jerry 2020-10-01
-3 jack 2020-10-01
-
--- !select3 --
-
--- !select4 --
-1 jerry 2020-10-01
-2 tom 2020-10-02
-3 jack 2020-10-01
-4 tony 2020-10-02
-
--- !select5 --
-2 tom 2020-10-02
-4 tony 2020-10-02
-
--- !select6 --
-
diff --git a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy b/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
deleted file mode 100644
index 768e04b4c3..0000000000
--- a/regression-test/suites/query_p0/test_select_with_predicate_prune.groovy
+++ /dev/null
@@ -1,67 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-suite("test_select_with_predicate_prune") {
- sql """
- drop table if exists `test_select_with_predicate_prune`;
- """
- sql """
- CREATE TABLE IF NOT EXISTS `test_select_with_predicate_prune` (
- id int,
- name string,
- birthday date not null
- )
- duplicate key(`id`)
- AUTO PARTITION BY LIST (`birthday`)()
- DISTRIBUTED BY HASH(`id`) buckets 1
- PROPERTIES
- (
- "replication_allocation" = "tag.location.default: 1"
- );
- """
-
- sql """
- insert into test_select_with_predicate_prune values (1, 'jerry', '2020-10-01'), (2, 'tom', '2020-10-02');
- """
- sql """
- insert into test_select_with_predicate_prune values (3, 'jack', '2020-10-01'), (4, 'tony', '2020-10-02');
- """
-
- qt_select1 """
- select * from test_select_with_predicate_prune where birthday < '2020-10-03' order by id;
- """
-
- qt_select2 """
- select * from test_select_with_predicate_prune where birthday < '2020-10-02' order by id;
- """
-
- qt_select3 """
- select * from test_select_with_predicate_prune where birthday < '2020-10-01' order by id;
- """
-
-
- qt_select4 """
- select * from test_select_with_predicate_prune where birthday > '2020-09-30' order by id;
- """
-
- qt_select5 """
- select * from test_select_with_predicate_prune where birthday > '2020-10-01' order by id;
- """
-
- qt_select6 """
- select * from test_select_with_predicate_prune where birthday > '2020-10-02' order by id;
- """
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org