You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2021/01/24 02:11:32 UTC
[incubator-doris] branch master updated: [Storage] Optimize Zone
map create policy (#5260)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 139709d [Storage] Optimize Zone map create policy (#5260)
139709d is described below
commit 139709d060c83355c99e7ccd59e634ede2031cd7
Author: Zhengguo Yang <ya...@gmail.com>
AuthorDate: Sun Jan 24 10:11:21 2021 +0800
[Storage] Optimize Zone map create policy (#5260)
If there are too large fields in the table, there may be only one row in each page,
and this row also has a zone map index
This causes the stored data to expand three times the original data,
It also takes up more memory when reading those segments
Therefore, we need to Disable the creation of zonemap indexes for segments with too few rows
---
be/src/common/config.h | 3 ++
be/src/olap/rowset/segment_v2/column_reader.cpp | 32 ++++++++++++----------
be/src/olap/rowset/segment_v2/column_writer.cpp | 4 ++-
be/src/olap/rowset/segment_v2/zone_map_index.cpp | 8 ++++++
be/src/olap/rowset/segment_v2/zone_map_index.h | 16 +++++++++--
docs/en/administrator-guide/config/be_config.md | 6 ++++
docs/zh-CN/administrator-guide/config/be_config.md | 7 +++++
gensrc/proto/segment_v2.proto | 2 ++
8 files changed, 61 insertions(+), 17 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 2d5526e..4d0132d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -562,6 +562,9 @@ CONF_mInt32(max_tablet_version_num, "500");
// the thrift_server_type_of_fe should be set THREADED to make be thrift client to fe constructed with TFramedTransport
CONF_String(thrift_server_type_of_fe, "THREAD_POOL");
+// disable zone map index when page row is too few
+CONF_mInt32(zone_map_row_num_threshold, "20");
+
} // namespace config
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 6452d6d..095f6a0 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -187,7 +187,7 @@ bool ColumnReader::_zone_map_match_condition(const ZoneMapPB& zone_map,
return false; // no data in this zone
}
- if (cond == nullptr) {
+ if (cond == nullptr || zone_map.pass_all()) {
return true;
}
@@ -204,20 +204,24 @@ Status ColumnReader::_get_filtered_pages(
std::unique_ptr<WrapperField> min_value(WrapperField::create_by_type(type, _meta.length()));
std::unique_ptr<WrapperField> max_value(WrapperField::create_by_type(type, _meta.length()));
for (int32_t i = 0; i < page_size; ++i) {
- _parse_zone_map(zone_maps[i], min_value.get(), max_value.get());
- if (_zone_map_match_condition(zone_maps[i], min_value.get(), max_value.get(),
- cond_column)) {
- bool should_read = true;
- if (delete_condition != nullptr) {
- int state = delete_condition->del_eval({min_value.get(), max_value.get()});
- if (state == DEL_SATISFIED) {
- should_read = false;
- } else if (state == DEL_PARTIAL_SATISFIED) {
- delete_partial_filtered_pages->insert(i);
+ if (zone_maps[i].pass_all()) {
+ page_indexes->push_back(i);
+ } else {
+ _parse_zone_map(zone_maps[i], min_value.get(), max_value.get());
+ if (_zone_map_match_condition(zone_maps[i], min_value.get(), max_value.get(),
+ cond_column)) {
+ bool should_read = true;
+ if (delete_condition != nullptr) {
+ int state = delete_condition->del_eval({min_value.get(), max_value.get()});
+ if (state == DEL_SATISFIED) {
+ should_read = false;
+ } else if (state == DEL_PARTIAL_SATISFIED) {
+ delete_partial_filtered_pages->insert(i);
+ }
+ }
+ if (should_read) {
+ page_indexes->push_back(i);
}
- }
- if (should_read) {
- page_indexes->push_back(i);
}
}
}
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 5f716c5..3e9654d 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -359,8 +359,10 @@ Status ScalarColumnWriter::finish_current_page() {
if (_next_rowid == _first_rowid) {
return Status::OK();
}
-
if (_opts.need_zone_map) {
+ if (_next_rowid - _first_rowid < config::zone_map_row_num_threshold) {
+ _zone_map_index_builder->reset_page_zone_map();
+ }
RETURN_IF_ERROR(_zone_map_index_builder->flush());
}
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.cpp b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
index 4c6519c..d9775e5 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.cpp
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.cpp
@@ -57,6 +57,14 @@ void ZoneMapIndexWriter::add_values(const void* values, size_t count) {
}
}
+void ZoneMapIndexWriter::reset_page_zone_map() {
+ _page_zone_map.pass_all = true;
+}
+
+void ZoneMapIndexWriter::reset_segment_zone_map() {
+ _segment_zone_map.pass_all = true;
+}
+
Status ZoneMapIndexWriter::flush() {
// Update segment zone map.
if (_field->compare(_segment_zone_map.min_value, _page_zone_map.min_value) > 0) {
diff --git a/be/src/olap/rowset/segment_v2/zone_map_index.h b/be/src/olap/rowset/segment_v2/zone_map_index.h
index 8c6b2df..a3f4bbe 100644
--- a/be/src/olap/rowset/segment_v2/zone_map_index.h
+++ b/be/src/olap/rowset/segment_v2/zone_map_index.h
@@ -52,11 +52,19 @@ struct ZoneMap {
// has_not_null means whether zone has none-null value
bool has_not_null = false;
+ bool pass_all = false;
+
void to_proto(ZoneMapPB* dst, Field* field) {
- dst->set_min(field->to_string(min_value));
- dst->set_max(field->to_string(max_value));
+ if (pass_all) {
+ dst->set_min("");
+ dst->set_max("");
+ } else {
+ dst->set_min(field->to_string(min_value));
+ dst->set_max(field->to_string(max_value));
+ }
dst->set_has_null(has_null);
dst->set_has_not_null(has_not_null);
+ dst->set_pass_all(pass_all);
}
};
@@ -79,6 +87,9 @@ public:
uint64_t size() { return _estimated_size; }
+ void reset_page_zone_map();
+ void reset_segment_zone_map();
+
private:
void _reset_zone_map(ZoneMap* zone_map) {
// we should allocate max varchar length and set to max for min value
@@ -86,6 +97,7 @@ private:
_field->set_to_min(zone_map->max_value);
zone_map->has_null = false;
zone_map->has_not_null = false;
+ zone_map->pass_all = false;
}
Field* _field;
diff --git a/docs/en/administrator-guide/config/be_config.md b/docs/en/administrator-guide/config/be_config.md
index 3437639..e3deada 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -864,3 +864,9 @@ If the parameter is `THREAD_POOL`, the model is a blocking I/O model.
* Default: 8040
### `write_buffer_size`
+
+### `zone_map_row_num_threshold`
+
+* Type: int32
+* Description: If the number of rows in a page is less than this value, no zonemap will be created to reduce data expansion
+* Default: 20
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md b/docs/zh-CN/administrator-guide/config/be_config.md
index 7cc4c91..289e208 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -864,3 +864,10 @@ Stream Load 一般适用于导入几个GB以内的数据，不适合导入过大
* 默认值：8040
### `write_buffer_size`
+
+### `zone_map_row_num_threshold`
+
+* 类型： int32
+* 描述: 如果一个page中的行数小于这个值就不会创建zonemap，用来减少数据膨胀
+* 默认值: 20
+
diff --git a/gensrc/proto/segment_v2.proto b/gensrc/proto/segment_v2.proto
index 90aefec..cc4f9a4 100644
--- a/gensrc/proto/segment_v2.proto
+++ b/gensrc/proto/segment_v2.proto
@@ -131,6 +131,8 @@ message ZoneMapPB {
optional bool has_null = 3;
// whether the zone has not-null value
optional bool has_not_null = 4;
+ // whether this zone is including all values;
+ optional bool pass_all = 5 [default = false];
}
message ColumnMetaPB {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org