You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/12/20 09:11:54 UTC
[doris] branch master updated: [chore](BE) remove all useless segment group related code #15193
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 821c12a456 [chore](BE) remove all useless segment group related code #15193
821c12a456 is described below
commit 821c12a4562dde2585baa7be5d8a10bc86f3095c
Author: AlexYue <yj...@gmail.com>
AuthorDate: Tue Dec 20 17:11:47 2022 +0800
[chore](BE) remove all useless segment group related code #15193
The segment group is useless in current codebase, remove all the related code inside Doris. As for the related protobuf code, use reserved flag to prevent any future user from using that field.
---
be/src/olap/push_handler.cpp | 4 +-
be/src/olap/push_handler.h | 2 +-
be/src/olap/rowset/segment_v2/segment.h | 1 -
be/test/olap/delete_handler_test.cpp | 30 +-----------
be/test/olap/ordered_data_compaction_test.cpp | 30 +-----------
be/test/olap/rowid_conversion_test.cpp | 30 +-----------
be/test/olap/tablet_test.cpp | 30 +-----------
be/test/olap/test_data/header_without_inc_rs.txt | 45 +-----------------
be/test/olap/timestamped_version_tracker_test.cpp | 56 +----------------------
be/test/vec/olap/vertical_compaction_test.cpp | 30 +-----------
docs/images/cpu-flame-demo.svg | 12 -----
gensrc/proto/olap_file.proto | 19 +-------
12 files changed, 13 insertions(+), 276 deletions(-)
diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp
index b99e1f4952..9b2c7149ed 100644
--- a/be/src/olap/push_handler.cpp
+++ b/be/src/olap/push_handler.cpp
@@ -240,7 +240,7 @@ Status PushHandler::_convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur
uint8_t* tuple_buf = reader->mem_pool()->allocate(schema->schema_size());
ContiguousRow row(schema.get(), tuple_buf);
- // 4. Read data from broker and write into SegmentGroup of cur_tablet
+ // 4. Read data from broker and write into cur_tablet
// Convert from raw to delta
VLOG_NOTICE << "start to convert etl file to delta.";
while (!reader->eof()) {
@@ -368,7 +368,7 @@ Status PushHandler::_convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_ro
break;
}
- // 5. Read data from raw file and write into SegmentGroup of cur_tablet
+ // 5. Read data from raw file and write into cur_tablet
if (_request.__isset.http_file_path) {
// Convert from raw to delta
VLOG_NOTICE << "start to convert row file to delta.";
diff --git a/be/src/olap/push_handler.h b/be/src/olap/push_handler.h
index 02384e9f2d..1d8536f85e 100644
--- a/be/src/olap/push_handler.h
+++ b/be/src/olap/push_handler.h
@@ -58,7 +58,7 @@ private:
Status _convert_v2(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset,
TabletSchemaSPtr tablet_schema);
// Convert local data file to internal formatted delta,
- // return new delta's SegmentGroup
+ // return new delta's rowset
Status _convert(TabletSharedPtr cur_tablet, RowsetSharedPtr* cur_rowset,
TabletSchemaSPtr tablet_schema);
diff --git a/be/src/olap/rowset/segment_v2/segment.h b/be/src/olap/rowset/segment_v2/segment.h
index b0dea5aeb8..b5e54dfa4f 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -37,7 +37,6 @@
namespace doris {
-class SegmentGroup;
class TabletSchema;
class ShortKeyIndexDecoder;
class Schema;
diff --git a/be/test/olap/delete_handler_test.cpp b/be/test/olap/delete_handler_test.cpp
index d47efff4e3..eab780df86 100644
--- a/be/test/olap/delete_handler_test.cpp
+++ b/be/test/olap/delete_handler_test.cpp
@@ -832,35 +832,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 2,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }]
- }
+ "creation_time": 1553765670
})";
}
diff --git a/be/test/olap/ordered_data_compaction_test.cpp b/be/test/olap/ordered_data_compaction_test.cpp
index ad6813c657..1cd6c17212 100644
--- a/be/test/olap/ordered_data_compaction_test.cpp
+++ b/be/test/olap/ordered_data_compaction_test.cpp
@@ -258,35 +258,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 2,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }]
- }
+ "creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);
diff --git a/be/test/olap/rowid_conversion_test.cpp b/be/test/olap/rowid_conversion_test.cpp
index 66d61044aa..3d0852fd22 100644
--- a/be/test/olap/rowid_conversion_test.cpp
+++ b/be/test/olap/rowid_conversion_test.cpp
@@ -218,35 +218,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 2,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }]
- }
+ "creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);
diff --git a/be/test/olap/tablet_test.cpp b/be/test/olap/tablet_test.cpp
index 152aa00c3e..b1f949489f 100644
--- a/be/test/olap/tablet_test.cpp
+++ b/be/test/olap/tablet_test.cpp
@@ -63,35 +63,7 @@ public:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 2,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }]
- }
+ "creation_time": 1553765670
})";
doris::EngineOptions options;
diff --git a/be/test/olap/test_data/header_without_inc_rs.txt b/be/test/olap/test_data/header_without_inc_rs.txt
index 44c191d3fc..a2beb4bb50 100644
--- a/be/test/olap/test_data/header_without_inc_rs.txt
+++ b/be/test/olap/test_data/header_without_inc_rs.txt
@@ -73,19 +73,7 @@
"index_disk_size": 0,
"empty": true,
"creation_time": 1553765664,
- "num_segments": 0,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 0,
- "index_size": 0,
- "data_size": 0,
- "num_rows": 0,
- "empty": true
- }
- ]
- }
+ "num_segments": 0
},
{
"rowset_id": 540081,
@@ -106,36 +94,7 @@
"lo": -6717994719194512122
},
"creation_time": 1553765670,
- "num_segments": 1,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 1,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }
- ]
- }
+ "num_segments": 1
}
],
"in_restore_mode": false,
diff --git a/be/test/olap/timestamped_version_tracker_test.cpp b/be/test/olap/timestamped_version_tracker_test.cpp
index 4ca1013c40..578349e794 100644
--- a/be/test/olap/timestamped_version_tracker_test.cpp
+++ b/be/test/olap/timestamped_version_tracker_test.cpp
@@ -51,61 +51,7 @@ public:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 1,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- },
- {
- "segment_group_id": 1,
- "num_segments": 1,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }
- ]
- }
+ "creation_time": 1553765670
})";
}
void TearDown() override {}
diff --git a/be/test/vec/olap/vertical_compaction_test.cpp b/be/test/vec/olap/vertical_compaction_test.cpp
index 5c952aa786..09d7d4656d 100644
--- a/be/test/vec/olap/vertical_compaction_test.cpp
+++ b/be/test/vec/olap/vertical_compaction_test.cpp
@@ -252,35 +252,7 @@ protected:
"hi": -5350970832824939812,
"lo": -6717994719194512122
},
- "creation_time": 1553765670,
- "alpha_rowset_extra_meta_pb": {
- "segment_groups": [
- {
- "segment_group_id": 0,
- "num_segments": 2,
- "index_size": 132,
- "data_size": 576,
- "num_rows": 5,
- "zone_maps": [
- {
- "min": "MQ==",
- "max": "NQ==",
- "null_flag": false
- },
- {
- "min": "MQ==",
- "max": "Mw==",
- "null_flag": false
- },
- {
- "min": "J2J1c2gn",
- "max": "J3RvbSc=",
- "null_flag": false
- }
- ],
- "empty": false
- }]
- }
+ "creation_time": 1553765670
})";
pb1->init_from_json(json_rowset_meta);
pb1->set_start_version(start);
diff --git a/docs/images/cpu-flame-demo.svg b/docs/images/cpu-flame-demo.svg
index f08a8a7605..fd2d435a08 100644
--- a/docs/images/cpu-flame-demo.svg
+++ b/docs/images/cpu-flame-demo.svg
@@ -1441,10 +1441,6 @@
<text text-anchor="" x="1096.63" y="171.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
-<title>doris::SegmentGroup::find_short_key (86 samples, 0.06%)</title><rect x="1180.9" y="193" width="0.6" height="15.0" fill="rgb(219,48,22)" rx="2" ry="2" />
-<text text-anchor="" x="1183.87" y="203.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
-</g>
-<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ReadOnlyFileStream::_assure_data (46 samples, 0.03%)</title><rect x="1150.2" y="97" width="0.4" height="15.0" fill="rgb(247,11,43)" rx="2" ry="2" />
<text text-anchor="" x="1153.21" y="107.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
@@ -1489,18 +1485,10 @@
<text text-anchor="" x="145.38" y="347.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
-<title>doris::SegmentGroup::add_short_key (362 samples, 0.23%)</title><rect x="330.8" y="209" width="2.7" height="15.0" fill="rgb(229,195,16)" rx="2" ry="2" />
-<text text-anchor="" x="333.79" y="219.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
-</g>
-<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ColumnData::_seek_to_row (41,249 samples, 26.61%)</title><rect x="874.1" y="225" width="314.0" height="15.0" fill="rgb(254,128,35)" rx="2" ry="2" />
<text text-anchor="" x="877.12" y="235.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" >doris::ColumnData::_seek_to_row</text>
</g>
<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
-<title>doris::SegmentGroup::add_short_key (100 samples, 0.06%)</title><rect x="149.8" y="209" width="0.8" height="15.0" fill="rgb(239,109,35)" rx="2" ry="2" />
-<text text-anchor="" x="152.85" y="219.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
-</g>
-<g class="func_g" onmouseover="s(this)" onmouseout="c()" onclick="zoom(this)">
<title>doris::ReadOnlyFileStream::_assure_data (178 samples, 0.11%)</title><rect x="1094.6" y="145" width="1.3" height="15.0" fill="rgb(226,175,25)" rx="2" ry="2" />
<text text-anchor="" x="1097.57" y="155.5" font-size="12" font-family="Verdana" fill="rgb(0,0,0)" ></text>
</g>
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 30b127fc78..3e20ff8345 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -109,8 +109,8 @@ message RowsetMetaPB {
repeated KeyBoundsPB segments_key_bounds = 27;
// tablet meta pb, for compaction
optional TabletSchemaPB tablet_schema = 28;
- // spare field id for future use
- optional AlphaRowsetExtraMetaPB alpha_rowset_extra_meta_pb = 50;
+ // alpha_rowset_extra_meta_pb is deleted
+ reserved 50;
// to indicate whether the data between the segments overlap
optional SegmentsOverlapPB segments_overlap_pb = 51 [default = OVERLAP_UNKNOWN];
}
@@ -122,21 +122,6 @@ message RemoteRowsetGcPB {
required int64 num_segments = 3;
}
-message AlphaRowsetExtraMetaPB {
- repeated SegmentGroupPB segment_groups = 1;
-}
-
-message SegmentGroupPB {
- required int32 segment_group_id = 1;
- required int32 num_segments = 2;
- required int64 index_size = 3;
- required int64 data_size = 4;
- required int64 num_rows = 5;
- repeated ZoneMap zone_maps = 6;
- optional bool empty = 7;
- optional PUniqueId load_id = 8;
-}
-
enum DataFileType {
OLAP_DATA_FILE = 0; //Deprecated. Only columnar-wise format is supported.
COLUMN_ORIENTED_FILE = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org