You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by "github-actions[bot] (via GitHub)" <gi...@apache.org> on 2023/02/21 11:28:21 UTC
[GitHub] [doris] github-actions[bot] commented on a diff in pull request #16731: segcompaction featuring verticalcompaction
github-actions[bot] commented on code in PR #16731:
URL: https://github.com/apache/doris/pull/16731#discussion_r1112928298
##########
be/src/olap/rowset/beta_rowset_writer.cpp:
##########
@@ -116,35 +118,46 @@ Status BetaRowsetWriter::add_block(const vectorized::Block* block) {
return _add_block(block, &_segment_writer);
}
-RowwiseIteratorUPtr BetaRowsetWriter::_get_segcompaction_reader(
- SegCompactionCandidatesSharedPtr segments, std::shared_ptr<Schema> schema,
- OlapReaderStatistics* stat, uint64_t* merged_row_stat) {
+std::unique_ptr<vectorized::VerticalBlockReader> BetaRowsetWriter::_get_segcompaction_reader(
Review Comment:
warning: method '_get_segcompaction_reader' can be made const [readability-make-member-function-const]
be/src/olap/rowset/beta_rowset_writer.cpp:122:
```diff
- std::shared_ptr<Schema> schema, OlapReaderStatistics* stat, uint64_t* merged_row_stat, vectorized::RowSourcesBuffer& row_sources_buf, bool is_key, std::vector<uint32_t>& return_columns) {
+ std::shared_ptr<Schema> schema, OlapReaderStatistics* stat, uint64_t* merged_row_stat, vectorized::RowSourcesBuffer& row_sources_buf, bool is_key, std::vector<uint32_t>& return_columns) const {
```
be/src/olap/rowset/beta_rowset_writer.h:121:
```diff
- vectorized::RowSourcesBuffer& row_sources_buf, bool is_key, std::vector<uint32_t>& return_columns);
+ vectorized::RowSourcesBuffer& row_sources_buf, bool is_key, std::vector<uint32_t>& return_columns) const;
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -86,9 +86,26 @@ Status VerticalBlockReader::_init_collect_iter(const ReaderParams& read_params)
std::vector<RowwiseIteratorUPtr> segment_iters;
std::vector<bool> iterator_init_flag;
std::vector<RowsetId> rowset_ids;
- RETURN_IF_ERROR(
- _get_segment_iterators(read_params, &segment_iters, &iterator_init_flag, &rowset_ids));
- CHECK(segment_iters.size() == iterator_init_flag.size());
+ std::vector<RowwiseIterator*>* segment_iters_ptr = read_params.segment_iters_ptr;
+ std::vector<RowwiseIterator*> segment_iters;
Review Comment:
warning: redefinition of 'segment_iters' with a different type: 'vector<doris::RowwiseIterator *>' vs 'vector<doris::RowwiseIteratorUPtr>' [clang-diagnostic-error]
```cpp
std::vector<RowwiseIterator*> segment_iters;
^
```
**be/src/vec/olap/vertical_block_reader.cpp:85:** previous definition is here
```cpp
std::vector<RowwiseIteratorUPtr> segment_iters;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false, &rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1), &mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2), &mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3), &mem_pool);
+ s = rowset_writer->add_row(input_row);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor' [clang-diagnostic-error]
```cpp
input_row.set_field_content(2, reinterpret_cast<char*>(&k3), &mem_pool);
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet = l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir = std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
+ auto tablet = std::make_shared<Tablet>(tablet_meta, data_dir.get(), "test_str");
Review Comment:
warning: '_schema' is a private member of 'doris::TabletMeta' [clang-diagnostic-error]
```cpp
tablet_meta->_schema = tablet_schema;
^
```
**be/src/olap/tablet_meta.h:229:** declared private here
```cpp
TabletSchemaSPtr _schema;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false, &rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1), &mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2), &mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3), &mem_pool);
+ s = rowset_writer->add_row(input_row);
+ EXPECT_EQ(Status::OK(), s);
Review Comment:
warning: no member named 'add_row' in 'doris::RowsetWriter' [clang-diagnostic-error]
```cpp
s = rowset_writer->add_row(input_row);
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -98,11 +115,11 @@
seq_col_idx = read_params.tablet->tablet_schema()->sequence_col_idx();
}
_vcollect_iter = new_vertical_heap_merge_iterator(
Review Comment:
warning: no matching function for call to 'new_vertical_heap_merge_iterator' [clang-diagnostic-error]
```cpp
_vcollect_iter = new_vertical_heap_merge_iterator(
^
```
**be/src/vec/olap/vertical_merge_iterator.h:330:** candidate function not viable: no known conversion from 'vector<doris::RowwiseIterator *>' to 'vector<doris::RowwiseIteratorUPtr>' for 1st argument
```cpp
std::shared_ptr<RowwiseIterator> new_vertical_heap_merge_iterator(
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@ class SegCompactionTest : public testing::Test {
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet = l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir = std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
Review Comment:
warning: '_tablet_id' is a private member of 'doris::TabletMeta' [clang-diagnostic-error]
```cpp
tablet_meta->_tablet_id = 1;
^
```
**be/src/olap/tablet_meta.h:217:** declared private here
```cpp
int64_t _tablet_id = 0;
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -86,9 +86,26 @@
std::vector<RowwiseIteratorUPtr> segment_iters;
std::vector<bool> iterator_init_flag;
std::vector<RowsetId> rowset_ids;
- RETURN_IF_ERROR(
- _get_segment_iterators(read_params, &segment_iters, &iterator_init_flag, &rowset_ids));
- CHECK(segment_iters.size() == iterator_init_flag.size());
+ std::vector<RowwiseIterator*>* segment_iters_ptr = read_params.segment_iters_ptr;
+ std::vector<RowwiseIterator*> segment_iters;
+
+ if (!segment_iters_ptr) {
+ RETURN_IF_ERROR(_get_segment_iterators(read_params, &segment_iters, &iterator_init_flag,
+ &rowset_ids));
+ CHECK(segment_iters.size() == iterator_init_flag.size());
+ segment_iters_ptr = &segment_iters;
Review Comment:
warning: incompatible pointer types assigning to 'std::vector<RowwiseIterator *> *' from 'std::vector<RowwiseIteratorUPtr> *' (aka 'vector<unique_ptr<doris::RowwiseIterator>> *') [clang-diagnostic-error]
```cpp
segment_iters_ptr = &segment_iters;
^
```
##########
be/src/vec/olap/vertical_block_reader.cpp:
##########
@@ -98,11 +115,11 @@
seq_col_idx = read_params.tablet->tablet_schema()->sequence_col_idx();
}
_vcollect_iter = new_vertical_heap_merge_iterator(
- std::move(segment_iters), iterator_init_flag, rowset_ids, ori_return_col_size,
+ *segment_iters_ptr, iterator_init_flag, rowset_ids, ori_return_col_size,
read_params.tablet->keys_type(), seq_col_idx, _row_sources_buffer);
} else {
- _vcollect_iter = new_vertical_mask_merge_iterator(std::move(segment_iters),
- ori_return_col_size, _row_sources_buffer);
+ _vcollect_iter = new_vertical_mask_merge_iterator(*segment_iters_ptr, ori_return_col_size,
Review Comment:
warning: no matching function for call to 'new_vertical_mask_merge_iterator' [clang-diagnostic-error]
```cpp
_vcollect_iter = new_vertical_mask_merge_iterator(*segment_iters_ptr, ori_return_col_size,
^
```
**be/src/vec/olap/vertical_merge_iterator.h:335:** candidate function not viable: no known conversion from 'vector<doris::RowwiseIterator *>' to 'vector<doris::RowwiseIteratorUPtr>' for 1st argument
```cpp
std::shared_ptr<RowwiseIterator> new_vertical_mask_merge_iterator(
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false, &rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1), &mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2), &mem_pool);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor' [clang-diagnostic-error]
```cpp
input_row.set_field_content(0, reinterpret_cast<char*>(&k1), &mem_pool);
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -184,6 +186,27 @@
rowset_writer_context->tablet_schema = tablet_schema;
rowset_writer_context->version.first = 10;
rowset_writer_context->version.second = 10;
+
+#if 0
+ TCreateTabletReq req;
+ req.table_id =
+ req.tablet_id =
+ req.tablet_scheme =
+ req.partition_id =
+ l_engine->create_tablet(req);
+ rowset_writer_context->tablet = l_engine->tablet_manager()->get_tablet(TTabletId tablet_id);
+#endif
+ std::shared_ptr<DataDir> data_dir = std::make_shared<DataDir>(lTestDir);
+ TabletMetaSharedPtr tablet_meta = std::make_shared<TabletMeta>();
+ tablet_meta->_tablet_id = 1;
+ tablet_meta->_schema = tablet_schema;
+ auto tablet = std::make_shared<Tablet>(tablet_meta, data_dir.get(), "test_str");
+ char* tmp_str = (char*)malloc(20);
+ strncpy(tmp_str, "test_tablet_name", 20);
+
+ tablet->_full_name = tmp_str;
+ // tablet->key
Review Comment:
warning: '_full_name' is a protected member of 'doris::BaseTablet' [clang-diagnostic-error]
```cpp
tablet->_full_name = tmp_str;
^
```
**be/src/olap/base_tablet.h:84:** declared protected here
```cpp
std::string _full_name;
^
```
##########
be/test/olap/segcompaction_test.cpp:
##########
@@ -200,12 +223,124 @@
std::unique_ptr<DataDir> _data_dir;
};
+TEST_F(SegCompactionTest, SegCompactionThenRead) {
+ config::enable_segcompaction = true;
+ config::enable_storage_vectorization = true;
+ Status s;
+ TabletSchemaSPtr tablet_schema = std::make_shared<TabletSchema>();
+ create_tablet_schema(tablet_schema, DUP_KEYS);
+
+ RowsetSharedPtr rowset;
+ const int num_segments = 15;
+ const uint32_t rows_per_segment = 4096;
+ config::segcompaction_small_threshold = 6000; // set threshold above
+ // rows_per_segment
+ config::segcompaction_threshold_segment_num = 10;
+ std::vector<uint32_t> segment_num_rows;
+ { // write `num_segments * rows_per_segment` rows to rowset
+ RowsetWriterContext writer_context;
+ create_rowset_writer_context(10047, tablet_schema, &writer_context);
+
+ std::unique_ptr<RowsetWriter> rowset_writer;
+ s = RowsetFactory::create_rowset_writer(writer_context, false, &rowset_writer);
+ EXPECT_EQ(Status::OK(), s);
+
+ RowCursor input_row;
+ input_row.init(tablet_schema);
+
+ // for segment "i", row "rid"
+ // k1 := rid*10 + i
+ // k2 := k1 * 10
+ // k3 := rid
+ for (int i = 0; i < num_segments; ++i) {
+ MemPool mem_pool;
+ for (int rid = 0; rid < rows_per_segment; ++rid) {
+ uint32_t k1 = rid * 100 + i;
+ uint32_t k2 = i;
+ uint32_t k3 = rid;
+ input_row.set_field_content(0, reinterpret_cast<char*>(&k1), &mem_pool);
+ input_row.set_field_content(1, reinterpret_cast<char*>(&k2), &mem_pool);
+ input_row.set_field_content(2, reinterpret_cast<char*>(&k3), &mem_pool);
Review Comment:
warning: no member named 'set_field_content' in 'doris::RowCursor' [clang-diagnostic-error]
```cpp
input_row.set_field_content(1, reinterpret_cast<char*>(&k2), &mem_pool);
^
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org