You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ya...@apache.org on 2020/08/04 09:31:20 UTC
[incubator-doris] branch support_batch_delete_in_fe updated: add
write and read delete flag in segment
This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch support_batch_delete_in_fe
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/support_batch_delete_in_fe by this push:
new 1813e16 add write and read delete flag in segment
1813e16 is described below
commit 1813e1685cd0f3d5d54cd3b4dbdddfb01750c564
Author: yangzhg <ya...@gmail.com>
AuthorDate: Tue Aug 4 17:24:37 2020 +0800
add write and read delete flag in segment
---
be/src/olap/reader.cpp | 7 +++---
be/src/olap/row_cursor.h | 2 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 27 +++++++++++++++++++-----
be/src/olap/rowset/segment_v2/segment_writer.h | 4 ++--
4 files changed, 29 insertions(+), 11 deletions(-)
diff --git a/be/src/olap/reader.cpp b/be/src/olap/reader.cpp
index 5a2017f..cca3b2f 100644
--- a/be/src/olap/reader.cpp
+++ b/be/src/olap/reader.cpp
@@ -411,13 +411,14 @@ OLAPStatus Reader::_unique_key_next_row(RowCursor* row_cursor, MemPool* mem_pool
*eof = true;
return OLAP_SUCCESS;
}
-
cur_delete_flag = _next_delete_flag;
// the verion is in reverse order, the first row is the highest version,
// in UNIQUE_KEY highest version is the final result, there is no need to
// merge the lower versions
- direct_copy_row(row_cursor, *_next_key);
- agg_finalize_row(_value_cids, row_cursor, mem_pool);
+ if (!_next_key->is_delete()) {
+ direct_copy_row(row_cursor, *_next_key);
+ agg_finalize_row(_value_cids, row_cursor, mem_pool);
+ }
// skip the lower version rows;
while (nullptr != _next_key) {
auto res = _collect_iter->next(&_next_key, &_next_delete_flag);
diff --git a/be/src/olap/row_cursor.h b/be/src/olap/row_cursor.h
index fb77fe7..1678ef4 100644
--- a/be/src/olap/row_cursor.h
+++ b/be/src/olap/row_cursor.h
@@ -76,7 +76,7 @@ public:
void set_is_delete(bool is_delete) { _is_delete = is_delete; }
- bool is_delete() { return _is_delete; }
+ bool is_delete() const { return _is_delete; }
// deep copy field content (ignore null-byte)
void set_field_content(size_t index, const char* buf, MemPool* mem_pool) {
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index a9f7aba..aa8b2d5 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -25,6 +25,7 @@
#include "olap/rowset/segment_v2/column_writer.h" // ColumnWriter
#include "olap/rowset/segment_v2/page_io.h"
#include "olap/short_key_index.h"
+#include "olap/delete_bitmap_index.h"
#include "olap/schema.h"
#include "util/crc32c.h"
#include "util/faststring.h"
@@ -77,7 +78,7 @@ Status SegmentWriter::init(uint32_t write_mbytes_per_sec __attribute__((unused))
_column_writers.push_back(std::move(writer));
}
_index_builder.reset(new ShortKeyIndexBuilder(_segment_id, _opts.num_rows_per_block));
- _delete_bitmap_builder.reset(new DeleteBitmapIndexBuilder());
+ _delete_index_builder.reset(new DeleteBitmapIndexBuilder());
return Status::OK();
}
@@ -99,7 +100,24 @@ Status SegmentWriter::append_row(const RowType& row) {
}
template Status SegmentWriter::append_row(const RowCursor& row);
-template Status SegmentWriter::append_row(const ContiguousRow& row);
+template<> Status SegmentWriter::append_row(const ContiguousRow& row) {
+ for (size_t cid = 0; cid < _column_writers.size(); ++cid) {
+ auto cell = row.cell(cid);
+ RETURN_IF_ERROR(_column_writers[cid]->append(cell));
+ }
+
+ // At the begin of one block, so add a short key index entry
+ if ((_row_count % _opts.num_rows_per_block) == 0) {
+ std::string encoded_key;
+ encode_key(&encoded_key, row, _tablet_schema->num_short_key_columns());
+ RETURN_IF_ERROR(_index_builder->add_item(encoded_key));
+ }
+ if (row.is_delete()) {
+ RETURN_IF_ERROR(_delete_index_builder->add_delete_item(_row_count));
+ }
+ ++_row_count;
+ return Status::OK();
+}
// TODO(lingbin): Currently this function does not include the size of various indexes,
// We should make this more precise.
@@ -112,7 +130,6 @@ uint64_t SegmentWriter::estimate_segment_size() {
size += column_writer->estimate_buffer_size();
}
size += _index_builder->size();
- size += _delete_bitmap_builder->size();
return size;
}
@@ -186,9 +203,9 @@ Status SegmentWriter::_write_short_key_index() {
Status SegmentWriter::_write_delete_index() {
std::vector<Slice> body;
PageFooterPB footer;
- RETURN_IF_ERROR(_delete_bitmap_builder->finalize(&body, &footer));
+ RETURN_IF_ERROR(_delete_index_builder->finalize(&body, &footer));
PagePointer pp;
- // delete index page is not compressed right now
+ // short key index page is not compressed right now
RETURN_IF_ERROR(PageIO::write_page(_wblock, body, footer, &pp));
pp.to_proto(_footer.mutable_delete_index_page());
return Status::OK();
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h
index 8eee044..6093785 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/segment_writer.h
@@ -25,7 +25,6 @@
#include "common/status.h" // Status
#include "gen_cpp/segment_v2.pb.h"
#include "gutil/macros.h"
-#include "olap/delete_bitmap_index.h"
namespace doris {
@@ -33,6 +32,7 @@ class RowBlock;
class RowCursor;
class TabletSchema;
class ShortKeyIndexBuilder;
+class DeleteBitmapIndexBuilder;
namespace fs {
class WritableBlock;
@@ -90,7 +90,7 @@ private:
SegmentFooterPB _footer;
std::unique_ptr<ShortKeyIndexBuilder> _index_builder;
- std::unique_ptr<DeleteBitmapIndexBuilder> _delete_bitmap_builder;
+ std::unique_ptr<DeleteBitmapIndexBuilder> _delete_index_builder;
std::vector<std::unique_ptr<ColumnWriter>> _column_writers;
uint32_t _row_count = 0;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org