You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/19 07:15:25 UTC
[doris] 01/08: [fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1be148080dd4f4b62578cbb01886c1e351633721
Author: zhengyu <fr...@gmail.com>
AuthorDate: Mon Dec 19 14:17:58 2022 +0800
[fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)
Currently, newly created segment could be chosen to be compaction
candidate, which is prone to bugs and segment file open failures. We
should skip last (maybe active) segment while doing segcompaction.
---
be/src/olap/rowset/beta_rowset_writer.cpp | 9 +++++----
be/src/olap/rowset/beta_rowset_writer.h | 3 ++-
be/test/olap/segcompaction_test.cpp | 1 +
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 512e4d85b1..687e8bac35 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -382,12 +382,12 @@ void BetaRowsetWriter::compact_segments(SegCompactionCandidatesSharedPtr segment
}
Status BetaRowsetWriter::_load_noncompacted_segments(
- std::vector<segment_v2::SegmentSharedPtr>* segments) {
+ std::vector<segment_v2::SegmentSharedPtr>* segments, size_t num) {
auto fs = _rowset_meta->fs();
if (!fs) {
return Status::OLAPInternalError(OLAP_ERR_INIT_FAILED);
}
- for (int seg_id = _segcompacted_point; seg_id < _num_segment; ++seg_id) {
+ for (int seg_id = _segcompacted_point; seg_id < num; ++seg_id) {
auto seg_path =
BetaRowset::segment_file_path(_context.rowset_dir, _context.rowset_id, seg_id);
auto cache_path =
@@ -414,7 +414,8 @@ Status BetaRowsetWriter::_load_noncompacted_segments(
Status BetaRowsetWriter::_find_longest_consecutive_small_segment(
SegCompactionCandidatesSharedPtr segments) {
std::vector<segment_v2::SegmentSharedPtr> all_segments;
- RETURN_NOT_OK(_load_noncompacted_segments(&all_segments));
+ // subtract one to skip last (maybe active) segment
+ RETURN_NOT_OK(_load_noncompacted_segments(&all_segments, _num_segment - 1));
if (VLOG_DEBUG_IS_ON) {
vlog_buffer.clear();
@@ -472,7 +473,7 @@ Status BetaRowsetWriter::_get_segcompaction_candidates(SegCompactionCandidatesSh
VLOG_DEBUG << "segcompaction last few segments";
// currently we only rename remaining segments to reduce wait time
// so that transaction can be committed ASAP
- RETURN_NOT_OK(_load_noncompacted_segments(segments.get()));
+ RETURN_NOT_OK(_load_noncompacted_segments(segments.get(), _num_segment));
for (int i = 0; i < segments->size(); ++i) {
RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
}
diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h
index 5f4077d4b5..04e13810ae 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -110,7 +110,8 @@ private:
Status _delete_original_segments(uint32_t begin, uint32_t end);
Status _rename_compacted_segments(int64_t begin, int64_t end);
Status _rename_compacted_segment_plain(uint64_t seg_id);
- Status _load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* segments);
+ Status _load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* segments,
+ size_t num);
Status _find_longest_consecutive_small_segment(SegCompactionCandidatesSharedPtr segments);
Status _get_segcompaction_candidates(SegCompactionCandidatesSharedPtr& segments, bool is_last);
Status _wait_flying_segcompaction();
diff --git a/be/test/olap/segcompaction_test.cpp b/be/test/olap/segcompaction_test.cpp
index 205fa0cbbd..cb9fb010c0 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -252,6 +252,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
ls.push_back("10047_3.dat");
ls.push_back("10047_4.dat");
ls.push_back("10047_5.dat");
+ ls.push_back("10047_6.dat");
EXPECT_TRUE(check_dir(ls));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org