You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/12/19 06:18:04 UTC
[doris] branch master updated: [fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a7180c5ad8 [fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)
a7180c5ad8 is described below
commit a7180c5ad845a5bdf43eaa38a3ccce26ab9e0574
Author: zhengyu <fr...@gmail.com>
AuthorDate: Mon Dec 19 14:17:58 2022 +0800
[fix](segcompaction) fix segcompaction failed for newly created segment (#15022) (#15023)
Currently, newly created segment could be chosen to be compaction
candidate, which is prone to bugs and segment file open failures. We
should skip last (maybe active) segment while doing segcompaction.
---
be/src/olap/rowset/beta_rowset_writer.cpp | 9 +++++----
be/src/olap/rowset/beta_rowset_writer.h | 3 ++-
be/test/olap/segcompaction_test.cpp | 1 +
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index ac5bf223e5..8da5ca117a 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -383,12 +383,12 @@ void BetaRowsetWriter::compact_segments(SegCompactionCandidatesSharedPtr segment
}
Status BetaRowsetWriter::_load_noncompacted_segments(
- std::vector<segment_v2::SegmentSharedPtr>* segments) {
+ std::vector<segment_v2::SegmentSharedPtr>* segments, size_t num) {
auto fs = _rowset_meta->fs();
if (!fs) {
return Status::Error<INIT_FAILED>();
}
- for (int seg_id = _segcompacted_point; seg_id < _num_segment; ++seg_id) {
+ for (int seg_id = _segcompacted_point; seg_id < num; ++seg_id) {
auto seg_path =
BetaRowset::segment_file_path(_context.rowset_dir, _context.rowset_id, seg_id);
auto cache_path =
@@ -415,7 +415,8 @@ Status BetaRowsetWriter::_load_noncompacted_segments(
Status BetaRowsetWriter::_find_longest_consecutive_small_segment(
SegCompactionCandidatesSharedPtr segments) {
std::vector<segment_v2::SegmentSharedPtr> all_segments;
- RETURN_NOT_OK(_load_noncompacted_segments(&all_segments));
+ // subtract one to skip last (maybe active) segment
+ RETURN_NOT_OK(_load_noncompacted_segments(&all_segments, _num_segment - 1));
if (VLOG_DEBUG_IS_ON) {
vlog_buffer.clear();
@@ -473,7 +474,7 @@ Status BetaRowsetWriter::_get_segcompaction_candidates(SegCompactionCandidatesSh
VLOG_DEBUG << "segcompaction last few segments";
// currently we only rename remaining segments to reduce wait time
// so that transaction can be committed ASAP
- RETURN_NOT_OK(_load_noncompacted_segments(segments.get()));
+ RETURN_NOT_OK(_load_noncompacted_segments(segments.get(), _num_segment));
for (int i = 0; i < segments->size(); ++i) {
RETURN_NOT_OK(_rename_compacted_segment_plain(_segcompacted_point++));
}
diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h
index c9ec00c9b4..d291253937 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -112,7 +112,8 @@ private:
Status _delete_original_segments(uint32_t begin, uint32_t end);
Status _rename_compacted_segments(int64_t begin, int64_t end);
Status _rename_compacted_segment_plain(uint64_t seg_id);
- Status _load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* segments);
+ Status _load_noncompacted_segments(std::vector<segment_v2::SegmentSharedPtr>* segments,
+ size_t num);
Status _find_longest_consecutive_small_segment(SegCompactionCandidatesSharedPtr segments);
Status _get_segcompaction_candidates(SegCompactionCandidatesSharedPtr& segments, bool is_last);
Status _wait_flying_segcompaction();
diff --git a/be/test/olap/segcompaction_test.cpp b/be/test/olap/segcompaction_test.cpp
index eec98ca741..80af09f65f 100644
--- a/be/test/olap/segcompaction_test.cpp
+++ b/be/test/olap/segcompaction_test.cpp
@@ -253,6 +253,7 @@ TEST_F(SegCompactionTest, SegCompactionThenRead) {
ls.push_back("10047_3.dat");
ls.push_back("10047_4.dat");
ls.push_back("10047_5.dat");
+ ls.push_back("10047_6.dat");
EXPECT_TRUE(check_dir(ls));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org