You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2026/10/20 01:47:23 UTC
(doris) branch master updated: [fix](cloud) Restart incomplete delete bitmap reads after transaction expiry (#68100)
This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e1afc16b13a [fix](cloud) Restart incomplete delete bitmap reads after transaction expiry (#68100)
e1afc16b13a is described below
commit e1afc16b13a775354ae4d2e9af59d8e876b6f4dd
Author: meiyi <me...@selectdb.com>
AuthorDate: Tue Oct 20 09:47:16 2026 +0800
[fix](cloud) Restart incomplete delete bitmap reads after transaction expiry (#68100)
Problem Summary: Retrying a paginated delete bitmap read after
TXN_TOO_OLD could return truncated or mixed data if aggregation replaced
the bitmap concurrently.
Restart only the unfinished bitmap and preserve completed results and
byte counts.
Allow three retries per bitmap, resetting the budget only when reading
advances to another bitmap.
---
cloud/src/meta-service/meta_service.cpp | 86 ++++----
cloud/test/meta_service_test.cpp | 336 ++++++++++++++++++++++++++++++++
2 files changed, 385 insertions(+), 37 deletions(-)
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index b12569a9afc..19e1ed1ccd3 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -4663,10 +4663,15 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
std::unique_ptr<RangeGetIterator> it;
int64_t last_ver = -1;
int64_t last_seg_id = -1;
- std::string last_delete_bitmap_first_blob_key;
+ std::string bitmap_start_key = start_key;
+ int bitmap_response_size = response->rowset_ids_size();
+ int64_t bitmap_num_before = delete_bitmap_num;
+ int64_t bitmap_bytes_before = delete_bitmap_byte;
uint16_t next_blob_sequence = 0;
+ size_t first_blob_size = 0;
bool skip_last_delete_bitmap = false;
int64_t round = 0;
+ int64_t retry = 0;
while (it == nullptr /* may be not init */ || it->more()) {
if (test) {
LOG(INFO) << "test";
@@ -4675,8 +4680,21 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
err = txn->get(start_key, end_key, &it);
}
TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_err", &round, &err);
- int64_t retry = 0;
- while (err == TxnErrorCode::TXN_TOO_OLD && retry < 3) {
+ if (err == TxnErrorCode::TXN_TOO_OLD && retry < 3) {
+ // Only earlier bitmaps are known to be complete. Never carry the page-tail
+ // bitmap across snapshots: aggregation can replace it under the same key.
+ while (response->rowset_ids_size() > bitmap_response_size) {
+ response->mutable_rowset_ids()->RemoveLast();
+ response->mutable_segment_ids()->RemoveLast();
+ response->mutable_versions()->RemoveLast();
+ response->mutable_segment_delete_bitmaps()->RemoveLast();
+ }
+ delete_bitmap_num = bitmap_num_before;
+ delete_bitmap_byte = bitmap_bytes_before;
+ start_key = bitmap_start_key;
+ it.reset();
+ last_ver = -1;
+ last_seg_id = -1;
stats.get_bytes += txn->get_bytes();
stats.get_counter += txn->num_get_keys();
txn = nullptr;
@@ -4687,41 +4705,14 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
msg = ss.str();
return;
}
- if (!skip_last_delete_bitmap && !last_delete_bitmap_first_blob_key.empty()) {
- std::string first_blob_value;
- err = txn->get(last_delete_bitmap_first_blob_key, &first_blob_value);
- if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- delete_bitmap_byte -=
- response->segment_delete_bitmaps(
- response->segment_delete_bitmaps_size() - 1)
- .size();
- delete_bitmap_num--;
- response->mutable_rowset_ids()->RemoveLast();
- response->mutable_segment_ids()->RemoveLast();
- response->mutable_versions()->RemoveLast();
- response->mutable_segment_delete_bitmaps()->RemoveLast();
- skip_last_delete_bitmap = true;
- LOG(WARNING)
- << "skip incomplete delete bitmap whose first blob key "
- "disappeared after transaction retry"
- << ", tablet_id=" << tablet_id
- << ", rowset_id=" << rowset_ids[i] << ", version=" << last_ver
- << ", segment_id=" << last_seg_id;
- } else if (err != TxnErrorCode::TXN_OK) {
- retry++;
- continue;
- }
- }
- if (test) {
- err = txn->get(start_key, end_key, &it, false, 2);
- } else {
- err = txn->get(start_key, end_key, &it);
- }
retry++;
LOG(INFO) << "retry get delete bitmap, tablet=" << tablet_id
+ << ", rowset=" << rowset_ids[i]
+ << ", bitmap_start_key=" << hex(bitmap_start_key)
<< ", retry=" << retry << ", internal round=" << round
<< ", delete_bitmap_num=" << delete_bitmap_num
<< ", delete_bitmap_byte=" << delete_bitmap_byte;
+ continue;
}
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::READ>(err);
@@ -4754,6 +4745,18 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
// FIXME: Don't expose the implementation details of splitting large value.
// merge splitted large values (>90*1000)
if (ver != last_ver || seg_id != last_seg_id) {
+ auto key = meta_delete_bitmap_key(
+ {instance_id, tablet_id, rowset_ids[i], ver, seg_id});
+ // A successful page is not progress if a retry will reread the same
+ // bitmap. Reset the budget only when its logical start moves forward.
+ if (key != bitmap_start_key) {
+ DCHECK(key > bitmap_start_key);
+ retry = 0;
+ bitmap_start_key = std::move(key);
+ }
+ bitmap_response_size = response->rowset_ids_size();
+ bitmap_num_before = delete_bitmap_num;
+ bitmap_bytes_before = delete_bitmap_byte;
auto sequence =
out.size() == 8
? static_cast<uint16_t>(
@@ -4761,7 +4764,7 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
: uint16_t {0};
last_ver = ver;
last_seg_id = seg_id;
- last_delete_bitmap_first_blob_key.clear();
+ first_blob_size = v.size();
next_blob_sequence = static_cast<uint16_t>(sequence + 1);
// Key-based pre-rowset cleanup may leave an obsolete tail if its size estimate is
// too small. The bitmap is already aggregated, so the tail can be skipped and will
@@ -4776,9 +4779,6 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
<< ", segment_id=" << seg_id << ", first_sequence=" << sequence;
continue;
}
- if (out.size() == 8) {
- last_delete_bitmap_first_blob_key.assign(k.data(), k.size());
- }
response->add_rowset_ids(rowset_ids[i]);
response->add_segment_ids(seg_id);
response->add_versions(ver);
@@ -4816,6 +4816,18 @@ void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
delete_bitmap_byte += v.length();
response->mutable_segment_delete_bitmaps()->rbegin()->append(v);
}
+ // blob_put uses equal-sized chunks except for the last, with a minimum
+ // split size. A legacy value or a short chunk is therefore complete.
+ if (out.size() == 7 || v.size() < MIN_BLOB_SPLIT_SIZE ||
+ v.size() < first_blob_size) {
+ // Skip this entire logical bitmap on retry, including any new fragments
+ // written by a concurrent replacement after this snapshot.
+ encode_int64(INT64_MAX, &bitmap_start_key);
+ bitmap_response_size = response->rowset_ids_size();
+ bitmap_num_before = delete_bitmap_num;
+ bitmap_bytes_before = delete_bitmap_byte;
+ retry = 0;
+ }
}
check_delete_bitmap_bytes_exceed_limit();
if (code != MetaServiceCode::OK) return;
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index f1b6a5734d2..a2ec86680de 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -8377,6 +8377,342 @@ TEST(MetaServiceTest, GetDeleteBitmapDropsPartialValueAfterAllBlobKeysAreRemoved
test_get_delete_bitmap_during_point_cleanup(DEFAULT_BLOB_SPLIT_SIZE * 3 + 1);
}
+TEST(MetaServiceTest, GetDeleteBitmapReturnsCompleteReplacementAfterTxnTooOld) {
+ for (size_t replacement_size : {size_t {1}, DEFAULT_BLOB_SPLIT_SIZE * 3}) {
+ SCOPED_TRACE(fmt::format("replacement_size={}", replacement_size));
+ const bool read_v2 = replacement_size != 1;
+ auto meta_service = get_meta_service();
+ auto sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>& rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id");
+ constexpr int64_t tablet_id = 652;
+ const std::string rowset_id = "replacement_during_paginated_read";
+ put_delete_bitmap_test_rowset(meta_service.get(), instance_id, tablet_id, 1, rowset_id);
+
+ auto bitmap_key = meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, 2, 0});
+ std::string old_bitmap(DEFAULT_BLOB_SPLIT_SIZE, 'a');
+ old_bitmap.append(DEFAULT_BLOB_SPLIT_SIZE, 'b');
+ old_bitmap.append(DEFAULT_BLOB_SPLIT_SIZE, 'c');
+ const std::string previous_rowset = "completed_rowset";
+ const std::string following_rowset = "following_rowset";
+ const std::string completed_bitmap = "completed_bitmap";
+ DeleteBitmapStoragePB v2_storage;
+ v2_storage.set_store_in_fdb(true);
+ auto* v2_bitmap = v2_storage.mutable_delete_bitmap();
+ v2_bitmap->add_rowset_ids(previous_rowset);
+ v2_bitmap->add_segment_ids(0);
+ v2_bitmap->add_versions(1);
+ v2_bitmap->add_segment_delete_bitmaps(completed_bitmap);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ // Keep complete legacy values both in an earlier rowset and before the replaced bitmap.
+ for (const auto& id : {previous_rowset, rowset_id, following_rowset}) {
+ txn->put(meta_delete_bitmap_key({instance_id, tablet_id, id, 1, 0}), completed_bitmap);
+ }
+ blob_put(txn.get(),
+ versioned::meta_delete_bitmap_key({instance_id, tablet_id, previous_rowset}),
+ v2_storage, 0);
+ blob_put(txn.get(), bitmap_key, old_bitmap, 0);
+ TabletStatsPB tablet_stats;
+ tablet_stats.set_base_compaction_cnt(9);
+ tablet_stats.set_cumulative_compaction_cnt(19);
+ tablet_stats.set_cumulative_point(20);
+ txn->put(stats_tablet_key({instance_id, 650, 653, 651, tablet_id}),
+ tablet_stats.SerializeAsString());
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ // The MS treats bitmap bytes as opaque; distinct chunks expose truncation and splicing.
+ const std::string replacement_bitmap(replacement_size, 'x');
+ UpdateDeleteBitmapRequest update_req;
+ update_req.set_cloud_unique_id("test_cloud_unique_id");
+ update_req.set_table_id(650);
+ update_req.set_partition_id(651);
+ update_req.set_tablet_id(tablet_id);
+ update_req.set_lock_id(-3);
+ update_req.set_without_lock(true);
+ update_req.set_initiator(tablet_id);
+ update_req.set_pre_rowset_agg_start_version(1);
+ update_req.set_pre_rowset_agg_end_version(2);
+ update_req.add_rowset_ids(rowset_id);
+ update_req.add_segment_ids(0);
+ update_req.add_versions(2);
+ update_req.add_segment_delete_bitmaps(replacement_bitmap);
+ update_req.add_pre_rowset_versions(1);
+
+ bool replacement_done = false;
+ sp->set_call_back("get_delete_bitmap_test",
+ [&](auto&& args) { *try_any_cast<bool*>(args[0]) = true; });
+ sp->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
+ auto round = *try_any_cast<int64_t*>(args[0]);
+ if (round != 1 || replacement_done) {
+ return;
+ }
+ replacement_done = true;
+ // Page 0 has already appended a complete bitmap and old chunk 0. Replace the same
+ // logical bitmap, then expire the read before page 1 is consumed. The new
+ // transaction still finds sequence 0, but must not keep the old prefix.
+ brpc::Controller update_cntl;
+ UpdateDeleteBitmapResponse update_res;
+ meta_service->update_delete_bitmap(
+ reinterpret_cast<google::protobuf::RpcController*>(&update_cntl), &update_req,
+ &update_res, nullptr);
+ ASSERT_EQ(update_res.status().code(), MetaServiceCode::OK);
+ *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
+ });
+ sp->enable_processing();
+
+ brpc::Controller get_cntl;
+ GetDeleteBitmapRequest get_req;
+ GetDeleteBitmapResponse get_res;
+ get_req.set_cloud_unique_id("test_cloud_unique_id");
+ get_req.set_tablet_id(tablet_id);
+ get_req.set_store_version(read_v2 ? 3 : 1);
+ for (const auto& id : {previous_rowset, rowset_id, following_rowset}) {
+ get_req.add_rowset_ids(id);
+ get_req.add_begin_versions(id == previous_rowset ? 0 : 1);
+ get_req.add_end_versions(3);
+ }
+ auto* idx = get_req.mutable_idx();
+ idx->set_db_id(1);
+ idx->set_table_id(650);
+ idx->set_index_id(653);
+ idx->set_partition_id(651);
+ idx->set_tablet_id(tablet_id);
+ get_req.set_base_compaction_cnt(tablet_stats.base_compaction_cnt());
+ get_req.set_cumulative_compaction_cnt(tablet_stats.cumulative_compaction_cnt());
+ get_req.set_cumulative_point(tablet_stats.cumulative_point());
+ // v1 must leave room for the following rowset after dropping the old prefix.
+ // Mixed reads must retain the earlier v2 bytes and stop exactly at the threshold.
+ get_req.set_dbm_bytes_threshold(completed_bitmap.size() * 2 + replacement_size +
+ (read_v2 ? v2_storage.ByteSizeLong() : 1));
+ meta_service->get_delete_bitmap(
+ reinterpret_cast<google::protobuf::RpcController*>(&get_cntl), &get_req, &get_res,
+ nullptr);
+
+ ASSERT_TRUE(replacement_done);
+ ASSERT_EQ(get_res.status().code(), MetaServiceCode::OK);
+ const int expected_bitmap_count = read_v2 ? 3 : 4;
+ ASSERT_EQ(get_res.rowset_ids_size(), expected_bitmap_count);
+ ASSERT_EQ(get_res.segment_ids_size(), expected_bitmap_count);
+ ASSERT_EQ(get_res.versions_size(), expected_bitmap_count);
+ ASSERT_EQ(get_res.segment_delete_bitmaps_size(), expected_bitmap_count);
+ EXPECT_EQ(get_res.rowset_ids(0), previous_rowset);
+ EXPECT_EQ(get_res.rowset_ids(1), rowset_id);
+ EXPECT_EQ(get_res.rowset_ids(2), rowset_id);
+ if (!read_v2) {
+ EXPECT_EQ(get_res.rowset_ids(3), following_rowset);
+ }
+ for (int i = 0; i < expected_bitmap_count; ++i) {
+ EXPECT_EQ(get_res.segment_ids(i), 0);
+ EXPECT_EQ(get_res.versions(i), i == 2 ? 2 : 1);
+ EXPECT_EQ(get_res.segment_delete_bitmaps(i),
+ i == 2 ? replacement_bitmap : completed_bitmap);
+ }
+ EXPECT_EQ(get_res.has_more(), read_v2);
+ EXPECT_EQ(get_res.returned_rowset_ids_size(), read_v2 ? 2 : 3);
+ ASSERT_EQ(get_res.delta_rowset_ids_size(), read_v2 ? 1 : 0);
+ ASSERT_EQ(get_res.delete_bitmap_storages_size(), read_v2 ? 1 : 0);
+ if (read_v2) {
+ EXPECT_EQ(get_res.delta_rowset_ids(0), previous_rowset);
+ EXPECT_EQ(get_res.delete_bitmap_storages(0).SerializeAsString(),
+ v2_storage.SerializeAsString());
+ }
+ }
+}
+
+TEST(MetaServiceTest, GetDeleteBitmapRetryBudgetRequiresBitmapProgress) {
+ for (const std::string scenario : {"first_page", "same_bitmap", "next_bitmap"}) {
+ SCOPED_TRACE(scenario);
+ auto meta_service = get_meta_service();
+ auto sp = SyncPoint::get_instance();
+ const bool old_enable_txn_store_retry = config::enable_txn_store_retry;
+ config::enable_txn_store_retry = false;
+ DORIS_CLOUD_DEFER {
+ config::enable_txn_store_retry = old_enable_txn_store_retry;
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ };
+
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>& rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id");
+ constexpr int64_t tablet_id = 662;
+ constexpr int bitmap_count = 5;
+ const std::string rowset_id = "retry_budget_rowset";
+ const std::string bitmap(DEFAULT_BLOB_SPLIT_SIZE * 5, 'a');
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ for (int version = 1; version <= bitmap_count; ++version) {
+ blob_put(txn.get(),
+ meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, version, 0}),
+ bitmap, 0);
+ }
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ brpc::Controller cntl;
+ GetDeleteBitmapRequest req;
+ GetDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_tablet_id(tablet_id);
+ req.add_rowset_ids(rowset_id);
+ req.add_begin_versions(0);
+ req.add_end_versions(bitmap_count);
+ int injected_errors = 0;
+ int64_t last_retried_version = 0;
+ const int expected_errors = scenario == "next_bitmap" ? bitmap_count : 4;
+ sp->set_call_back("get_delete_bitmap_test",
+ [&](auto&& args) { *try_any_cast<bool*>(args[0]) = true; });
+ sp->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
+ if (scenario != "first_page") {
+ // Read several pages successfully before expiring the transaction. Those
+ // pages must not reset the budget while retrying the same logical bitmap.
+ if (res.versions_size() == 0 ||
+ res.segment_delete_bitmaps(res.segment_delete_bitmaps_size() - 1).size() <
+ DEFAULT_BLOB_SPLIT_SIZE * 4) {
+ return;
+ }
+ auto version = res.versions(res.versions_size() - 1);
+ if (scenario == "next_bitmap" && version <= last_retried_version) {
+ return;
+ }
+ last_retried_version = version;
+ }
+ ++injected_errors;
+ // Terminate even if a broken retry loop fails to enforce its own budget.
+ *try_any_cast<TxnErrorCode*>(args[1]) = injected_errors <= expected_errors
+ ? TxnErrorCode::TXN_TOO_OLD
+ : TxnErrorCode::TXN_INVALID_ARGUMENT;
+ });
+ sp->enable_processing();
+ meta_service->get_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&cntl),
+ &req, &res, nullptr);
+
+ EXPECT_EQ(injected_errors, expected_errors);
+ if (scenario != "next_bitmap") {
+ EXPECT_EQ(res.status().code(), MetaServiceCode::KV_TXN_TOO_OLD);
+ } else {
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ ASSERT_EQ(res.rowset_ids_size(), bitmap_count);
+ ASSERT_EQ(res.segment_ids_size(), bitmap_count);
+ ASSERT_EQ(res.versions_size(), bitmap_count);
+ ASSERT_EQ(res.segment_delete_bitmaps_size(), bitmap_count);
+ for (int i = 0; i < bitmap_count; ++i) {
+ EXPECT_EQ(res.rowset_ids(i), rowset_id);
+ EXPECT_EQ(res.segment_ids(i), 0);
+ EXPECT_EQ(res.versions(i), i + 1);
+ EXPECT_EQ(res.segment_delete_bitmaps(i), bitmap);
+ }
+ }
+ }
+}
+
+TEST(MetaServiceTest, GetDeleteBitmapResetsRetryBudgetAfterCompletePageTail) {
+ for (const std::string scenario : {"legacy", "single_blob", "short_tail", "full_chunks"}) {
+ SCOPED_TRACE(scenario);
+ auto meta_service = get_meta_service();
+ auto sp = SyncPoint::get_instance();
+ const bool old_enable_txn_store_retry = config::enable_txn_store_retry;
+ config::enable_txn_store_retry = false;
+ DORIS_CLOUD_DEFER {
+ sp->clear_all_call_backs();
+ sp->disable_processing();
+ config::enable_txn_store_retry = old_enable_txn_store_retry;
+ };
+
+ extern std::string get_instance_id(const std::shared_ptr<ResourceManager>& rc_mgr,
+ const std::string& cloud_unique_id);
+ auto instance_id = get_instance_id(meta_service->resource_mgr(), "test_cloud_unique_id");
+ constexpr int64_t tablet_id = 663;
+ const std::string rowset_id = "complete_page_tail";
+ const size_t split_size =
+ scenario == "full_chunks" ? MIN_BLOB_SPLIT_SIZE : DEFAULT_BLOB_SPLIT_SIZE;
+ const size_t bitmap_size = scenario == "short_tail" ? split_size + MIN_BLOB_SPLIT_SIZE * 2
+ : scenario == "full_chunks" ? split_size * 2
+ : MIN_BLOB_SPLIT_SIZE / 2;
+ const std::string bitmap(bitmap_size, 'a');
+ const std::string following_bitmap = "following_bitmap";
+ auto bitmap_key = meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, 1, 0});
+ auto bitmap_end_key = bitmap_key;
+ encode_int64(INT64_MAX, &bitmap_end_key);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ if (scenario == "legacy") {
+ txn->put(bitmap_key, bitmap);
+ } else {
+ blob_put(txn.get(), bitmap_key, bitmap, 0, split_size);
+ }
+ txn->put(meta_delete_bitmap_key({instance_id, tablet_id, rowset_id, 2, 0}),
+ following_bitmap);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ GetDeleteBitmapRequest req;
+ GetDeleteBitmapResponse res;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_tablet_id(tablet_id);
+ req.set_store_version(1);
+ req.add_rowset_ids(rowset_id);
+ req.add_begin_versions(1);
+ req.add_end_versions(2);
+ req.add_rowset_ids("next_rowset");
+ req.add_begin_versions(1);
+ req.add_end_versions(2);
+ req.set_dbm_bytes_threshold(bitmap.size() + following_bitmap.size());
+ // Put the last fragment of A at a page boundary, before any key from B is decoded.
+ sp->set_call_back("memkv::Transaction::get",
+ [](auto&& args) { *try_any_cast<int*>(args[0]) = 1; });
+ int injected_errors = 0;
+ sp->set_call_back("get_delete_bitmap_err", [&](auto&& args) {
+ if (injected_errors < 3) {
+ ++injected_errors;
+ *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
+ } else if (injected_errors == 3 && res.segment_delete_bitmaps_size() == 1 &&
+ res.segment_delete_bitmaps(0) == bitmap) {
+ // Grow A before expiring the continuation read. A completed checkpoint must
+ // retain the original A and skip all of its replacement's fragments.
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->remove(bitmap_key, bitmap_end_key);
+ blob_put(txn.get(), bitmap_key, std::string(DEFAULT_BLOB_SPLIT_SIZE * 3, 'x'), 0);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ ++injected_errors;
+ *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_TOO_OLD;
+ }
+ });
+ sp->enable_processing();
+ brpc::Controller cntl;
+ meta_service->get_delete_bitmap(reinterpret_cast<google::protobuf::RpcController*>(&cntl),
+ &req, &res, nullptr);
+
+ EXPECT_EQ(injected_errors, 4);
+ if (scenario == "full_chunks") {
+ // Full chunks below the default split size do not prove completion. Keep the
+ // exhausted budget rather than guessing that A has ended.
+ EXPECT_EQ(res.status().code(), MetaServiceCode::KV_TXN_TOO_OLD);
+ } else {
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ ASSERT_EQ(res.rowset_ids_size(), 2);
+ ASSERT_EQ(res.segment_ids_size(), 2);
+ ASSERT_EQ(res.versions_size(), 2);
+ ASSERT_EQ(res.segment_delete_bitmaps_size(), 2);
+ for (int i = 0; i < 2; ++i) {
+ EXPECT_EQ(res.rowset_ids(i), rowset_id);
+ EXPECT_EQ(res.segment_ids(i), 0);
+ EXPECT_EQ(res.versions(i), i + 1);
+ }
+ EXPECT_EQ(res.segment_delete_bitmaps(0), bitmap);
+ EXPECT_EQ(res.segment_delete_bitmaps(1), following_bitmap);
+ EXPECT_TRUE(res.has_more());
+ ASSERT_EQ(res.returned_rowset_ids_size(), 1);
+ EXPECT_EQ(res.returned_rowset_ids(0), rowset_id);
+ }
+ }
+}
+
TEST(MetaServiceTest, RemovePreDeleteBitmapBatchesEachBlobKey) {
auto meta_service = get_meta_service();
auto sp = SyncPoint::get_instance();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org