You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/11 07:29:41 UTC
(doris) branch branch-4.0 updated: branch-4.0: [fix](cloud) Avoid deleting nonexistent delete bitmap files #62967 (#63015)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 6c562b8755c branch-4.0: [fix](cloud) Avoid deleting nonexistent delete bitmap files #62967 (#63015)
6c562b8755c is described below
commit 6c562b8755c8bfb015a4ef9a25fe2e6aed8c3d6a
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Mon May 11 15:29:34 2026 +0800
branch-4.0: [fix](cloud) Avoid deleting nonexistent delete bitmap files #62967 (#63015)
Cherry-picked from #62967
Co-authored-by: Yixuan Wang <wa...@selectdb.com>
---
cloud/src/recycler/recycler.cpp | 45 ++++++++++++++++++++---------------
cloud/src/recycler/recycler.h | 14 +++++++----
cloud/test/mock_accessor.h | 2 +-
cloud/test/recycler_test.cpp | 52 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 89 insertions(+), 24 deletions(-)
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index 0ab0c3977f3..59f43596037 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -2981,18 +2981,17 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
}
}
- // Process delete bitmap - check if it's stored in packed file
- bool delete_bitmap_is_packed = false;
+ // Process delete bitmap - check where it's stored.
+ DeleteBitmapStorageType delete_bitmap_storage_type = DeleteBitmapStorageType::NOT_FOUND;
if (decrement_delete_bitmap_packed_file_ref_counts(tablet_id, rowset_id,
- &delete_bitmap_is_packed) != 0) {
+ &delete_bitmap_storage_type) != 0) {
LOG_WARNING("failed to decrement delete bitmap packed file ref count")
.tag("instance_id", instance_id_)
.tag("tablet_id", tablet_id)
.tag("rowset_id", rowset_id);
return -1;
}
- // Only delete standalone delete bitmap file if not stored in packed file
- if (!delete_bitmap_is_packed) {
+ if (delete_bitmap_storage_type == DeleteBitmapStorageType::STANDALONE_FILE) {
file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
}
// TODO(AlexYue): seems could do do batch
@@ -3238,11 +3237,11 @@ int InstanceRecycler::decrement_packed_file_ref_counts(const doris::RowsetMetaCl
return ret;
}
-int InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts(int64_t tablet_id,
- const std::string& rowset_id,
- bool* out_is_packed) {
- if (out_is_packed) {
- *out_is_packed = false;
+int InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts(
+ int64_t tablet_id, const std::string& rowset_id,
+ DeleteBitmapStorageType* out_storage_type) {
+ if (out_storage_type) {
+ *out_storage_type = DeleteBitmapStorageType::NOT_FOUND;
}
// Get delete bitmap storage info from FDB
@@ -3286,15 +3285,24 @@ int InstanceRecycler::decrement_delete_bitmap_packed_file_ref_counts(int64_t tab
return -1;
}
- // Check if delete bitmap is stored in packed file
+ if (storage.store_in_fdb()) {
+ if (out_storage_type) {
+ *out_storage_type = DeleteBitmapStorageType::IN_FDB;
+ }
+ return 0;
+ }
+
+ // Check if delete bitmap is stored in standalone file.
if (!storage.has_packed_slice_location() ||
storage.packed_slice_location().packed_file_path().empty()) {
- // Not stored in packed file, nothing to do
+ if (out_storage_type) {
+ *out_storage_type = DeleteBitmapStorageType::STANDALONE_FILE;
+ }
return 0;
}
- if (out_is_packed) {
- *out_is_packed = true;
+ if (out_storage_type) {
+ *out_storage_type = DeleteBitmapStorageType::PACKED_FILE;
}
const auto& packed_loc = storage.packed_slice_location();
@@ -3633,10 +3641,10 @@ int InstanceRecycler::delete_rowset_data(
continue;
}
- // Process delete bitmap - check if it's stored in packed file
- bool delete_bitmap_is_packed = false;
+ // Process delete bitmap - check where it's stored.
+ DeleteBitmapStorageType delete_bitmap_storage_type = DeleteBitmapStorageType::NOT_FOUND;
if (decrement_delete_bitmap_packed_file_ref_counts(tablet_id, rowset_id,
- &delete_bitmap_is_packed) != 0) {
+ &delete_bitmap_storage_type) != 0) {
LOG_WARNING("failed to decrement delete bitmap packed file ref count")
.tag("instance_id", instance_id_)
.tag("tablet_id", tablet_id)
@@ -3644,8 +3652,7 @@ int InstanceRecycler::delete_rowset_data(
ret = -1;
continue;
}
- // Only delete standalone delete bitmap file if not stored in packed file
- if (!delete_bitmap_is_packed) {
+ if (delete_bitmap_storage_type == DeleteBitmapStorageType::STANDALONE_FILE) {
file_paths.push_back(delete_bitmap_path(tablet_id, rowset_id));
}
diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h
index c717a84aaed..2c3b60042e6 100644
--- a/cloud/src/recycler/recycler.h
+++ b/cloud/src/recycler/recycler.h
@@ -454,13 +454,19 @@ private:
// Returns 0 for success, -1 for error.
int decrement_packed_file_ref_counts(const doris::RowsetMetaCloudPB& rs_meta_pb);
- // Decrement packed file ref count for delete bitmap if it's stored in packed file.
+ enum class DeleteBitmapStorageType {
+ NOT_FOUND,
+ IN_FDB,
+ STANDALONE_FILE,
+ PACKED_FILE,
+ };
+
+ // Process delete bitmap storage and decrement packed file ref count when needed.
// Returns 0 for success, -1 for error.
- // If delete bitmap is not stored in packed file, this function does nothing and returns 0.
- // out_is_packed: if not null, will be set to true if delete bitmap is stored in packed file.
+ // out_storage_type: if not null, will be set to the delete bitmap storage type.
int decrement_delete_bitmap_packed_file_ref_counts(int64_t tablet_id,
const std::string& rowset_id,
- bool* out_is_packed);
+ DeleteBitmapStorageType* out_storage_type);
int delete_packed_file_and_kv(const std::string& packed_file_path,
const std::string& packed_key,
diff --git a/cloud/test/mock_accessor.h b/cloud/test/mock_accessor.h
index 6c187b7ca99..0e29c383899 100644
--- a/cloud/test/mock_accessor.h
+++ b/cloud/test/mock_accessor.h
@@ -151,7 +151,7 @@ inline int MockAccessor::delete_all(int64_t expiration_time) {
}
inline int MockAccessor::delete_files(const std::vector<std::string>& paths) {
- TEST_SYNC_POINT_RETURN_WITH_VALUE("MockAccessor::delete_files", (int)0);
+ TEST_SYNC_POINT_RETURN_WITH_VALUE("MockAccessor::delete_files", (int)0, &paths);
for (auto&& path : paths) {
delete_file(path);
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 976ca7a18d9..a538707a312 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -33,6 +33,7 @@
#include <exception>
#include <future>
#include <memory>
+#include <numeric>
#include <random>
#include <string>
#include <string_view>
@@ -5481,6 +5482,57 @@ TEST(RecyclerTest, delete_rowset_data) {
}
}
+TEST(RecyclerTest, delete_rowset_data_without_delete_bitmap_meta) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ constexpr std::string_view resource_id = "delete_rowset_data_without_delete_bitmap_meta";
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id(std::string(resource_id));
+ obj_info->set_ak(config::test_s3_ak);
+ obj_info->set_sk(config::test_s3_sk);
+ obj_info->set_endpoint(config::test_s3_endpoint);
+ obj_info->set_region(config::test_s3_region);
+ obj_info->set_bucket(config::test_s3_bucket);
+ obj_info->set_prefix(std::string(resource_id));
+
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.begin()->second;
+
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+
+ auto rowset = create_rowset(std::string(resource_id), 10002, 10001, 1, schema);
+ ASSERT_EQ(create_recycle_rowset(txn_kv.get(), accessor.get(), rowset, RecycleRowsetPB::COMPACT,
+ true),
+ 0);
+
+ auto sp = SyncPoint::get_instance();
+ DORIS_CLOUD_DEFER {
+ SyncPoint::get_instance()->clear_all_call_backs();
+ };
+ std::vector<std::string> deleted_paths;
+ sp->set_call_back("MockAccessor::delete_files", [&](auto&& args) {
+ auto* paths = try_any_cast<const std::vector<std::string>*>(args[0]);
+ deleted_paths.insert(deleted_paths.end(), paths->begin(), paths->end());
+ });
+ sp->enable_processing();
+
+ ASSERT_EQ(recycler.delete_rowset_data(rowset), 0);
+ ASSERT_EQ(deleted_paths.size(), 1)
+ << " size: " << deleted_paths.size() << " content: "
+ << std::accumulate(deleted_paths.begin(), deleted_paths.end(), std::string(),
+ [](const std::string& str, const std::string& it) {
+ return str.empty() ? it : str + ", " + it;
+ });
+ EXPECT_EQ(deleted_paths[0], segment_path(rowset.tablet_id(), rowset.rowset_id_v2(), 0));
+}
+
TEST(RecyclerTest, delete_rowset_data_packed_file_single_rowset) {
auto txn_kv = std::make_shared<MemTxnKv>();
ASSERT_EQ(txn_kv->init(), 0);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org