You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by la...@apache.org on 2022/09/21 16:14:22 UTC
[incubator-pegasus] branch backup_restore-dev updated: feat(backup): 9. replica handle backup clear context and invalid status (#1150)
This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch backup_restore-dev
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/backup_restore-dev by this push:
new 3c33c999e feat(backup): 9. replica handle backup clear context and invalid status (#1150)
3c33c999e is described below
commit 3c33c999ee4c71661103ae247e6f09828fbb608f
Author: HeYuchen <he...@xiaomi.com>
AuthorDate: Thu Sep 22 00:14:16 2022 +0800
feat(backup): 9. replica handle backup clear context and invalid status (#1150)
---
docker/README.md | 2 +-
src/rdsn/src/common/backup_restore_common.cpp | 4 +
src/rdsn/src/common/replication_common.cpp | 8 -
src/rdsn/src/common/replication_common.h | 2 -
.../src/replica/backup/replica_backup_manager.cpp | 79 +++++++-
.../src/replica/backup/replica_backup_manager.h | 3 +
.../backup/test/replica_backup_manager_test.cpp | 213 ++++++++++++++++++++-
src/rdsn/src/replica/replica_config.cpp | 2 +
8 files changed, 298 insertions(+), 15 deletions(-)
diff --git a/docker/README.md b/docker/README.md
index 2faecbc66..ca2afa28f 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -27,7 +27,7 @@ to deploying a standalone cluster of Pegasus containers on your local machine.
[](https://github.com/apache/incubator-pegasus/actions/workflows/thirdparty-regular-push.yml)
-[](https://github.com/apache/incubator-pegasus/actions/workflows/pegasus-regular-build.yml)
+[](https://github.com/apache/incubator-pegasus/actions/workflows/regular-build.yml)
## pegasus-build-env
diff --git a/src/rdsn/src/common/backup_restore_common.cpp b/src/rdsn/src/common/backup_restore_common.cpp
index 231ae5e1a..c0da9d583 100644
--- a/src/rdsn/src/common/backup_restore_common.cpp
+++ b/src/rdsn/src/common/backup_restore_common.cpp
@@ -27,6 +27,10 @@ DSN_DEFINE_string("replication",
cold_backup_root,
"",
"cold backup remote block service storage path prefix");
+DSN_DEFINE_uint32("replication",
+ cold_backup_checkpoint_reserve_minutes,
+ 10,
+ "reserve minutes of cold backup checkpoint");
const std::string backup_constant::APP_METADATA("app_metadata");
const std::string backup_constant::APP_BACKUP_STATUS("app_backup_status");
diff --git a/src/rdsn/src/common/replication_common.cpp b/src/rdsn/src/common/replication_common.cpp
index 78bed8405..7143565b1 100644
--- a/src/rdsn/src/common/replication_common.cpp
+++ b/src/rdsn/src/common/replication_common.cpp
@@ -110,8 +110,6 @@ replication_options::replication_options()
lb_interval_ms = 10000;
learn_app_max_concurrent_count = 5;
-
- cold_backup_checkpoint_reserve_minutes = 10;
}
replication_options::~replication_options() {}
@@ -398,12 +396,6 @@ void replication_options::initialize()
learn_app_max_concurrent_count,
"max count of learning app concurrently");
- cold_backup_checkpoint_reserve_minutes =
- (int)dsn_config_get_value_uint64("replication",
- "cold_backup_checkpoint_reserve_minutes",
- cold_backup_checkpoint_reserve_minutes,
- "reserve minutes of cold backup checkpoint");
-
max_concurrent_bulk_load_downloading_count = FLAGS_max_concurrent_bulk_load_downloading_count;
dassert_f(replica_helper::load_meta_servers(meta_servers), "invalid meta server config");
diff --git a/src/rdsn/src/common/replication_common.h b/src/rdsn/src/common/replication_common.h
index 2e46bec99..34536c695 100644
--- a/src/rdsn/src/common/replication_common.h
+++ b/src/rdsn/src/common/replication_common.h
@@ -109,8 +109,6 @@ public:
int32_t learn_app_max_concurrent_count;
- int32_t cold_backup_checkpoint_reserve_minutes;
-
int32_t max_concurrent_bulk_load_downloading_count;
public:
diff --git a/src/rdsn/src/replica/backup/replica_backup_manager.cpp b/src/rdsn/src/replica/backup/replica_backup_manager.cpp
index 1da4e708a..265ffc69b 100644
--- a/src/rdsn/src/replica/backup/replica_backup_manager.cpp
+++ b/src/rdsn/src/replica/backup/replica_backup_manager.cpp
@@ -23,7 +23,7 @@
namespace dsn {
namespace replication {
-// TODO(heyuchen): implement it
+DSN_DECLARE_uint32(cold_backup_checkpoint_reserve_minutes);
replica_backup_manager::replica_backup_manager(replica *r)
: replica_base(r), _replica(r), _stub(r->get_replica_stub())
@@ -36,7 +36,12 @@ replica_backup_manager::~replica_backup_manager() {}
void replica_backup_manager::on_backup(const backup_request &request,
/*out*/ backup_response &response)
{
- // TODO(heyuchen): add other status
+ if (request.status == backup_status::FAILED || request.status == backup_status::CANCELED) {
+ ddebug_replica("start to cleanup backup context, request status = {}",
+ enum_to_string(request.status));
+ clear_context();
+ return;
+ }
if (request.status == backup_status::CHECKPOINTING) {
try_to_checkpoint(request.backup_id, response);
@@ -50,6 +55,11 @@ void replica_backup_manager::on_backup(const backup_request &request,
response);
return;
}
+
+ // other status:
+ // backup_status::UNINITIALIZED, backup_status::CHECKPOINTED, backup_status::SUCCEED
+ derror_replica("invalid request backup_status = {}", enum_to_string(request.status));
+ response.err = ERR_INVALID_STATE;
}
// ThreadPool: THREAD_POOL_REPLICATION
@@ -414,7 +424,70 @@ void replica_backup_manager::clear_context()
{
FAIL_POINT_INJECT_F("replica_backup_clear_context",
[this](dsn::string_view) { _status = backup_status::UNINITIALIZED; });
- // TODO(heyuchen): TBD
+
+ zauto_write_lock l(_lock);
+ if (_status == backup_status::UNINITIALIZED) {
+ return;
+ }
+ ddebug_replica("start to clear backup context, old status = {}", enum_to_string(_status));
+
+ for (const auto &kv : _upload_files_task) {
+ cleanup_backup_task(kv.second);
+ }
+ cleanup_backup_task(_uploading_task);
+ _upload_files_task.clear();
+ _uploading_task = nullptr;
+ _upload_err = ERR_OK;
+ _upload_file_size.store(0);
+
+ cleanup_backup_task(_checkpointing_task);
+ _checkpointing_task = nullptr;
+ _checkpoint_err = ERR_OK;
+ _backup_metadata.files.clear();
+ _backup_metadata.checkpoint_total_size = 0;
+ _backup_metadata.checkpoint_decree = 0;
+ _backup_metadata.checkpoint_timestamp = 0;
+
+ background_clear_backup_checkpoint(_backup_id);
+ _backup_id = 0;
+ _status = backup_status::UNINITIALIZED;
+}
+
+// ThreadPool: THREAD_POOL_REPLICATION
+void replica_backup_manager::background_clear_backup_checkpoint(int64_t backup_id)
+{
+ ddebug_replica("schedule to checkpoint dir of backup({}) after {} minutes",
+ backup_id,
+ FLAGS_cold_backup_checkpoint_reserve_minutes);
+ tasking::enqueue(LPC_BACKGROUND_COLD_BACKUP,
+ tracker(),
+ std::bind(&replica_backup_manager::clear_backup_checkpoint, this, backup_id),
+ get_gpid().thread_hash(),
+ std::chrono::minutes(FLAGS_cold_backup_checkpoint_reserve_minutes));
+}
+
+// ThreadPool: THREAD_POOL_REPLICATION_LONG
+void replica_backup_manager::clear_backup_checkpoint(int64_t backup_id)
+{
+ ddebug_replica("clear checkpoint dirs of backup({})", backup_id);
+ const auto &backup_dir = _replica->_app->backup_dir();
+ if (!utils::filesystem::directory_exists(backup_dir)) {
+ return;
+ }
+
+ const auto &path = utils::filesystem::path_combine(backup_dir, std::to_string(backup_id));
+ if (utils::filesystem::remove_path(path)) {
+ ddebug_replica("remove backup checkpoint dir({}) succeed", path);
+ } else {
+ derror_replica("remove backup checkpoint dir({}) failed", path);
+ }
+}
+
+// ThreadPool: THREAD_POOL_REPLICATION
+bool replica_backup_manager::cleanup_backup_task(task_ptr task_)
+{
+ CLEANUP_TASK(task_, false)
+ return true;
}
} // namespace replication
diff --git a/src/rdsn/src/replica/backup/replica_backup_manager.h b/src/rdsn/src/replica/backup/replica_backup_manager.h
index aca0989ff..a1ca798b8 100644
--- a/src/rdsn/src/replica/backup/replica_backup_manager.h
+++ b/src/rdsn/src/replica/backup/replica_backup_manager.h
@@ -93,6 +93,9 @@ private:
int32_t calc_upload_progress();
void clear_context();
+ void background_clear_backup_checkpoint(int64_t backup_id);
+ void clear_backup_checkpoint(int64_t backup_id);
+ bool cleanup_backup_task(task_ptr task_);
task_tracker *tracker() { return _replica->tracker(); }
diff --git a/src/rdsn/src/replica/backup/test/replica_backup_manager_test.cpp b/src/rdsn/src/replica/backup/test/replica_backup_manager_test.cpp
index 1afd81045..60ce643bf 100644
--- a/src/rdsn/src/replica/backup/test/replica_backup_manager_test.cpp
+++ b/src/rdsn/src/replica/backup/test/replica_backup_manager_test.cpp
@@ -24,6 +24,8 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_uint32(cold_backup_checkpoint_reserve_minutes);
+
class replica_backup_manager_test : public replica_test_base
{
public:
@@ -42,6 +44,23 @@ public:
fail::teardown();
}
+ error_code on_backup(backup_status::type local_status, backup_status::type request_status)
+ {
+ mock_local_backup_states(local_status);
+
+ backup_request req;
+ req.pid = _replica->get_gpid();
+ req.app_name = APP_NAME;
+ req.backup_id = dsn_now_ms();
+ req.status = request_status;
+ req.backup_provider_type = PROVIDER;
+ req.__set_backup_root_path(PATH);
+
+ backup_response resp;
+ _backup_mgr->on_backup(req, resp);
+ return resp.err;
+ }
+
void generate_checkpoint() { _backup_mgr->generate_checkpoint(); }
bool set_backup_metadata()
@@ -65,6 +84,12 @@ public:
void report_uploading(backup_response &response) { _backup_mgr->report_uploading(response); }
+ void clear_context()
+ {
+ _backup_mgr->clear_context();
+ _backup_mgr->tracker()->wait_outstanding_tasks();
+ }
+
void mock_local_backup_states(backup_status::type status,
error_code checkpoint_err = ERR_OK,
error_code upload_err = ERR_OK,
@@ -120,7 +145,178 @@ protected:
std::unique_ptr<replica_backup_manager> _backup_mgr;
};
-// TODO(heyuchen): add unit test for on_backup after implement all status
+TEST_F(replica_backup_manager_test, on_backup_test)
+{
+ fail::cfg("replica_backup_start_checkpointing", "return()");
+ fail::cfg("replica_backup_start_uploading", "return()");
+ fail::cfg("replica_backup_upload_completed", "return()");
+ fail::cfg("replica_backup_clear_context", "return()");
+
+ struct test_struct
+ {
+ backup_status::type local_status;
+ backup_status::type request_status;
+ error_code expected_err;
+ backup_status::type expected_status;
+ } tests[]{
+ // request_status = UNINITIALIZED
+ {backup_status::UNINITIALIZED,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTED,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTED},
+ {backup_status::UPLOADING,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::UPLOADING},
+ {backup_status::SUCCEED,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::SUCCEED},
+ {backup_status::FAILED,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::FAILED},
+ {backup_status::CANCELED,
+ backup_status::UNINITIALIZED,
+ ERR_INVALID_STATE,
+ backup_status::CANCELED},
+ // request_status = CHECKPOINTING
+ {backup_status::UNINITIALIZED,
+ backup_status::CHECKPOINTING,
+ ERR_OK,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTING,
+ backup_status::CHECKPOINTING,
+ ERR_OK,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTED,
+ backup_status::CHECKPOINTING,
+ ERR_OK,
+ backup_status::CHECKPOINTED},
+ {backup_status::UPLOADING,
+ backup_status::CHECKPOINTING,
+ ERR_INVALID_STATE,
+ backup_status::UPLOADING},
+ {backup_status::SUCCEED,
+ backup_status::CHECKPOINTING,
+ ERR_INVALID_STATE,
+ backup_status::SUCCEED},
+ {backup_status::FAILED,
+ backup_status::CHECKPOINTING,
+ ERR_INVALID_STATE,
+ backup_status::FAILED},
+ {backup_status::CANCELED,
+ backup_status::CHECKPOINTING,
+ ERR_INVALID_STATE,
+ backup_status::CANCELED},
+ // request_status = CHECKPOINTED
+ {backup_status::UNINITIALIZED,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTED,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTED},
+ {backup_status::UPLOADING,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::UPLOADING},
+ {backup_status::SUCCEED,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::SUCCEED},
+ {backup_status::FAILED,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::FAILED},
+ {backup_status::CANCELED,
+ backup_status::CHECKPOINTED,
+ ERR_INVALID_STATE,
+ backup_status::CANCELED},
+ // request_status = UPLOADING
+ {backup_status::UNINITIALIZED,
+ backup_status::UPLOADING,
+ ERR_INVALID_STATE,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING,
+ backup_status::UPLOADING,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTED, backup_status::UPLOADING, ERR_OK, backup_status::UPLOADING},
+ {backup_status::UPLOADING, backup_status::UPLOADING, ERR_OK, backup_status::UPLOADING},
+ {backup_status::SUCCEED, backup_status::UPLOADING, ERR_OK, backup_status::SUCCEED},
+ {backup_status::FAILED, backup_status::UPLOADING, ERR_INVALID_STATE, backup_status::FAILED},
+ {backup_status::CANCELED,
+ backup_status::UPLOADING,
+ ERR_INVALID_STATE,
+ backup_status::CANCELED},
+ // request_status = SUCCEED
+ {backup_status::UNINITIALIZED,
+ backup_status::SUCCEED,
+ ERR_INVALID_STATE,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING,
+ backup_status::SUCCEED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTING},
+ {backup_status::CHECKPOINTED,
+ backup_status::SUCCEED,
+ ERR_INVALID_STATE,
+ backup_status::CHECKPOINTED},
+ {backup_status::UPLOADING,
+ backup_status::SUCCEED,
+ ERR_INVALID_STATE,
+ backup_status::UPLOADING},
+ {backup_status::SUCCEED, backup_status::SUCCEED, ERR_INVALID_STATE, backup_status::SUCCEED},
+ {backup_status::FAILED, backup_status::SUCCEED, ERR_INVALID_STATE, backup_status::FAILED},
+ {backup_status::CANCELED,
+ backup_status::SUCCEED,
+ ERR_INVALID_STATE,
+ backup_status::CANCELED},
+ // request_status = FAILED
+ {backup_status::UNINITIALIZED, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTED, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::UPLOADING, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::SUCCEED, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::FAILED, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::CANCELED, backup_status::FAILED, ERR_OK, backup_status::UNINITIALIZED},
+ // request_status = CANCELED
+ {backup_status::UNINITIALIZED,
+ backup_status::CANCELED,
+ ERR_OK,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTING,
+ backup_status::CANCELED,
+ ERR_OK,
+ backup_status::UNINITIALIZED},
+ {backup_status::CHECKPOINTED,
+ backup_status::CANCELED,
+ ERR_OK,
+ backup_status::UNINITIALIZED},
+ {backup_status::UPLOADING, backup_status::CANCELED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::SUCCEED, backup_status::CANCELED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::FAILED, backup_status::CANCELED, ERR_OK, backup_status::UNINITIALIZED},
+ {backup_status::CANCELED, backup_status::CANCELED, ERR_OK, backup_status::UNINITIALIZED}};
+
+ for (const auto &test : tests) {
+ ASSERT_EQ(on_backup(test.local_status, test.request_status), test.expected_err);
+ ASSERT_EQ(get_status(), test.expected_status);
+ }
+}
TEST_F(replica_backup_manager_test, generate_checkpoint_test)
{
@@ -213,5 +409,20 @@ TEST_F(replica_backup_manager_test, report_uploading_test)
}
}
+TEST_F(replica_backup_manager_test, clear_context_test)
+{
+ FLAGS_cold_backup_checkpoint_reserve_minutes = 0;
+ mock_local_backup_states(backup_status::SUCCEED, ERR_OK, ERR_OK, 100);
+ auto dir_name = create_local_backup_checkpoint_dir();
+ create_local_backup_file(dir_name, FILE_NAME1);
+
+ clear_context();
+
+ ASSERT_EQ(get_status(), backup_status::UNINITIALIZED);
+ ASSERT_EQ(get_checkpoint_err(), ERR_OK);
+ ASSERT_EQ(get_upload_err(), ERR_OK);
+ ASSERT_FALSE(utils::filesystem::directory_exists(dir_name));
+}
+
} // namespace replication
} // namespace dsn
diff --git a/src/rdsn/src/replica/replica_config.cpp b/src/rdsn/src/replica/replica_config.cpp
index ca98759ea..d2d9ac4ed 100644
--- a/src/rdsn/src/replica/replica_config.cpp
+++ b/src/rdsn/src/replica/replica_config.cpp
@@ -38,6 +38,7 @@
#include "mutation.h"
#include "mutation_log.h"
#include "replica_stub.h"
+#include "backup/replica_backup_manager.h"
#include "bulk_load/replica_bulk_loader.h"
#include "runtime/security/access_controller.h"
#include "split/replica_split_manager.h"
@@ -849,6 +850,7 @@ bool replica::update_local_configuration(const replica_configuration &config,
switch (old_status) {
case partition_status::PS_PRIMARY:
cleanup_preparing_mutations(false);
+ _backup_mgr->clear_context();
switch (config.status) {
case partition_status::PS_PRIMARY:
replay_prepare_list();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org