You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by zh...@apache.org on 2023/03/01 07:44:26 UTC
[incubator-pegasus] branch master updated: refactor(conf): use DSN_DEFINE_string to load string type of configs (#1371)
This is an automated email from the ASF dual-hosted git repository.
zhaoliwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 98e359888 refactor(conf): use DSN_DEFINE_string to load string type of configs (#1371)
98e359888 is described below
commit 98e3598881657426537b81e55e5072ee8bc097e6
Author: Yingchun Lai <la...@apache.org>
AuthorDate: Wed Mar 1 15:44:19 2023 +0800
refactor(conf): use DSN_DEFINE_string to load string type of configs (#1371)
---
src/common/replication_common.cpp | 27 +++++----
src/common/replication_common.h | 2 -
src/meta/meta_http_service.cpp | 8 +--
src/meta/meta_options.cpp | 69 ++++++++--------------
src/meta/meta_options.h | 12 +---
src/meta/meta_server_failure_detector.cpp | 6 +-
src/meta/meta_service.cpp | 36 ++++++++---
src/meta/test/backup_test.cpp | 8 ++-
src/meta/test/meta_test_base.cpp | 6 +-
src/meta/test/server_state_test.cpp | 6 +-
src/meta/test/state_sync_test.cpp | 8 ++-
src/replica/backup/replica_backup_server.cpp | 12 ++--
src/replica/replica_backup.cpp | 6 +-
src/replica/storage/simple_kv/test/checker.cpp | 4 +-
src/replica/test/replica_test.cpp | 13 ++--
src/runtime/rpc/network.cpp | 41 +++++--------
src/runtime/test/main.cpp | 6 +-
src/runtime/test/service_api_c.cpp | 5 +-
src/server/available_detector.cpp | 56 ++++++++++--------
src/server/available_detector.h | 3 -
src/server/info_collector.cpp | 12 ++--
src/server/info_collector.h | 2 -
src/server/pegasus_server_impl_init.cpp | 48 +++++++--------
src/test/bench_test/benchmark.cpp | 24 +++++++-
src/test/bench_test/config.cpp | 17 +-----
src/test/bench_test/config.h | 2 -
.../function_test/recovery_test/test_recovery.cpp | 2 +-
src/test/kill_test/data_verifier.cpp | 26 ++++----
src/test/kill_test/data_verifier.h | 4 ++
src/test/kill_test/kill_testor.cpp | 22 +++----
src/test/kill_test/kill_testor.h | 2 -
src/test/kill_test/killer_handler_shell.cpp | 23 +++-----
src/test/kill_test/killer_handler_shell.h | 6 +-
src/test/kill_test/main.cpp | 4 +-
src/test/kill_test/process_kill_testor.cpp | 14 ++---
src/test/pressure_test/main.cpp | 31 +++++-----
src/zookeeper/zookeeper_session.cpp | 11 ++--
src/zookeeper/zookeeper_session_mgr.cpp | 9 ++-
src/zookeeper/zookeeper_session_mgr.h | 6 --
39 files changed, 291 insertions(+), 308 deletions(-)
diff --git a/src/common/replication_common.cpp b/src/common/replication_common.cpp
index 97681e428..49de27202 100644
--- a/src/common/replication_common.cpp
+++ b/src/common/replication_common.cpp
@@ -88,6 +88,17 @@ DSN_DEFINE_bool(replication,
"whether to disable empty write, default is false");
DSN_TAG_VARIABLE(empty_write_disabled, FT_MUTABLE);
+DSN_DEFINE_string(replication, slog_dir, "", "The shared log directory");
+DSN_DEFINE_string(replication, data_dirs, "", "replica directory list");
+DSN_DEFINE_string(replication,
+ data_dirs_black_list_file,
+ "/home/work/.pegasus_data_dirs_black_list",
+ "replica directory black list file");
+DSN_DEFINE_string(replication,
+ cold_backup_root,
+ "",
+ "The prefix of cold backup data path on remote storage");
+
replication_options::~replication_options() {}
void replication_options::initialize()
@@ -99,7 +110,7 @@ void replication_options::initialize()
// slog_dir:
// - if config[slog_dir] is empty: "app_dir/slog"
// - else: "config[slog_dir]/app_name/slog"
- slog_dir = dsn_config_get_value_string("replication", "slog_dir", "", "shared log directory");
+ slog_dir = FLAGS_slog_dir;
if (slog_dir.empty()) {
slog_dir = app_dir;
} else {
@@ -108,23 +119,16 @@ void replication_options::initialize()
slog_dir = utils::filesystem::path_combine(slog_dir, "slog");
// get config_data_dirs and config_data_dir_tags from config
- const std::string &dirs_str =
- dsn_config_get_value_string("replication", "data_dirs", "", "replica directory list");
std::vector<std::string> config_data_dirs;
std::vector<std::string> config_data_dir_tags;
std::string error_msg = "";
bool flag = get_data_dir_and_tag(
- dirs_str, app_dir, app_name, config_data_dirs, config_data_dir_tags, error_msg);
+ FLAGS_data_dirs, app_dir, app_name, config_data_dirs, config_data_dir_tags, error_msg);
CHECK(flag, error_msg);
// check if data_dir in black list, data_dirs doesn't contain dir in black list
- std::string black_list_file =
- dsn_config_get_value_string("replication",
- "data_dirs_black_list_file",
- "/home/work/.pegasus_data_dirs_black_list",
- "replica directory black list file");
std::vector<std::string> black_list_dirs;
- get_data_dirs_in_black_list(black_list_file, black_list_dirs);
+ get_data_dirs_in_black_list(FLAGS_data_dirs_black_list_file, black_list_dirs);
for (auto i = 0; i < config_data_dirs.size(); ++i) {
if (check_if_in_black_list(black_list_dirs, config_data_dirs[i])) {
continue;
@@ -135,9 +139,6 @@ void replication_options::initialize()
CHECK(!data_dirs.empty(), "no replica data dir found, maybe not set or excluded by black list");
- cold_backup_root = dsn_config_get_value_string(
- "replication", "cold_backup_root", "", "cold backup remote storage path prefix");
-
max_concurrent_bulk_load_downloading_count = FLAGS_max_concurrent_bulk_load_downloading_count;
CHECK(replica_helper::load_meta_servers(meta_servers), "invalid meta server config");
diff --git a/src/common/replication_common.h b/src/common/replication_common.h
index 531f8d18d..1e3ec0312 100644
--- a/src/common/replication_common.h
+++ b/src/common/replication_common.h
@@ -65,8 +65,6 @@ public:
std::vector<std::string> data_dirs;
std::vector<std::string> data_dir_tags;
- std::string cold_backup_root;
-
int32_t max_concurrent_bulk_load_downloading_count;
public:
diff --git a/src/meta/meta_http_service.cpp b/src/meta/meta_http_service.cpp
index 2091fc2b7..f6d9c7e49 100644
--- a/src/meta/meta_http_service.cpp
+++ b/src/meta/meta_http_service.cpp
@@ -40,6 +40,9 @@
#include "meta/meta_bulk_load_service.h"
namespace dsn {
+namespace dist {
+DSN_DECLARE_string(hosts_list);
+} // namespace dist
namespace replication {
struct list_nodes_helper
@@ -456,10 +459,7 @@ void meta_http_service::get_cluster_info_handler(const http_request &req, http_r
}
tp.add_row_name_and_data("meta_servers", meta_servers_str);
tp.add_row_name_and_data("primary_meta_server", dsn_primary_address().to_std_string());
- std::string zk_hosts =
- dsn_config_get_value_string("zookeeper", "hosts_list", "", "zookeeper_hosts");
- zk_hosts.erase(std::remove_if(zk_hosts.begin(), zk_hosts.end(), ::isspace), zk_hosts.end());
- tp.add_row_name_and_data("zookeeper_hosts", zk_hosts);
+ tp.add_row_name_and_data("zookeeper_hosts", dsn::dist::FLAGS_hosts_list);
tp.add_row_name_and_data("zookeeper_root", _service->_cluster_root);
tp.add_row_name_and_data(
"meta_function_level",
diff --git a/src/meta/meta_options.cpp b/src/meta/meta_options.cpp
index 6457564e8..979fce956 100644
--- a/src/meta/meta_options.cpp
+++ b/src/meta/meta_options.cpp
@@ -38,6 +38,24 @@
namespace dsn {
namespace replication {
+// TODO(yingchun): add more description for string configs, and add validators
+DSN_DEFINE_string(meta_server,
+ meta_state_service_parameters,
+ "",
+ "meta_state_service provider parameters");
+DSN_DEFINE_string(meta_server,
+ meta_function_level_on_start,
+ "steady",
+ "meta function level on start");
+DSN_DEFINE_string(meta_server,
+ distributed_lock_service_parameters,
+ "",
+ "distributed_lock_service provider parameters");
+DSN_DEFINE_string(meta_server,
+ replica_white_list,
+ "",
+ "white list of replica-servers in meta-server");
+
std::string meta_options::concat_path_unix_style(const std::string &prefix,
const std::string &postfix)
{
@@ -52,24 +70,10 @@ std::string meta_options::concat_path_unix_style(const std::string &prefix,
void meta_options::initialize()
{
- cluster_root = dsn_config_get_value_string(
- "meta_server", "cluster_root", "/", "cluster root of meta state service on remote");
-
- meta_state_service_type = dsn_config_get_value_string("meta_server",
- "meta_state_service_type",
- "meta_state_service_simple",
- "meta_state_service provider type");
- const char *meta_state_service_parameters =
- dsn_config_get_value_string("meta_server",
- "meta_state_service_parameters",
- "",
- "meta_state_service provider parameters");
- utils::split_args(meta_state_service_parameters, meta_state_service_args);
+ utils::split_args(FLAGS_meta_state_service_parameters, meta_state_service_args);
meta_function_level_on_start = meta_function_level::fl_invalid;
- const char *level_str = dsn_config_get_value_string(
- "meta_server", "meta_function_level_on_start", "steady", "meta function level on start");
- std::string level = std::string("fl_") + level_str;
+ std::string level = std::string("fl_") + FLAGS_meta_function_level_on_start;
for (auto &kv : _meta_function_level_VALUES_TO_NAMES) {
if (level == kv.second) {
meta_function_level_on_start = (meta_function_level::type)kv.first;
@@ -79,36 +83,11 @@ void meta_options::initialize()
CHECK_NE_MSG(meta_function_level_on_start,
meta_function_level::fl_invalid,
"invalid function level: {}",
- level_str);
-
- /// failure detector options
- _fd_opts.distributed_lock_service_type =
- dsn_config_get_value_string("meta_server",
- "distributed_lock_service_type",
- "distributed_lock_service_simple",
- "dist lock provider");
- const char *distributed_lock_service_parameters =
- dsn_config_get_value_string("meta_server",
- "distributed_lock_service_parameters",
- "",
- "distributed_lock_service provider parameters");
- utils::split_args(distributed_lock_service_parameters, _fd_opts.distributed_lock_service_args);
-
- /// load balancer options
- _lb_opts.server_load_balancer_type =
- dsn_config_get_value_string("meta_server",
- "server_load_balancer_type",
- "greedy_load_balancer",
- "server load balancer provider");
-
- partition_guardian_type = dsn_config_get_value_string("meta_server",
- "partition_guardian_type",
- "partition_guardian",
- "partition guardian provider");
+ FLAGS_meta_function_level_on_start);
- const char *replica_white_list_raw = dsn_config_get_value_string(
- "meta_server", "replica_white_list", "", "white list of replica-servers in meta-server");
- utils::split_args(replica_white_list_raw, replica_white_list, ',');
+ utils::split_args(FLAGS_distributed_lock_service_parameters,
+ _fd_opts.distributed_lock_service_args);
+ utils::split_args(FLAGS_replica_white_list, replica_white_list, ',');
}
} // namespace replication
} // namespace dsn
diff --git a/src/meta/meta_options.h b/src/meta/meta_options.h
index 3a62eaf04..a3ec5c5d9 100644
--- a/src/meta/meta_options.h
+++ b/src/meta/meta_options.h
@@ -54,31 +54,21 @@
namespace dsn {
namespace replication {
+// TODO(yingchun): remove it
class fd_suboptions
{
public:
- std::string distributed_lock_service_type;
std::vector<std::string> distributed_lock_service_args;
};
-class lb_suboptions
-{
-public:
- std::string server_load_balancer_type;
-};
-
class meta_options
{
public:
- std::string cluster_root;
- std::string meta_state_service_type;
std::vector<std::string> meta_state_service_args;
meta_function_level::type meta_function_level_on_start;
fd_suboptions _fd_opts;
- lb_suboptions _lb_opts;
- std::string partition_guardian_type;
std::vector<std::string> replica_white_list;
public:
diff --git a/src/meta/meta_server_failure_detector.cpp b/src/meta/meta_server_failure_detector.cpp
index 919dbec12..d61c2c05c 100644
--- a/src/meta/meta_server_failure_detector.cpp
+++ b/src/meta/meta_server_failure_detector.cpp
@@ -46,6 +46,10 @@ DSN_DEFINE_uint64(meta_server,
stable_rs_min_running_seconds,
600,
"The minimal running seconds for a stable replica server");
+DSN_DEFINE_string(meta_server,
+ distributed_lock_service_type,
+ "distributed_lock_service_simple",
+ "dist lock provider");
namespace dsn {
namespace replication {
@@ -59,7 +63,7 @@ meta_server_failure_detector::meta_server_failure_detector(meta_service *svc)
{
_fd_opts = &(svc->get_meta_options()._fd_opts);
_lock_svc = dsn::utils::factory_store<dist::distributed_lock_service>::create(
- _fd_opts->distributed_lock_service_type.c_str(), PROVIDER_TYPE_MAIN);
+ FLAGS_distributed_lock_service_type, PROVIDER_TYPE_MAIN);
error_code err = _lock_svc->initialize(_fd_opts->distributed_lock_service_args);
CHECK_EQ_MSG(err, ERR_OK, "init distributed_lock_service failed");
}
diff --git a/src/meta/meta_service.cpp b/src/meta/meta_service.cpp
index 1249cce26..a61e7be28 100644
--- a/src/meta/meta_service.cpp
+++ b/src/meta/meta_service.cpp
@@ -48,6 +48,10 @@
#include "meta_bulk_load_service.h"
namespace dsn {
+namespace dist {
+DSN_DECLARE_string(hosts_list);
+} // namespace dist
+
namespace replication {
DSN_DEFINE_bool(meta_server,
recover_from_replica_server,
@@ -75,12 +79,29 @@ DSN_DEFINE_uint64(meta_server,
65,
"If live_node_count * 100 < total_node_count * "
"node_live_percentage_threshold_for_update, then freeze the cluster.");
+DSN_DEFINE_string(meta_server,
+ meta_state_service_type,
+ "meta_state_service_simple",
+ "meta_state_service provider type");
+DSN_DEFINE_string(meta_server,
+ cluster_root,
+ "/",
+ "The root of the cluster meta state service to be stored on remote storage");
+DSN_DEFINE_string(meta_server,
+ server_load_balancer_type,
+ "greedy_load_balancer",
+ "server load balancer provider");
+DSN_DEFINE_string(meta_server,
+ partition_guardian_type,
+ "partition_guardian",
+ "partition guardian provider");
DSN_DECLARE_bool(duplication_enabled);
DSN_DECLARE_int32(fd_beacon_interval_seconds);
DSN_DECLARE_int32(fd_check_interval_seconds);
DSN_DECLARE_int32(fd_grace_seconds);
DSN_DECLARE_int32(fd_lease_seconds);
+DSN_DECLARE_string(cold_backup_root);
meta_service::meta_service()
: serverlet("meta_service"), _failure_detector(nullptr), _started(false), _recovering(false)
@@ -144,7 +165,7 @@ error_code meta_service::remote_storage_initialize()
// create storage
dsn::dist::meta_state_service *storage =
dsn::utils::factory_store<::dsn::dist::meta_state_service>::create(
- _meta_opts.meta_state_service_type.c_str(), PROVIDER_TYPE_MAIN);
+ FLAGS_meta_state_service_type, PROVIDER_TYPE_MAIN);
error_code err = storage->initialize(_meta_opts.meta_state_service_args);
if (err != ERR_OK) {
LOG_ERROR("init meta_state_service failed, err = {}", err);
@@ -154,7 +175,7 @@ error_code meta_service::remote_storage_initialize()
_meta_storage.reset(new mss::meta_storage(_storage.get(), &_tracker));
std::vector<std::string> slices;
- utils::split_args(_meta_opts.cluster_root.c_str(), slices, '/');
+ utils::split_args(FLAGS_cluster_root, slices, '/');
std::string current = "";
for (unsigned int i = 0; i != slices.size(); ++i) {
current = meta_options::concat_path_unix_style(current, slices[i]);
@@ -357,13 +378,13 @@ error_code meta_service::start()
// initialize the load balancer
server_load_balancer *balancer = utils::factory_store<server_load_balancer>::create(
- _meta_opts._lb_opts.server_load_balancer_type.c_str(), PROVIDER_TYPE_MAIN, this);
+ FLAGS_server_load_balancer_type, PROVIDER_TYPE_MAIN, this);
_balancer.reset(balancer);
// register control command to singleton-container for load balancer
_balancer->register_ctrl_commands();
partition_guardian *guardian = utils::factory_store<partition_guardian>::create(
- _meta_opts.partition_guardian_type.c_str(), PROVIDER_TYPE_MAIN, this);
+ FLAGS_partition_guardian_type, PROVIDER_TYPE_MAIN, this);
_partition_guardian.reset(guardian);
_partition_guardian->register_ctrl_commands();
@@ -374,7 +395,7 @@ error_code meta_service::start()
_backup_handler = std::make_shared<backup_service>(
this,
meta_options::concat_path_unix_style(_cluster_root, "backup"),
- _opts.cold_backup_root,
+ FLAGS_cold_backup_root,
[](backup_service *bs) { return std::make_shared<policy_context>(bs); });
}
@@ -633,11 +654,8 @@ void meta_service::on_query_cluster_info(configuration_cluster_info_rpc rpc)
response.values.push_back(oss.str());
response.keys.push_back("primary_meta_server");
response.values.push_back(dsn_primary_address().to_std_string());
- std::string zk_hosts =
- dsn_config_get_value_string("zookeeper", "hosts_list", "", "zookeeper_hosts");
- zk_hosts.erase(std::remove_if(zk_hosts.begin(), zk_hosts.end(), ::isspace), zk_hosts.end());
response.keys.push_back("zookeeper_hosts");
- response.values.push_back(zk_hosts);
+ response.values.push_back(dsn::dist::FLAGS_hosts_list);
response.keys.push_back("zookeeper_root");
response.values.push_back(_cluster_root);
response.keys.push_back("meta_function_level");
diff --git a/src/meta/test/backup_test.cpp b/src/meta/test/backup_test.cpp
index 0bba049cd..53065bba8 100644
--- a/src/meta/test/backup_test.cpp
+++ b/src/meta/test/backup_test.cpp
@@ -42,6 +42,8 @@ namespace dsn {
namespace replication {
DSN_DECLARE_int32(cold_backup_checkpoint_reserve_minutes);
+DSN_DECLARE_string(cluster_root);
+DSN_DECLARE_string(meta_state_service_type);
struct method_record
{
@@ -728,11 +730,11 @@ protected:
meta_test_base::SetUp();
meta_options &opt = _meta_svc->_meta_opts;
- opt.cluster_root = "/meta_test";
- opt.meta_state_service_type = "meta_state_service_simple";
+ FLAGS_cluster_root = "/meta_test";
+ FLAGS_meta_state_service_type = "meta_state_service_simple";
_meta_svc->remote_storage_initialize();
std::string backup_root = "/backup_test";
- std::string policy_meta_root = opt.cluster_root + "/backup_policies";
+ std::string policy_meta_root = std::string(FLAGS_cluster_root) + "/backup_policies";
_meta_svc->_backup_handler = std::make_shared<backup_service>(
_meta_svc.get(), policy_meta_root, backup_root, [](backup_service *bs) {
return std::make_shared<mock_policy>(bs);
diff --git a/src/meta/test/meta_test_base.cpp b/src/meta/test/meta_test_base.cpp
index 2fd486e9e..9a34cdbd8 100644
--- a/src/meta/test/meta_test_base.cpp
+++ b/src/meta/test/meta_test_base.cpp
@@ -31,6 +31,8 @@ namespace dsn {
namespace replication {
DSN_DECLARE_uint64(min_live_node_count_for_unfreeze);
+DSN_DECLARE_string(partition_guardian_type);
+DSN_DECLARE_string(server_load_balancer_type);
meta_test_base::~meta_test_base() {}
@@ -39,9 +41,9 @@ void meta_test_base::SetUp()
_ms = make_unique<fake_receiver_meta_service>();
_ms->_failure_detector.reset(new meta_server_failure_detector(_ms.get()));
_ms->_balancer.reset(utils::factory_store<server_load_balancer>::create(
- _ms->_meta_opts._lb_opts.server_load_balancer_type.c_str(), PROVIDER_TYPE_MAIN, _ms.get()));
+ FLAGS_server_load_balancer_type, PROVIDER_TYPE_MAIN, _ms.get()));
_ms->_partition_guardian.reset(utils::factory_store<partition_guardian>::create(
- _ms->_meta_opts.partition_guardian_type.c_str(), PROVIDER_TYPE_MAIN, _ms.get()));
+ FLAGS_partition_guardian_type, PROVIDER_TYPE_MAIN, _ms.get()));
ASSERT_EQ(_ms->remote_storage_initialize(), ERR_OK);
_ms->initialize_duplication_service();
ASSERT_TRUE(_ms->_dup_svc);
diff --git a/src/meta/test/server_state_test.cpp b/src/meta/test/server_state_test.cpp
index c3135612a..6350483f1 100644
--- a/src/meta/test/server_state_test.cpp
+++ b/src/meta/test/server_state_test.cpp
@@ -37,6 +37,8 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_string(cluster_root);
+DSN_DECLARE_string(meta_state_service_type);
static const std::vector<std::string> keys = {"manual_compact.once.trigger_time",
"manual_compact.once.target_level",
@@ -89,8 +91,8 @@ void meta_service_test_app::app_envs_basic_test()
std::shared_ptr<meta_service> meta_svc = std::make_shared<meta_service>();
meta_service *svc = meta_svc.get();
- svc->_meta_opts.cluster_root = "/meta_test";
- svc->_meta_opts.meta_state_service_type = "meta_state_service_simple";
+ FLAGS_cluster_root = "/meta_test";
+ FLAGS_meta_state_service_type = "meta_state_service_simple";
svc->remote_storage_initialize();
std::string apps_root = "/meta_test/apps";
diff --git a/src/meta/test/state_sync_test.cpp b/src/meta/test/state_sync_test.cpp
index a30c635d9..00b35fb36 100644
--- a/src/meta/test/state_sync_test.cpp
+++ b/src/meta/test/state_sync_test.cpp
@@ -52,6 +52,8 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_string(cluster_root);
+DSN_DECLARE_string(meta_state_service_type);
static void random_assign_partition_config(std::shared_ptr<app_state> &app,
const std::vector<dsn::rpc_address> &server_list,
@@ -119,8 +121,8 @@ void meta_service_test_app::state_sync_test()
std::shared_ptr<meta_service> meta_svc = std::make_shared<meta_service>();
meta_service *svc = meta_svc.get();
meta_options &opt = svc->_meta_opts;
- opt.cluster_root = "/meta_test";
- opt.meta_state_service_type = "meta_state_service_simple";
+ FLAGS_cluster_root = "/meta_test";
+ FLAGS_meta_state_service_type = "meta_state_service_simple";
svc->remote_storage_initialize();
std::string apps_root = "/meta_test/apps";
@@ -197,7 +199,7 @@ void meta_service_test_app::state_sync_test()
file_data_compare("meta_state.dump1", "meta_state.dump2");
}
- opt.meta_state_service_type = "meta_state_service_zookeeper";
+ FLAGS_meta_state_service_type = "meta_state_service_zookeeper";
svc->remote_storage_initialize();
// first clean up
std::cerr << "start to clean up zookeeper storage" << std::endl;
diff --git a/src/replica/backup/replica_backup_server.cpp b/src/replica/backup/replica_backup_server.cpp
index a8d604011..c4c218f99 100644
--- a/src/replica/backup/replica_backup_server.cpp
+++ b/src/replica/backup/replica_backup_server.cpp
@@ -22,6 +22,7 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_string(cold_backup_root);
replica_backup_server::replica_backup_server(const replica_stub *rs) : _stub(rs)
{
@@ -48,11 +49,12 @@ void replica_backup_server::on_cold_backup(backup_rpc rpc)
response.policy_name = request.policy.policy_name;
response.backup_id = request.backup_id;
- if (_stub->options().cold_backup_root.empty()) {
- LOG_ERROR("backup[{}.{}.{}]: cold_backup_root is empty, response ERR_OPERATION_DISABLED",
- request.pid,
- request.policy.policy_name,
- request.backup_id);
+ if (utils::is_empty(FLAGS_cold_backup_root)) {
+ LOG_ERROR(
+ "backup[{}.{}.{}]: FLAGS_cold_backup_root is empty, response ERR_OPERATION_DISABLED",
+ request.pid,
+ request.policy.policy_name,
+ request.backup_id);
response.err = ERR_OPERATION_DISABLED;
return;
}
diff --git a/src/replica/replica_backup.cpp b/src/replica/replica_backup.cpp
index c15c78ed9..02b942b55 100644
--- a/src/replica/replica_backup.cpp
+++ b/src/replica/replica_backup.cpp
@@ -38,6 +38,8 @@ DSN_DEFINE_uint64(replication,
10,
"concurrent uploading file count to block service");
+DSN_DECLARE_string(cold_backup_root);
+
void replica::on_cold_backup(const backup_request &request, /*out*/ backup_response &response)
{
_checker.only_one_thread_access();
@@ -77,8 +79,8 @@ void replica::on_cold_backup(const backup_request &request, /*out*/ backup_respo
backup_context->block_service = block_service;
backup_context->backup_root = request.__isset.backup_path
? dsn::utils::filesystem::path_combine(
- request.backup_path, _options->cold_backup_root)
- : _options->cold_backup_root;
+ request.backup_path, FLAGS_cold_backup_root)
+ : FLAGS_cold_backup_root;
}
CHECK_EQ_PREFIX(backup_context->request.policy.policy_name, policy_name);
diff --git a/src/replica/storage/simple_kv/test/checker.cpp b/src/replica/storage/simple_kv/test/checker.cpp
index 1514a5b33..10fcef7d4 100644
--- a/src/replica/storage/simple_kv/test/checker.cpp
+++ b/src/replica/storage/simple_kv/test/checker.cpp
@@ -54,6 +54,8 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_string(partition_guardian_type);
+
namespace test {
class checker_partition_guardian : public partition_guardian
@@ -174,7 +176,7 @@ bool test_checker::init(const std::string &name, const std::vector<service_app *
meta_service_app *meta_app = (meta_service_app *)app;
meta_app->_service->_state->set_config_change_subscriber_for_test(
std::bind(&test_checker::on_config_change, this, std::placeholders::_1));
- meta_app->_service->_meta_opts.partition_guardian_type = "checker_partition_guardian";
+ FLAGS_partition_guardian_type = "checker_partition_guardian";
_meta_servers.push_back(meta_app);
} else if (app->info().type == "replica") {
replication_service_app *replica_app = (replication_service_app *)app;
diff --git a/src/replica/test/replica_test.cpp b/src/replica/test/replica_test.cpp
index 3aec5176b..545dff489 100644
--- a/src/replica/test/replica_test.cpp
+++ b/src/replica/test/replica_test.cpp
@@ -28,6 +28,7 @@
namespace dsn {
namespace replication {
+DSN_DECLARE_string(cold_backup_root);
class replica_test : public replica_test_base
{
@@ -47,10 +48,10 @@ public:
mock_app_info();
_mock_replica = stub->generate_replica_ptr(_app_info, pid, partition_status::PS_PRIMARY, 1);
- // set cold_backup_root manually.
- // `cold_backup_root` is set by configuration "replication.cold_backup_root",
+ // set FLAGS_cold_backup_root manually.
+ // FLAGS_cold_backup_root is set by configuration "replication.cold_backup_root",
// which is usually the cluster_name of production clusters.
- _mock_replica->_options->cold_backup_root = "test_cluster";
+ FLAGS_cold_backup_root = "test_cluster";
}
int get_write_size_exceed_threshold_count()
@@ -134,8 +135,8 @@ public:
ASSERT_EQ(ERR_OK, resp.err);
// test checkpoint files have been uploaded successfully.
- std::string backup_root = dsn::utils::filesystem::path_combine(
- user_specified_path, _mock_replica->_options->cold_backup_root);
+ std::string backup_root =
+ dsn::utils::filesystem::path_combine(user_specified_path, FLAGS_cold_backup_root);
std::string current_chkpt_file =
cold_backup::get_current_chkpt_file(backup_root, req.app_name, req.pid, req.backup_id);
ASSERT_TRUE(dsn::utils::filesystem::file_exists(current_chkpt_file));
@@ -150,7 +151,7 @@ public:
req.app_id = _app_info.app_id;
req.app_name = _app_info.app_name;
req.backup_provider_name = _provider_name;
- req.cluster_name = _mock_replica->_options->cold_backup_root;
+ req.cluster_name = FLAGS_cold_backup_root;
req.time_stamp = _backup_id;
if (!user_specified_path.empty()) {
req.__set_restore_path(user_specified_path);
diff --git a/src/runtime/rpc/network.cpp b/src/runtime/rpc/network.cpp
index 83ee02266..e09cbe6ad 100644
--- a/src/runtime/rpc/network.cpp
+++ b/src/runtime/rpc/network.cpp
@@ -39,6 +39,17 @@ DSN_DEFINE_uint32(network,
conn_threshold_per_ip,
0,
"max connection count to each server per ip, 0 means no limit");
+DSN_DEFINE_string(network, unknown_message_header_format, "", "format for unknown message headers");
+DSN_DEFINE_string(network,
+ explicit_host_address,
+ "",
+ "explicit host name or ip (v4) assigned to this node (e.g., "
+ "service ip for pods in kubernets)");
+DSN_DEFINE_string(network,
+ primary_interface,
+ "",
+ "network interface name used to init primary ipv4 address, "
+ "if empty, means using a site local address");
/*static*/ join_point<void, rpc_session *>
rpc_session::on_rpc_session_connected("rpc.session.connected");
@@ -511,13 +522,8 @@ network::network(rpc_engine *srv, network *inner_provider)
{
_message_buffer_block_size = 1024 * 64;
_max_buffer_block_count_per_send = 64; // TODO: windows, how about the other platforms?
- _unknown_msg_header_format = network_header_format::from_string(
- dsn_config_get_value_string(
- "network",
- "unknown_message_header_format",
- NET_HDR_INVALID.to_string(),
- "format for unknown message headers, default is NET_HDR_INVALID"),
- NET_HDR_INVALID);
+ _unknown_msg_header_format =
+ network_header_format::from_string(FLAGS_unknown_message_header_format, NET_HDR_INVALID);
}
void network::reset_parser_attr(network_header_format client_hdr_format,
@@ -548,28 +554,13 @@ message_parser *network::new_message_parser(network_header_format hdr_format)
uint32_t network::get_local_ipv4()
{
- static const char *explicit_host =
- dsn_config_get_value_string("network",
- "explicit_host_address",
- "",
- "explicit host name or ip (v4) assigned to this "
- "node (e.g., service ip for pods in kubernets)");
-
- static const char *inteface =
- dsn_config_get_value_string("network",
- "primary_interface",
- "",
- "network interface name used to init primary ipv4 "
- "address, if empty, means using a site local address");
-
uint32_t ip = 0;
-
- if (!utils::is_empty(explicit_host)) {
- ip = rpc_address::ipv4_from_host(explicit_host);
+ if (!utils::is_empty(FLAGS_explicit_host_address)) {
+ ip = rpc_address::ipv4_from_host(FLAGS_explicit_host_address);
}
if (0 == ip) {
- ip = rpc_address::ipv4_from_network_interface(inteface);
+ ip = rpc_address::ipv4_from_network_interface(FLAGS_primary_interface);
}
if (0 == ip) {
diff --git a/src/runtime/test/main.cpp b/src/runtime/test/main.cpp
index d7ec9d2e9..5ef9fe347 100644
--- a/src/runtime/test/main.cpp
+++ b/src/runtime/test/main.cpp
@@ -38,6 +38,9 @@
#include "test_utils.h"
#include "utils/strings.h"
+#include "utils/flags.h"
+
+DSN_DEFINE_string(core, tool, "simulator", "");
int g_test_count = 0;
int g_test_ret = 0;
@@ -64,8 +67,7 @@ GTEST_API_ int main(int argc, char **argv)
return g_test_ret;
}
- if (!dsn::utils::equals("simulator",
- dsn_config_get_value_string("core", "tool", "simulator", ""))) {
+ if (!dsn::utils::equals("simulator", FLAGS_tool)) {
// run out-rDSN tests in other threads
std::cout << "=========================================================== " << std::endl;
std::cout << "================== run in non-rDSN threads ================ " << std::endl;
diff --git a/src/runtime/test/service_api_c.cpp b/src/runtime/test/service_api_c.cpp
index 01ceffb3c..29c283992 100644
--- a/src/runtime/test/service_api_c.cpp
+++ b/src/runtime/test/service_api_c.cpp
@@ -50,6 +50,9 @@
#include <thread>
#include "utils/rand.h"
#include "runtime/service_engine.h"
+#include "utils/flags.h"
+
+DSN_DECLARE_string(tool);
using namespace dsn;
@@ -219,7 +222,7 @@ TEST(core, dsn_system)
{
ASSERT_TRUE(tools::is_engine_ready());
tools::tool_app *tool = tools::get_current_tool();
- ASSERT_EQ(tool->name(), dsn_config_get_value_string("core", "tool", "", ""));
+ ASSERT_EQ(tool->name(), FLAGS_tool);
int app_count = 5;
int type_count = 1;
diff --git a/src/server/available_detector.cpp b/src/server/available_detector.cpp
index 6f2b83412..a94ce7c97 100644
--- a/src/server/available_detector.cpp
+++ b/src/server/available_detector.cpp
@@ -46,6 +46,20 @@ DSN_DEFINE_uint32(pegasus.collector,
available_detect_timeout,
1000,
"available detect timeout in millisecond");
+DSN_DEFINE_string(pegasus.collector, available_detect_app, "", "available detector app name");
+DSN_DEFINE_validator(available_detect_app,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+DSN_DEFINE_string(pegasus.collector,
+ available_detect_alert_script_dir,
+ ".",
+ "available detect alert script dir");
+DSN_DEFINE_validator(available_detect_alert_script_dir,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+DSN_DEFINE_string(pegasus.collector,
+ available_detect_alert_email_address,
+ "",
+ "available detect alert email address, empty means not send email");
+
available_detector::available_detector()
: _client(nullptr),
_ddl_client(nullptr),
@@ -60,19 +74,6 @@ available_detector::available_detector()
{
// initialize information for available_detector.
_cluster_name = dsn::get_current_cluster_name();
- _app_name = dsn_config_get_value_string(
- "pegasus.collector", "available_detect_app", "", "available detector app name");
- CHECK(!_app_name.empty(), "");
- _alert_script_dir = dsn_config_get_value_string("pegasus.collector",
- "available_detect_alert_script_dir",
- ".",
- "available detect alert script dir");
- CHECK(!_alert_script_dir.empty(), "");
- _alert_email_address = dsn_config_get_value_string(
- "pegasus.collector",
- "available_detect_alert_email_address",
- "",
- "available detect alert email address, empty means not send email");
_meta_list.clear();
dsn::replication::replica_helper::load_meta_servers(_meta_list);
CHECK(!_meta_list.empty(), "");
@@ -80,17 +81,20 @@ available_detector::available_detector()
if (!pegasus_client_factory::initialize(nullptr)) {
CHECK(false, "Initialize the pegasus client failed");
}
- _client = pegasus_client_factory::get_client(_cluster_name.c_str(), _app_name.c_str());
+ _client = pegasus_client_factory::get_client(_cluster_name.c_str(), FLAGS_available_detect_app);
CHECK_NOTNULL(_client, "Initialize the _client failed");
_result_writer = dsn::make_unique<result_writer>(_client);
_ddl_client.reset(new replication_ddl_client(_meta_list));
CHECK_NOTNULL(_ddl_client, "Initialize the _ddl_client failed");
- if (!_alert_email_address.empty()) {
- _send_alert_email_cmd = "cd " + _alert_script_dir + "; bash sendmail.sh alert " +
- _alert_email_address + " " + _cluster_name + " " + _app_name + " ";
- _send_availability_info_email_cmd = "cd " + _alert_script_dir +
- "; bash sendmail.sh availability_info " +
- _alert_email_address + " " + _cluster_name + " ";
+ if (!dsn::utils::is_empty(FLAGS_available_detect_alert_email_address)) {
+ _send_alert_email_cmd = std::string("cd ") + FLAGS_available_detect_alert_script_dir +
+ "; bash sendmail.sh alert " +
+ FLAGS_available_detect_alert_email_address + " " + _cluster_name +
+ " " + FLAGS_available_detect_app + " ";
+ _send_availability_info_email_cmd =
+ std::string("cd ") + FLAGS_available_detect_alert_script_dir +
+ "; bash sendmail.sh availability_info " + FLAGS_available_detect_alert_email_address +
+ " " + _cluster_name + " ";
}
_pfc_detect_times_day.init_app_counter("app.pegasus",
@@ -238,7 +242,8 @@ void available_detector::report_availability_info()
bool available_detector::generate_hash_keys()
{
// get app_id and partition_count.
- auto err = _ddl_client->list_app(_app_name, _app_id, _partition_count, partitions);
+ auto err =
+ _ddl_client->list_app(FLAGS_available_detect_app, _app_id, _partition_count, partitions);
if (err == ::dsn::ERR_OK && _app_id >= 0) {
_hash_keys.clear();
for (auto pidx = 0; pidx < _partition_count; pidx++) {
@@ -259,8 +264,9 @@ bool available_detector::generate_hash_keys()
}
return true;
} else {
- LOG_WARNING(
- "Get partition count of table '{}' on cluster '{}' failed", _app_name, _cluster_name);
+ LOG_WARNING("Get partition count of table '{}' on cluster '{}' failed",
+ FLAGS_available_detect_app,
+ _cluster_name);
return false;
}
}
@@ -272,7 +278,7 @@ void available_detector::on_detect(int32_t idx)
"recent_day_detect_times({}), recent_day_fail_times({}), "
"recent_hour_detect_times({}), recent_hour_fail_times({}) "
"recent_minute_detect_times({}), recent_minute_fail_times({})",
- _app_name,
+ FLAGS_available_detect_app,
_app_id,
_partition_count,
_cluster_name,
@@ -286,7 +292,7 @@ void available_detector::on_detect(int32_t idx)
LOG_DEBUG("available_detector begin to detect partition[{}] of table[{}] with id[{}] on the "
"cluster[{}]",
idx,
- _app_name,
+ FLAGS_available_detect_app,
_app_id,
_cluster_name);
auto time = dsn_now_ms();
diff --git a/src/server/available_detector.h b/src/server/available_detector.h
index ea7621c5a..8b18a4b40 100644
--- a/src/server/available_detector.h
+++ b/src/server/available_detector.h
@@ -54,7 +54,6 @@ private:
private:
dsn::task_tracker _tracker;
std::string _cluster_name;
- std::string _app_name;
// for writing detect result
std::unique_ptr<result_writer> _result_writer;
// client to access server.
@@ -74,8 +73,6 @@ private:
std::string _send_alert_email_cmd;
std::string _send_availability_info_email_cmd;
- std::string _alert_script_dir;
- std::string _alert_email_address;
// total detect times and total fail times
std::atomic<int64_t> _recent_day_detect_times;
diff --git a/src/server/info_collector.cpp b/src/server/info_collector.cpp
index c802cb723..eaf9a246d 100644
--- a/src/server/info_collector.cpp
+++ b/src/server/info_collector.cpp
@@ -54,6 +54,13 @@ DSN_DEFINE_uint32(pegasus.collector,
storage_size_fetch_interval_seconds,
3600,
"storage size fetch interval seconds");
+DSN_DEFINE_string(pegasus.collector,
+ usage_stat_app,
+ "",
+ "app for recording usage statistics, including read/write capacity unit and "
+ "storage size");
+DSN_DEFINE_validator(usage_stat_app,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
info_collector::info_collector()
{
@@ -72,12 +79,9 @@ info_collector::info_collector()
_shell_context->meta_list = meta_servers;
_shell_context->ddl_client.reset(new replication_ddl_client(meta_servers));
- _usage_stat_app = dsn_config_get_value_string(
- "pegasus.collector", "usage_stat_app", "", "app for recording usage statistics");
- CHECK(!_usage_stat_app.empty(), "");
// initialize the _client.
CHECK(pegasus_client_factory::initialize(nullptr), "Initialize the pegasus client failed");
- _client = pegasus_client_factory::get_client(_cluster_name.c_str(), _usage_stat_app.c_str());
+ _client = pegasus_client_factory::get_client(_cluster_name.c_str(), FLAGS_usage_stat_app);
CHECK_NOTNULL(_client, "Initialize the client failed");
_result_writer = dsn::make_unique<result_writer>(_client);
diff --git a/src/server/info_collector.h b/src/server/info_collector.h
index 2d9240abc..b320f6a99 100644
--- a/src/server/info_collector.h
+++ b/src/server/info_collector.h
@@ -229,8 +229,6 @@ private:
::dsn::utils::ex_lock_nr _app_stat_counter_lock;
std::map<std::string, app_stat_counters *> _app_stat_counters;
- // app for recording usage statistics, including read/write capacity unit and storage size.
- std::string _usage_stat_app;
// client to access server.
pegasus_client *_client;
// for writing cu stat result
diff --git a/src/server/pegasus_server_impl_init.cpp b/src/server/pegasus_server_impl_init.cpp
index 1d407ae1b..29e5180f3 100644
--- a/src/server/pegasus_server_impl_init.cpp
+++ b/src/server/pegasus_server_impl_init.cpp
@@ -333,6 +333,25 @@ DSN_DEFINE_double(pegasus.server,
rocksdb_bloom_filter_bits_per_key,
10,
"average bits allocated per key in bloom filter");
+DSN_DEFINE_string(pegasus.server,
+ rocksdb_compression_type,
+ "lz4",
+ "rocksdb options.compression. Available config: '[none|snappy|zstd|lz4]' for all "
+ "level 2 and higher levels, and "
+ "'per_level:[none|snappy|zstd|lz4],[none|snappy|zstd|lz4],...' for each level "
+ "0,1,..., the last compression type will be used for levels not specified in the "
+ "list.");
+DSN_DEFINE_string(pegasus.server,
+ rocksdb_index_type,
+ "binary_search",
+ "The index type that will be used for this table.");
+DSN_DEFINE_string(pegasus.server,
+ rocksdb_filter_type,
+ "prefix",
+ "Bloom filter type, should be either 'common' or 'prefix'");
+DSN_DEFINE_validator(rocksdb_filter_type, [](const char *value) -> bool {
+ return dsn::utils::equals(value, "common") || dsn::utils::equals(value, "prefix");
+});
static const std::unordered_map<std::string, rocksdb::BlockBasedTableOptions::IndexType>
INDEX_TYPE_STRING_MAP = {
@@ -405,15 +424,8 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
_data_cf_opts.level0_slowdown_writes_trigger = FLAGS_rocksdb_level0_slowdown_writes_trigger;
_data_cf_opts.level0_stop_writes_trigger = FLAGS_rocksdb_level0_stop_writes_trigger;
- std::string compression_str = dsn_config_get_value_string(
- "pegasus.server",
- "rocksdb_compression_type",
- "lz4",
- "rocksdb options.compression. Available config: '[none|snappy|zstd|lz4]' "
- "for all level 2 and higher levels, and "
- "'per_level:[none|snappy|zstd|lz4],[none|snappy|zstd|lz4],...' for each level 0,1,..., the "
- "last compression type will be used for levels not specified in the list.");
- CHECK(parse_compression_types(compression_str, _data_cf_opts.compression_per_level),
+ CHECK(parse_compression_types(FLAGS_rocksdb_compression_type,
+ _data_cf_opts.compression_per_level),
"parse rocksdb_compression_type failed.");
_meta_cf_opts = _data_cf_opts;
@@ -494,17 +506,12 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
_db_opts.keep_log_file_num = static_cast<size_t>(FLAGS_rocksdb_keep_log_file_num);
LOG_INFO_PREFIX("rocksdb_keep_log_file_num = {}", _db_opts.keep_log_file_num);
- std::string index_type =
- dsn_config_get_value_string("pegasus.server",
- "rocksdb_index_type",
- "binary_search",
- "The index type that will be used for this table.");
- auto index_type_item = INDEX_TYPE_STRING_MAP.find(index_type);
+ auto index_type_item = INDEX_TYPE_STRING_MAP.find(FLAGS_rocksdb_index_type);
CHECK(index_type_item != INDEX_TYPE_STRING_MAP.end(),
"[pegasus.server]rocksdb_index_type should be one among binary_search, "
"hash_search, two_level_index_search or binary_search_with_first_key.");
tbl_opts.index_type = index_type_item->second;
- LOG_INFO_PREFIX("rocksdb_index_type = {}", index_type.c_str());
+ LOG_INFO_PREFIX("rocksdb_index_type = {}", FLAGS_rocksdb_index_type);
tbl_opts.partition_filters = FLAGS_rocksdb_partition_filters;
// TODO(yingchun): clean up these useless log ?
@@ -550,14 +557,7 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
tbl_opts.filter_policy.reset(
rocksdb::NewBloomFilterPolicy(FLAGS_rocksdb_bloom_filter_bits_per_key, false));
- std::string filter_type =
- dsn_config_get_value_string("pegasus.server",
- "rocksdb_filter_type",
- "prefix",
- "Bloom filter type, should be either 'common' or 'prefix'");
- CHECK(filter_type == "common" || filter_type == "prefix",
- "[pegasus.server]rocksdb_filter_type should be either 'common' or 'prefix'.");
- if (filter_type == "prefix") {
+ if (dsn::utils::equals(FLAGS_rocksdb_filter_type, "prefix")) {
_data_cf_opts.prefix_extractor.reset(new HashkeyTransform());
_data_cf_opts.memtable_prefix_bloom_size_ratio = 0.1;
diff --git a/src/test/bench_test/benchmark.cpp b/src/test/bench_test/benchmark.cpp
index 843bd21c1..0a33c0350 100644
--- a/src/test/bench_test/benchmark.cpp
+++ b/src/test/bench_test/benchmark.cpp
@@ -19,6 +19,7 @@
#include "benchmark.h"
+#include <cstring>
#include <sstream>
#include "rand.h"
@@ -27,6 +28,7 @@
#include "utils/flags.h"
#include "utils/fmt_logging.h"
#include "utils/ports.h"
+#include "utils/strings.h"
namespace pegasus {
namespace test {
@@ -39,6 +41,22 @@ DSN_DEFINE_uint64(pegasus.benchmark,
benchmark_seed,
1000,
"Seed base for random number generators. When 0 it is deterministic");
+DSN_DEFINE_string(pegasus.benchmark, pegasus_cluster_name, "onebox", "The Pegasus cluster name");
+DSN_DEFINE_validator(pegasus_cluster_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+DSN_DEFINE_string(pegasus.benchmark, pegasus_app_name, "temp", "pegasus app name");
+DSN_DEFINE_validator(pegasus_app_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+DSN_DEFINE_string(
+ pegasus.benchmark,
+ benchmarks,
+ "fillrandom_pegasus,readrandom_pegasus,deleterandom_pegasus",
+ "Comma-separated list of operations to run in the specified order. Available benchmarks:\n"
+ "\tfillrandom_pegasus -- pegasus write N values in random key order\n"
+ "\treadrandom_pegasus -- pegasus read N times in random order\n"
+ "\tdeleterandom_pegasus -- pegasus delete N keys in random order\n");
+DSN_DEFINE_validator(benchmarks,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
DSN_DECLARE_int32(hashkey_size);
DSN_DECLARE_int32(pegasus_timeout_ms);
@@ -48,8 +66,8 @@ DSN_DECLARE_int32(value_size);
benchmark::benchmark()
{
- _client = pegasus_client_factory::get_client(config::instance().pegasus_cluster_name.c_str(),
- config::instance().pegasus_app_name.c_str());
+ _client =
+ pegasus_client_factory::get_client(FLAGS_pegasus_cluster_name, FLAGS_pegasus_app_name);
CHECK_NOTNULL(_client, "");
// init operation method map
@@ -64,7 +82,7 @@ void benchmark::run()
// print summarize information
print_header();
- std::stringstream benchmark_stream(config::instance().benchmarks);
+ std::stringstream benchmark_stream(FLAGS_benchmarks);
std::string name;
while (std::getline(benchmark_stream, name, ',')) {
// run the specified benchmark
diff --git a/src/test/bench_test/config.cpp b/src/test/bench_test/config.cpp
index 055e953c9..2383e2d2b 100644
--- a/src/test/bench_test/config.cpp
+++ b/src/test/bench_test/config.cpp
@@ -34,21 +34,6 @@ DSN_DEFINE_int32(pegasus.benchmark, hashkey_size, 16, "size of each hashkey");
DSN_DEFINE_int32(pegasus.benchmark, sortkey_size, 16, "size of each sortkey");
DSN_DEFINE_int32(pegasus.benchmark, value_size, 100, "Size of each value");
-config::config()
-{
- pegasus_cluster_name = dsn_config_get_value_string(
- "pegasus.benchmark", "pegasus_cluster_name", "onebox", "pegasus cluster name");
- pegasus_app_name = dsn_config_get_value_string(
- "pegasus.benchmark", "pegasus_app_name", "temp", "pegasus app name");
- benchmarks = dsn_config_get_value_string(
- "pegasus.benchmark",
- "benchmarks",
- "fillrandom_pegasus,readrandom_pegasus,deleterandom_pegasus",
- "Comma-separated list of operations to run in the specified order. Available benchmarks:\n"
- "\tfillrandom_pegasus -- pegasus write N values in random key order\n"
- "\treadrandom_pegasus -- pegasus read N times in random order\n"
- "\tdeleterandom_pegasus -- pegasus delete N keys in random order\n");
- env = rocksdb::Env::Default();
-}
+config::config() { env = rocksdb::Env::Default(); }
} // namespace test
} // namespace pegasus
diff --git a/src/test/bench_test/config.h b/src/test/bench_test/config.h
index 1b267dd18..1a8309971 100644
--- a/src/test/bench_test/config.h
+++ b/src/test/bench_test/config.h
@@ -28,8 +28,6 @@ namespace test {
/** Thread safety singleton */
struct config : public dsn::utils::singleton<config>
{
- std::string pegasus_cluster_name;
- std::string pegasus_app_name;
// Comma-separated list of operations to run
std::string benchmarks;
// Default environment suitable for the current operating system
diff --git a/src/test/function_test/recovery_test/test_recovery.cpp b/src/test/function_test/recovery_test/test_recovery.cpp
index 256ae4452..e9fc3de59 100644
--- a/src/test/function_test/recovery_test/test_recovery.cpp
+++ b/src/test/function_test/recovery_test/test_recovery.cpp
@@ -45,7 +45,7 @@ using namespace pegasus;
// TODO(yingchun): add a check for it, get config by curl
// NOTE: THREAD_POOL_META_SERVER worker count should be greater than 1
-// This function test update 'distributed_lock_service_type' to
+// This function test update FLAGS_distributed_lock_service_type to
// 'distributed_lock_service_simple', which executes in threadpool THREAD_POOL_META_SERVER
// As a result, failure detection lock executes in this pool
// if worker count = 1, it will lead to ERR_TIMEOUT when execute 'ddl_client_->do_recovery'
diff --git a/src/test/kill_test/data_verifier.cpp b/src/test/kill_test/data_verifier.cpp
index 6fcc22470..2bed38c4d 100644
--- a/src/test/kill_test/data_verifier.cpp
+++ b/src/test/kill_test/data_verifier.cpp
@@ -25,6 +25,7 @@
#include <unistd.h>
#include <chrono>
#include <thread>
+#include <cstring>
#include <atomic>
#include <memory>
#include <sys/time.h>
@@ -48,12 +49,10 @@
#include "utils/fmt_logging.h"
#include "utils/flags.h"
-using namespace std;
-using namespace ::pegasus;
+namespace pegasus {
+namespace test {
static pegasus_client *client = nullptr;
-static string app_name;
-static string pegasus_cluster_name;
static std::atomic_llong set_next(0);
static std::vector<long long> set_thread_setting_id;
@@ -80,6 +79,12 @@ DSN_DEFINE_uint32(pegasus.killtest,
get_thread_count,
FLAGS_set_thread_count * 4,
"Thread count of the getter.");
+DSN_DEFINE_string(pegasus.killtest, pegasus_cluster_name, "onebox", "The Pegasus cluster name");
+DSN_DEFINE_validator(pegasus_cluster_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+DSN_DEFINE_string(pegasus.killtest, verify_app_name, "temp", "verify app name");
+DSN_DEFINE_validator(verify_app_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
// return time in us.
long get_time()
@@ -382,20 +387,11 @@ void do_mark()
void verifier_initialize(const char *config_file)
{
- const char *section = "pegasus.killtest";
if (!pegasus_client_factory::initialize(config_file)) {
exit(-1);
}
- app_name = dsn_config_get_value_string(
- section, "verify_app_name", "temp", "verify app name"); // default using temp
- pegasus_cluster_name =
- dsn_config_get_value_string(section, "pegasus_cluster_name", "", "pegasus cluster name");
- if (pegasus_cluster_name.empty()) {
- LOG_ERROR("Should config the cluster name for verifier");
- exit(-1);
- }
- client = pegasus_client_factory::get_client(pegasus_cluster_name.c_str(), app_name.c_str());
+ client = pegasus_client_factory::get_client(FLAGS_pegasus_cluster_name, FLAGS_verify_app_name);
if (client == nullptr) {
LOG_ERROR("Initialize the _client failed");
exit(-1);
@@ -446,3 +442,5 @@ void verifier_start()
t.join();
}
}
+} // namespace test
+} // namespace pegasus
diff --git a/src/test/kill_test/data_verifier.h b/src/test/kill_test/data_verifier.h
index 7ec482d0b..3927ff1b2 100644
--- a/src/test/kill_test/data_verifier.h
+++ b/src/test/kill_test/data_verifier.h
@@ -19,5 +19,9 @@
#pragma once
+namespace pegasus {
+namespace test {
void verifier_initialize(const char *config_file);
void verifier_start();
+} // namespace test
+} // namespace pegasus
diff --git a/src/test/kill_test/kill_testor.cpp b/src/test/kill_test/kill_testor.cpp
index ac920e5dd..6fbfc315a 100644
--- a/src/test/kill_test/kill_testor.cpp
+++ b/src/test/kill_test/kill_testor.cpp
@@ -47,37 +47,30 @@ namespace test {
DSN_DEFINE_uint32(pegasus.killtest, kill_interval_seconds, 30, "");
DSN_DEFINE_uint32(pegasus.killtest, max_seconds_for_all_partitions_to_recover, 600, "");
+DSN_DECLARE_string(pegasus_cluster_name);
+DSN_DECLARE_string(verify_app_name);
+
kill_testor::kill_testor(const char *config_file)
{
- const char *section = "pegasus.killtest";
// initialize the _client.
if (!pegasus_client_factory::initialize(config_file)) {
exit(-1);
}
- app_name = dsn_config_get_value_string(
- section, "verify_app_name", "temp", "verify app name"); // default using temp
- pegasus_cluster_name =
- dsn_config_get_value_string(section, "pegasus_cluster_name", "", "pegasus cluster name");
- if (pegasus_cluster_name.empty()) {
- LOG_ERROR("Should config the cluster name for killer");
- exit(-1);
- }
-
// load meta_list
- meta_list.clear();
dsn::replication::replica_helper::load_meta_servers(
- meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), pegasus_cluster_name.c_str());
+ meta_list, PEGASUS_CLUSTER_SECTION_NAME.c_str(), FLAGS_pegasus_cluster_name);
if (meta_list.empty()) {
LOG_ERROR("Should config the meta address for killer");
exit(-1);
}
ddl_client.reset(new replication_ddl_client(meta_list));
- if (ddl_client == nullptr) {
+ if (!ddl_client) {
LOG_ERROR("Initialize the _ddl_client failed");
exit(-1);
}
+
srand((unsigned)time(nullptr));
}
@@ -116,7 +109,8 @@ dsn::error_code kill_testor::get_partition_info(bool debug_unhealthy,
int32_t app_id;
int32_t partition_count;
partitions.clear();
- dsn::error_code err = ddl_client->list_app(app_name, app_id, partition_count, partitions);
+ dsn::error_code err =
+ ddl_client->list_app(FLAGS_verify_app_name, app_id, partition_count, partitions);
if (err == ::dsn::ERR_OK) {
LOG_DEBUG("access meta and query partition status success");
diff --git a/src/test/kill_test/kill_testor.h b/src/test/kill_test/kill_testor.h
index 0ce6cf364..74b3efe0c 100644
--- a/src/test/kill_test/kill_testor.h
+++ b/src/test/kill_test/kill_testor.h
@@ -58,8 +58,6 @@ protected:
protected:
shared_ptr<replication_ddl_client> ddl_client;
- string app_name;
- string pegasus_cluster_name;
vector<dsn::rpc_address> meta_list;
std::vector<partition_configuration> partitions;
diff --git a/src/test/kill_test/killer_handler_shell.cpp b/src/test/kill_test/killer_handler_shell.cpp
index 2d59b746c..5e4c2ec8c 100644
--- a/src/test/kill_test/killer_handler_shell.cpp
+++ b/src/test/kill_test/killer_handler_shell.cpp
@@ -29,26 +29,21 @@
#include "utils/config_api.h"
#include "utils/fmt_logging.h"
#include "utils/safe_strerror_posix.h"
+#include "utils/flags.h"
+#include "utils/strings.h"
namespace pegasus {
namespace test {
-killer_handler_shell::killer_handler_shell()
-{
- const char *section = "killer.handler.shell";
- _run_script_path = dsn_config_get_value_string(
- section, "onebox_run_path", "~/pegasus/run.sh", "onebox run path");
- CHECK(!_run_script_path.empty(), "");
-}
+DSN_DEFINE_string(killer.handler.shell, onebox_run_path, "~/pegasus/run.sh", "onebox run path");
+DSN_DEFINE_validator(onebox_run_path,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
bool killer_handler_shell::has_meta_dumped_core(int index)
{
char find_core[1024];
- snprintf(find_core,
- 1024,
- "ls %s/onebox/meta%d | grep core | wc -l",
- _run_script_path.c_str(),
- index);
+ snprintf(
+ find_core, 1024, "ls %s/onebox/meta%d | grep core | wc -l", FLAGS_onebox_run_path, index);
std::stringstream output;
int core_count;
@@ -64,7 +59,7 @@ bool killer_handler_shell::has_replica_dumped_core(int index)
snprintf(find_core,
1024,
"ls %s/onebox/replica%d | grep core | wc -l",
- _run_script_path.c_str(),
+ FLAGS_onebox_run_path,
index);
std::stringstream output;
@@ -176,7 +171,7 @@ std::string
killer_handler_shell::generate_cmd(int index, const std::string &job, const std::string &action)
{
std::stringstream res;
- res << "cd " << _run_script_path << "; "
+ res << "cd " << FLAGS_onebox_run_path << "; "
<< "bash run.sh";
if (action == "stop")
res << " stop_onebox_instance ";
diff --git a/src/test/kill_test/killer_handler_shell.h b/src/test/kill_test/killer_handler_shell.h
index 06bba28f5..174e351a7 100644
--- a/src/test/kill_test/killer_handler_shell.h
+++ b/src/test/kill_test/killer_handler_shell.h
@@ -29,7 +29,7 @@ namespace test {
class killer_handler_shell : public killer_handler
{
public:
- killer_handler_shell();
+ killer_handler_shell() = default;
virtual ~killer_handler_shell() {}
// index begin from 1, not zero
// kill one
@@ -57,10 +57,6 @@ private:
std::string generate_cmd(int index, const std::string &job, const std::string &action);
// check whether the command execute success.
bool check(const std::string &job, int index, const std::string &type);
-
-private:
- // using ${_run_script_path}/run.sh to kill/start
- std::string _run_script_path;
};
}
} // end namespace
diff --git a/src/test/kill_test/main.cpp b/src/test/kill_test/main.cpp
index fc7299ad4..21c319669 100644
--- a/src/test/kill_test/main.cpp
+++ b/src/test/kill_test/main.cpp
@@ -32,8 +32,8 @@ int main(int argc, const char **argv)
"worker_type(verifier|process_killer|partition_killer)\n");
return -1;
} else if (dsn::utils::equals(argv[2], "verifier")) {
- verifier_initialize(argv[1]);
- verifier_start();
+ pegasus::test::verifier_initialize(argv[1]);
+ pegasus::test::verifier_start();
} else if (dsn::utils::equals(argv[2], "process_killer")) {
pegasus::test::kill_testor *killtestor = new pegasus::test::process_kill_testor(argv[1]);
killtestor->Run();
diff --git a/src/test/kill_test/process_kill_testor.cpp b/src/test/kill_test/process_kill_testor.cpp
index 8dd25a76b..80b5819a6 100644
--- a/src/test/kill_test/process_kill_testor.cpp
+++ b/src/test/kill_test/process_kill_testor.cpp
@@ -22,6 +22,7 @@
#include <thread>
#include <iostream>
#include <cstdio>
+#include <cstring>
#include <unistd.h>
#include <chrono>
#include <thread>
@@ -74,6 +75,10 @@ DSN_DEFINE_uint32(pegasus.killtest,
30,
"sleep time before recover seconds");
+DSN_DEFINE_string(pegasus.killtest, killer_handler, "", "killer handler");
+DSN_DEFINE_validator(killer_handler,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+
DSN_DECLARE_uint32(kill_interval_seconds);
process_kill_testor::process_kill_testor(const char *config_file) : kill_testor(config_file)
@@ -81,13 +86,8 @@ process_kill_testor::process_kill_testor(const char *config_file) : kill_testor(
register_kill_handlers();
kill_round = 0;
-
- // initialize killer_handler
- std::string killer_name =
- dsn_config_get_value_string("pegasus.killtest", "killer_handler", "", "killer handler");
- CHECK(!killer_name.empty(), "");
- _killer_handler.reset(killer_handler::new_handler(killer_name.c_str()));
- CHECK(_killer_handler, "invalid killer_name({})", killer_name);
+ _killer_handler.reset(killer_handler::new_handler(FLAGS_killer_handler));
+ CHECK(_killer_handler, "invalid FLAGS_killer_handler({})", FLAGS_killer_handler);
_job_types = {META, REPLICA, ZOOKEEPER};
_job_index_to_kill.resize(JOB_LENGTH);
diff --git a/src/test/pressure_test/main.cpp b/src/test/pressure_test/main.cpp
index 8a03ed61d..7c839fe21 100644
--- a/src/test/pressure_test/main.cpp
+++ b/src/test/pressure_test/main.cpp
@@ -48,11 +48,20 @@ DSN_DEFINE_int64(pressureclient,
sortkey_limit,
0,
"The sortkey range to generate, in format [0, ****key_limit].");
+DSN_DEFINE_string(pressureclient, cluster_name, "onebox", "cluster name");
+DSN_DEFINE_validator(cluster_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+
+DSN_DEFINE_string(pressureclient, app_name, "temp", "app name");
+DSN_DEFINE_validator(app_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
+
+DSN_DEFINE_string(pressureclient, operation_name, "", "operation name");
+DSN_DEFINE_validator(operation_name,
+ [](const char *value) -> bool { return !dsn::utils::is_empty(value); });
// for app
static pegasus_client *pg_client = nullptr;
-static string cluster_name;
-static string app_name;
static string op_name; // set/get/scan/del
// fill string in prefix, until with size(len)
std::string fill_string(const std::string &str, int len)
@@ -227,26 +236,18 @@ int main(int argc, const char **argv)
}
initialize();
LOG_INFO("Initialize client and load config.ini succeed");
- cluster_name =
- dsn_config_get_value_string("pressureclient", "cluster_name", "onebox", "cluster name");
-
- app_name = dsn_config_get_value_string("pressureclient", "app_name", "temp", "app name");
-
- op_name = dsn_config_get_value_string("pressureclient", "operation_name", "", "operation name");
-
- CHECK(!op_name.empty(), "must assign operation name");
- LOG_INFO("pressureclient {} qps = {}", op_name, FLAGS_qps);
+ LOG_INFO("pressureclient {} qps = {}", FLAGS_operation_name, FLAGS_qps);
- pg_client = pegasus_client_factory::get_client(cluster_name.c_str(), app_name.c_str());
+ pg_client = pegasus_client_factory::get_client(FLAGS_cluster_name, FLAGS_app_name);
CHECK_NOTNULL(pg_client, "initialize pg_client failed");
- auto it = _all_funcs.find(op_name);
+ auto it = _all_funcs.find(FLAGS_operation_name);
if (it != _all_funcs.end()) {
- LOG_INFO("start pressureclient with {} qps({})", op_name, FLAGS_qps);
+ LOG_INFO("start pressureclient with {} qps({})", FLAGS_operation_name, FLAGS_qps);
it->second(FLAGS_qps);
} else {
- CHECK(false, "Unknown operation name({})", op_name);
+ CHECK(false, "Unknown operation name({})", FLAGS_operation_name);
}
return 0;
}
diff --git a/src/zookeeper/zookeeper_session.cpp b/src/zookeeper/zookeeper_session.cpp
index 826a964a6..b42f584a4 100644
--- a/src/zookeeper/zookeeper_session.cpp
+++ b/src/zookeeper/zookeeper_session.cpp
@@ -58,6 +58,7 @@ DSN_DEFINE_int32(zookeeper,
timeout_ms,
30000,
"The timeout of accessing ZooKeeper, in milliseconds");
+DSN_DEFINE_string(zookeeper, hosts_list, "", "Zookeeper hosts list");
zookeeper_session::zoo_atomic_packet::zoo_atomic_packet(unsigned int size)
{
@@ -164,7 +165,7 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb)
zoo_sasl_params_t sasl_params = {0};
sasl_params.service = dsn::security::FLAGS_zookeeper_kerberos_service_name;
sasl_params.mechlist = "GSSAPI";
- _handle = zookeeper_init_sasl(zookeeper_session_mgr::instance().zoo_hosts(),
+ _handle = zookeeper_init_sasl(FLAGS_hosts_list,
global_watcher,
FLAGS_timeout_ms,
nullptr,
@@ -173,12 +174,8 @@ int zookeeper_session::attach(void *callback_owner, const state_callback &cb)
NULL,
&sasl_params);
} else {
- _handle = zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(),
- global_watcher,
- FLAGS_timeout_ms,
- nullptr,
- this,
- 0);
+ _handle = zookeeper_init(
+ FLAGS_hosts_list, global_watcher, FLAGS_timeout_ms, nullptr, this, 0);
}
CHECK_NOTNULL(_handle, "zookeeper session init failed");
}
diff --git a/src/zookeeper/zookeeper_session_mgr.cpp b/src/zookeeper/zookeeper_session_mgr.cpp
index 5eb86b53e..2ff3e9d1b 100644
--- a/src/zookeeper/zookeeper_session_mgr.cpp
+++ b/src/zookeeper/zookeeper_session_mgr.cpp
@@ -38,16 +38,16 @@
#include <stdio.h>
#include <zookeeper/zookeeper.h>
#include <stdexcept>
+#include "utils/flags.h"
namespace dsn {
namespace dist {
+DSN_DEFINE_string(zookeeper, logfile, "", "The Zookeeper logfile");
+
zookeeper_session_mgr::zookeeper_session_mgr()
{
- _zoo_hosts = dsn_config_get_value_string("zookeeper", "hosts_list", "", "zookeeper_hosts");
- _zoo_logfile = dsn_config_get_value_string("zookeeper", "logfile", "", "zookeeper logfile");
-
- FILE *fp = fopen(_zoo_logfile.c_str(), "a");
+ FILE *fp = fopen(FLAGS_logfile, "a");
if (fp != nullptr)
zoo_set_log_stream(fp);
}
@@ -56,7 +56,6 @@ zookeeper_session *zookeeper_session_mgr::get_session(const service_app_info &in
{
auto &store = utils::singleton_store<int, zookeeper_session *>::instance();
zookeeper_session *ans = nullptr;
- utils::auto_lock<utils::ex_lock_nr> l(_store_lock);
if (!store.get(info.entity_id, ans)) {
ans = new zookeeper_session(info);
store.put(info.entity_id, ans);
diff --git a/src/zookeeper/zookeeper_session_mgr.h b/src/zookeeper/zookeeper_session_mgr.h
index e3959547e..e7644c515 100644
--- a/src/zookeeper/zookeeper_session_mgr.h
+++ b/src/zookeeper/zookeeper_session_mgr.h
@@ -59,17 +59,11 @@ class zookeeper_session_mgr : public utils::singleton<zookeeper_session_mgr>
{
public:
zookeeper_session *get_session(const service_app_info &info);
- const char *zoo_hosts() const { return _zoo_hosts.c_str(); }
- const char *zoo_logfile() const { return _zoo_logfile.c_str(); }
private:
zookeeper_session_mgr();
~zookeeper_session_mgr() = default;
- utils::ex_lock_nr _store_lock;
- std::string _zoo_hosts;
- std::string _zoo_logfile;
-
friend class utils::singleton<zookeeper_session_mgr>;
};
} // namespace dist
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org