You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@pegasus.apache.org by GitBox <gi...@apache.org> on 2022/08/18 09:49:59 UTC
[GitHub] [incubator-pegasus] empiredan commented on a diff in pull request #1108: fix: rocksdb options not changed even if update in Pegasus config file
empiredan commented on code in PR #1108:
URL: https://github.com/apache/incubator-pegasus/pull/1108#discussion_r948879532
##########
src/server/pegasus_server_impl.cpp:
##########
@@ -3007,6 +3013,88 @@ void pegasus_server_impl::reset_usage_scenario_options(
target_opts->max_write_buffer_number = base_opts.max_write_buffer_number;
}
+void pegasus_server_impl::recalculate_data_cf_options(
+ const rocksdb::ColumnFamilyOptions &cur_data_cf_opts)
+{
+#define UPDATE_NUMBER_OPTION_IF_NEEDED(option, value) \
+ if ((value) != cur_data_cf_opts.option) { \
+ new_options[#option] = std::to_string((value)); \
+ }
Review Comment:
better to use `do { ... } while (0)` ?
##########
src/server/pegasus_server_impl.cpp:
##########
@@ -3007,6 +3013,88 @@ void pegasus_server_impl::reset_usage_scenario_options(
target_opts->max_write_buffer_number = base_opts.max_write_buffer_number;
}
+void pegasus_server_impl::recalculate_data_cf_options(
+ const rocksdb::ColumnFamilyOptions &cur_data_cf_opts)
+{
+#define UPDATE_NUMBER_OPTION_IF_NEEDED(option, value) \
+ if ((value) != cur_data_cf_opts.option) { \
+ new_options[#option] = std::to_string((value)); \
+ }
+#define UPDATE_BOOL_OPTION_IF_NEEDED(option, value) \
+ if ((value) != cur_data_cf_opts.option) { \
+ if ((value)) \
+ new_options[#option] = "true"; \
+ else \
+ new_options[#option] = "false"; \
+ }
+#define UPDATE_OPTION_IF_NEEDED(option) UPDATE_NUMBER_OPTION_IF_NEEDED(option, _data_cf_opts.option)
+
+ if (!_table_data_cf_opts_recalculated)
+ return;
+ std::unordered_map<std::string, std::string> new_options;
+ if (ROCKSDB_ENV_USAGE_SCENARIO_NORMAL == _usage_scenario ||
+ ROCKSDB_ENV_USAGE_SCENARIO_PREFER_WRITE == _usage_scenario) {
+ if (ROCKSDB_ENV_USAGE_SCENARIO_NORMAL == _usage_scenario) {
+ if (!check_value_if_nearby(_data_cf_opts.write_buffer_size,
+ cur_data_cf_opts.write_buffer_size)) {
+ new_options["write_buffer_size"] =
+ std::to_string(get_random_nearby(_data_cf_opts.write_buffer_size));
+ }
+ UPDATE_OPTION_IF_NEEDED(level0_file_num_compaction_trigger);
+ } else {
+ uint64_t buffer_size = dsn::rand::next_u64(_data_cf_opts.write_buffer_size,
+ _data_cf_opts.write_buffer_size * 2);
+ if (!(cur_data_cf_opts.write_buffer_size >= _data_cf_opts.write_buffer_size &&
+ cur_data_cf_opts.write_buffer_size <= _data_cf_opts.write_buffer_size * 2)) {
+ new_options["write_buffer_size"] = std::to_string(buffer_size);
+ uint64_t max_size = get_random_nearby(_data_cf_opts.max_bytes_for_level_base);
+ new_options["level0_file_num_compaction_trigger"] =
+ std::to_string(std::max<uint64_t>(4UL, max_size / buffer_size));
+ } else if (!check_value_if_nearby(_data_cf_opts.max_bytes_for_level_base,
+ cur_data_cf_opts.max_bytes_for_level_base)) {
+ uint64_t max_size = get_random_nearby(_data_cf_opts.max_bytes_for_level_base);
+ new_options["level0_file_num_compaction_trigger"] =
+ std::to_string(std::max<uint64_t>(4UL, max_size / buffer_size));
+ }
+ }
+ UPDATE_OPTION_IF_NEEDED(level0_slowdown_writes_trigger);
+ UPDATE_OPTION_IF_NEEDED(level0_stop_writes_trigger);
+ UPDATE_OPTION_IF_NEEDED(soft_pending_compaction_bytes_limit);
+ UPDATE_OPTION_IF_NEEDED(hard_pending_compaction_bytes_limit);
+ UPDATE_BOOL_OPTION_IF_NEEDED(disable_auto_compactions, false);
+ UPDATE_OPTION_IF_NEEDED(max_compaction_bytes);
+ UPDATE_OPTION_IF_NEEDED(max_write_buffer_number);
+ } else {
+ // ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD
+ UPDATE_NUMBER_OPTION_IF_NEEDED(level0_file_num_compaction_trigger, 1000000000);
+ UPDATE_NUMBER_OPTION_IF_NEEDED(level0_slowdown_writes_trigger, 1000000000);
+ UPDATE_NUMBER_OPTION_IF_NEEDED(level0_stop_writes_trigger, 1000000000);
+ UPDATE_NUMBER_OPTION_IF_NEEDED(soft_pending_compaction_bytes_limit, 0);
+ UPDATE_NUMBER_OPTION_IF_NEEDED(hard_pending_compaction_bytes_limit, 0);
+ UPDATE_BOOL_OPTION_IF_NEEDED(disable_auto_compactions, true);
+ UPDATE_NUMBER_OPTION_IF_NEEDED(max_compaction_bytes, static_cast<uint64_t>(1) << 60);
+ if (!check_value_if_nearby(_data_cf_opts.write_buffer_size * 4,
+ cur_data_cf_opts.write_buffer_size)) {
+ new_options["write_buffer_size"] =
+ std::to_string(get_random_nearby(_data_cf_opts.write_buffer_size * 4));
+ }
+ if (cur_data_cf_opts.max_write_buffer_number !=
+ std::max(_data_cf_opts.max_write_buffer_number, 6)) {
+ new_options["max_write_buffer_number"] =
+ std::to_string(std::max(_data_cf_opts.max_write_buffer_number, 6));
+ }
+ }
+ if (new_options.size() > 0) {
+ if (set_options(new_options)) {
+ ddebug_replica(
+ "{}: recalculate the value of the options related to usage scenario \"{}\"",
+ replica_name(),
+ _usage_scenario);
+ }
+ }
+ _table_data_cf_opts_recalculated = false;
Review Comment:
```c++
#undef UPDATE_OPTION_IF_NEEDED
#undef UPDATE_BOOL_OPTION_IF_NEEDED
#undef UPDATE_NUMBER_OPTION_IF_NEEDED
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@pegasus.apache.org
For additional commands, e-mail: dev-help@pegasus.apache.org