You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kvrocks.apache.org by tw...@apache.org on 2023/04/13 05:34:40 UTC
[incubator-kvrocks] branch unstable updated: Add max background jobs to configuration (#1374)
This is an automated email from the ASF dual-hosted git repository.
twice pushed a commit to branch unstable
in repository https://gitbox.apache.org/repos/asf/incubator-kvrocks.git
The following commit(s) were added to refs/heads/unstable by this push:
new 060612aa Add max background jobs to configuration (#1374)
060612aa is described below
commit 060612aa21f0522cb9fc8361f95832bed4bddcaa
Author: bruceqiwang <12...@users.noreply.github.com>
AuthorDate: Thu Apr 13 05:34:34 2023 +0000
Add max background jobs to configuration (#1374)
Co-authored-by: datavisorbiaoqi <bi...@datavisor.com>
Co-authored-by: Twice <tw...@apache.org>
Co-authored-by: Twice <tw...@gmail.com>
---
kvrocks.conf | 13 +++++++++++--
src/config/config.cc | 6 ++++--
src/config/config.h | 1 +
src/storage/storage.cc | 1 +
tests/cppunit/config_test.cc | 5 +++--
5 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/kvrocks.conf b/kvrocks.conf
index 24f05ce8..6ac511de 100644
--- a/kvrocks.conf
+++ b/kvrocks.conf
@@ -587,15 +587,24 @@ rocksdb.target_file_size_base 128
# allowed.
rocksdb.max_write_buffer_number 4
+# Maximum number of concurrent background jobs (compactions and flushes).
+# For backwards compatibility we will set `max_background_jobs =
+# max_background_compactions + max_background_flushes` in the case where user
+# sets at least one of `max_background_compactions` or `max_background_flushes`
+# (we replace -1 by 1 in case one option is unset).
+rocksdb.max_background_jobs 4
+
+# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs
# Maximum number of concurrent background compaction jobs, submitted to
# the default LOW priority thread pool.
-rocksdb.max_background_compactions 4
+rocksdb.max_background_compactions -1
+# DEPRECATED: it is automatically decided based on the value of rocksdb.max_background_jobs
# Maximum number of concurrent background memtable flush jobs, submitted by
# default to the HIGH priority thread pool. If the HIGH priority thread pool
# is configured to have zero threads, flush jobs will share the LOW priority
# thread pool with compaction jobs.
-rocksdb.max_background_flushes 4
+rocksdb.max_background_flushes -1
# This value represents the maximum number of threads that will
# concurrently perform a compaction job by breaking it into multiple,
diff --git a/src/config/config.cc b/src/config/config.cc
index 95cfbcd6..105bd22c 100644
--- a/src/config/config.cc
+++ b/src/config/config.cc
@@ -174,8 +174,8 @@ Config::Config() {
{"rocksdb.write_buffer_size", false, new IntField(&RocksDB.write_buffer_size, 64, 0, 4096)},
{"rocksdb.max_write_buffer_number", false, new IntField(&RocksDB.max_write_buffer_number, 4, 0, 256)},
{"rocksdb.target_file_size_base", false, new IntField(&RocksDB.target_file_size_base, 128, 1, 1024)},
- {"rocksdb.max_background_compactions", false, new IntField(&RocksDB.max_background_compactions, 2, 0, 32)},
- {"rocksdb.max_background_flushes", true, new IntField(&RocksDB.max_background_flushes, 2, 0, 32)},
+ {"rocksdb.max_background_compactions", false, new IntField(&RocksDB.max_background_compactions, 2, -1, 32)},
+ {"rocksdb.max_background_flushes", true, new IntField(&RocksDB.max_background_flushes, 2, -1, 32)},
{"rocksdb.max_sub_compactions", false, new IntField(&RocksDB.max_sub_compactions, 1, 0, 16)},
{"rocksdb.delayed_write_rate", false, new Int64Field(&RocksDB.delayed_write_rate, 0, 0, INT64_MAX)},
{"rocksdb.wal_ttl_seconds", true, new IntField(&RocksDB.WAL_ttl_seconds, 3 * 3600, 0, INT_MAX)},
@@ -209,6 +209,7 @@ Config::Config() {
new IntField(&RocksDB.max_bytes_for_level_multiplier, 10, 1, 100)},
{"rocksdb.level_compaction_dynamic_level_bytes", false,
new YesNoField(&RocksDB.level_compaction_dynamic_level_bytes, false)},
+ {"rocksdb.max_background_jobs", false, new IntField(&RocksDB.max_background_jobs, 4, 0, 32)},
/* rocksdb write options */
{"rocksdb.write_options.sync", true, new YesNoField(&RocksDB.write_options.sync, false)},
@@ -591,6 +592,7 @@ void Config::initFieldCallback() {
{"rocksdb.max_background_compactions", set_db_option_cb},
{"rocksdb.max_background_flushes", set_db_option_cb},
{"rocksdb.compaction_readahead_size", set_db_option_cb},
+ {"rocksdb.max_background_jobs", set_db_option_cb},
{"rocksdb.max_write_buffer_number", set_cf_option_cb},
{"rocksdb.level0_slowdown_writes_trigger", set_cf_option_cb},
diff --git a/src/config/config.h b/src/config/config.h
index 9845ba76..6496f14e 100644
--- a/src/config/config.h
+++ b/src/config/config.h
@@ -186,6 +186,7 @@ struct Config {
int max_bytes_for_level_base;
int max_bytes_for_level_multiplier;
bool level_compaction_dynamic_level_bytes;
+ int max_background_jobs;
struct WriteOptions {
bool sync;
diff --git a/src/storage/storage.cc b/src/storage/storage.cc
index db2e050e..7a69c69f 100644
--- a/src/storage/storage.cc
+++ b/src/storage/storage.cc
@@ -170,6 +170,7 @@ rocksdb::Options Storage::InitRocksDBOptions() {
options.max_bytes_for_level_base = config_->RocksDB.max_bytes_for_level_base;
options.max_bytes_for_level_multiplier = config_->RocksDB.max_bytes_for_level_multiplier;
options.level_compaction_dynamic_level_bytes = config_->RocksDB.level_compaction_dynamic_level_bytes;
+ options.max_background_jobs = config_->RocksDB.max_background_jobs;
return options;
}
diff --git a/tests/cppunit/config_test.cc b/tests/cppunit/config_test.cc
index dcee0ca5..4dc3e832 100644
--- a/tests/cppunit/config_test.cc
+++ b/tests/cppunit/config_test.cc
@@ -63,7 +63,7 @@ TEST(Config, GetAndSet) {
{"rocksdb.write_buffer_size", "1234"},
{"rocksdb.max_write_buffer_number", "1"},
{"rocksdb.target_file_size_base", "100"},
- {"rocksdb.max_background_compactions", "2"},
+ {"rocksdb.max_background_compactions", "-1"},
{"rocksdb.max_sub_compactions", "3"},
{"rocksdb.delayed_write_rate", "1234"},
{"rocksdb.stats_dump_period_sec", "600"},
@@ -78,6 +78,7 @@ TEST(Config, GetAndSet) {
{"rocksdb.max_bytes_for_level_base", "268435456"},
{"rocksdb.max_bytes_for_level_multiplier", "10"},
{"rocksdb.level_compaction_dynamic_level_bytes", "yes"},
+ {"rocksdb.max_background_jobs", "4"},
};
std::vector<std::string> values;
for (const auto &iter : mutable_cases) {
@@ -115,7 +116,7 @@ TEST(Config, GetAndSet) {
{"pidfile", "test.pid"},
{"supervised", "no"},
{"rocksdb.block_size", "1234"},
- {"rocksdb.max_background_flushes", "16"},
+ {"rocksdb.max_background_flushes", "-1"},
{"rocksdb.wal_ttl_seconds", "10000"},
{"rocksdb.wal_size_limit_mb", "16"},
{"rocksdb.enable_pipelined_write", "no"},