You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by yu...@apache.org on 2020/09/14 06:46:27 UTC
[incubator-pegasus] branch v2.1 updated (b6bf1aa -> 4f8db17)
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a change to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git.
omit b6bf1aa Release pegasus 2.1.0-RC1
new 839826d fix: Load options from file when open an exist DB (#587)
new a83343d fix: Use default options to open db when latest option file has incompatible db options (#593)
new 4276c2c fix: fix db is not nullptr when release_db (#594)
new c0389ae refactor: update license statement (#600)
new 4f8db17 Release pegasus 2.1.0-RC1
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (b6bf1aa)
\
N -- N -- N refs/heads/v2.1 (4f8db17)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
src/server/meta_store.cpp | 58 ++++++++--
src/server/meta_store.h | 12 ++
src/server/pegasus_server_impl.cpp | 160 ++++++++++++++++++++-------
src/server/pegasus_server_impl.h | 10 +-
src/server/test/pegasus_server_impl_test.cpp | 53 ++++++++-
6 files changed, 245 insertions(+), 50 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
[incubator-pegasus] 04/05: refactor: update license statement (#600)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit c0389aedf65dde9cd2a09af7a37e177d0b803f46
Author: von gosling <vo...@apache.org>
AuthorDate: Thu Sep 10 16:08:09 2020 +0800
refactor: update license statement (#600)
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 074bea0..9ceb3de 100644
--- a/README.md
+++ b/README.md
@@ -79,5 +79,5 @@ Data import/export tools:
## License
-Copyright 2015-now Xiaomi, Inc. Licensed under the Apache License, Version 2.0:
+Copyright 2020 The Apache Software Foundation. Licensed under the Apache License, Version 2.0:
<http://www.apache.org/licenses/LICENSE-2.0>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
[incubator-pegasus] 03/05: fix: fix db is not nullptr when
release_db (#594)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 4276c2cede6a4f65f86790f987568a8dcfe3655f
Author: HeYuchen <37...@qq.com>
AuthorDate: Tue Sep 8 18:11:36 2020 +0800
fix: fix db is not nullptr when release_db (#594)
---
src/server/pegasus_server_impl.cpp | 32 +++++++++++++++++-----------
src/server/pegasus_server_impl.h | 2 +-
src/server/test/pegasus_server_impl_test.cpp | 16 ++++++++++++++
3 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
index 1889e98..577c907 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -1813,7 +1813,16 @@ private:
if (remove_checkpoint && !::dsn::utils::filesystem::remove_path(checkpoint_dir)) {
derror_replica("remove checkpoint directory {} failed", checkpoint_dir);
}
- release_db(snapshot_db, handles_opened);
+ if (snapshot_db) {
+ for (auto handle : handles_opened) {
+ if (handle) {
+ snapshot_db->DestroyColumnFamilyHandle(handle);
+ handle = nullptr;
+ }
+ }
+ delete snapshot_db;
+ snapshot_db = nullptr;
+ }
};
// Because of RocksDB's restriction, we have to to open default column family even though
@@ -2716,19 +2725,16 @@ void pegasus_server_impl::set_partition_version(int32_t partition_version)
return ::dsn::ERR_OK;
}
-void pegasus_server_impl::release_db() { release_db(_db, {_data_cf, _meta_cf}); }
-
-void pegasus_server_impl::release_db(rocksdb::DB *db,
- const std::vector<rocksdb::ColumnFamilyHandle *> &handles)
+void pegasus_server_impl::release_db()
{
- if (db) {
- for (auto handle : handles) {
- dassert_replica(handle != nullptr, "");
- db->DestroyColumnFamilyHandle(handle);
- handle = nullptr;
- }
- delete db;
- db = nullptr;
+ if (_db) {
+ dassert_replica(_data_cf != nullptr && _meta_cf != nullptr, "");
+ _db->DestroyColumnFamilyHandle(_data_cf);
+ _data_cf = nullptr;
+ _db->DestroyColumnFamilyHandle(_meta_cf);
+ _meta_cf = nullptr;
+ delete _db;
+ _db = nullptr;
}
}
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 009f97f..1db2077 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -168,6 +168,7 @@ private:
FRIEND_TEST(pegasus_server_impl_test, default_data_version);
FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_latest_options);
FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_app_envs);
+ FRIEND_TEST(pegasus_server_impl_test, test_stop_db_twice);
friend class pegasus_manual_compact_service;
friend class pegasus_write_service;
@@ -314,7 +315,6 @@ private:
check_column_families(const std::string &path, bool *missing_meta_cf, bool *miss_data_cf);
void release_db();
- void release_db(rocksdb::DB *db, const std::vector<rocksdb::ColumnFamilyHandle *> &handles);
::dsn::error_code flush_all_family_columns(bool wait);
diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp
index d9d161a..0302d8f 100644
--- a/src/server/test/pegasus_server_impl_test.cpp
+++ b/src/server/test/pegasus_server_impl_test.cpp
@@ -99,5 +99,21 @@ TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs)
ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario);
}
+TEST_F(pegasus_server_impl_test, test_stop_db_twice)
+{
+ start();
+ ASSERT_TRUE(_server->_is_open);
+ ASSERT_TRUE(_server->_db != nullptr);
+
+ _server->stop(false);
+ ASSERT_FALSE(_server->_is_open);
+ ASSERT_TRUE(_server->_db == nullptr);
+
+ // stop again
+ _server->stop(false);
+ ASSERT_FALSE(_server->_is_open);
+ ASSERT_TRUE(_server->_db == nullptr);
+}
+
} // namespace server
} // namespace pegasus
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
[incubator-pegasus] 01/05: fix: Load options from file when open an
exist DB (#587)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 839826d288159767a4048eb9d961eb88cfd5d99c
Author: Zhang Yifan <ch...@163.com>
AuthorDate: Mon Aug 31 11:58:57 2020 +0800
fix: Load options from file when open an exist DB (#587)
---
rdsn | 2 +-
src/server/meta_store.cpp | 58 +++++++++++++++---
src/server/meta_store.h | 12 ++++
src/server/pegasus_server_impl.cpp | 90 ++++++++++++++++++++--------
src/server/pegasus_server_impl.h | 7 ++-
src/server/test/pegasus_server_impl_test.cpp | 37 +++++++++++-
6 files changed, 169 insertions(+), 37 deletions(-)
diff --git a/rdsn b/rdsn
index 40d86f7..2158aeb 160000
--- a/rdsn
+++ b/rdsn
@@ -1 +1 @@
-Subproject commit 40d86f7f7aa51ba874d9cc91cb7e267f18b2bd11
+Subproject commit 2158aeb5022a9881c18bbad3addd02c68eb93323
diff --git a/src/server/meta_store.cpp b/src/server/meta_store.cpp
index e1562c4..fee4e51 100644
--- a/src/server/meta_store.cpp
+++ b/src/server/meta_store.cpp
@@ -58,6 +58,19 @@ uint64_t meta_store::get_decree_from_readonly_db(rocksdb::DB *db,
return last_flushed_decree;
}
+std::string meta_store::get_usage_scenario() const
+{
+ // If couldn't find rocksdb usage scenario in meta column family, return normal in default.
+ std::string usage_scenario = ROCKSDB_ENV_USAGE_SCENARIO_NORMAL;
+ auto ec = get_string_value_from_meta_cf(false, ROCKSDB_ENV_USAGE_SCENARIO_KEY, &usage_scenario);
+ dassert_replica(ec == ::dsn::ERR_OK || ec == ::dsn::ERR_OBJECT_NOT_FOUND,
+ "rocksdb {} get {} from meta column family failed: {}",
+ _db->GetName(),
+ ROCKSDB_ENV_USAGE_SCENARIO_KEY,
+ ec.to_string());
+ return usage_scenario;
+}
+
::dsn::error_code meta_store::get_value_from_meta_cf(bool read_flushed_data,
const std::string &key,
uint64_t *value) const
@@ -72,18 +85,37 @@ uint64_t meta_store::get_decree_from_readonly_db(rocksdb::DB *db,
uint64_t *value)
{
std::string data;
+ auto ec = get_string_value_from_meta_cf(db, cf, read_flushed_data, key, &data);
+ if (ec != ::dsn::ERR_OK) {
+ return ec;
+ }
+ dassert_f(dsn::buf2uint64(data, *value),
+ "rocksdb {} get \"{}\" from meta column family failed to parse into uint64",
+ db->GetName(),
+ data);
+ return ::dsn::ERR_OK;
+}
+
+::dsn::error_code meta_store::get_string_value_from_meta_cf(bool read_flushed_data,
+ const std::string &key,
+ std::string *value) const
+{
+ return get_string_value_from_meta_cf(_db, _meta_cf, read_flushed_data, key, value);
+}
+
+::dsn::error_code meta_store::get_string_value_from_meta_cf(rocksdb::DB *db,
+ rocksdb::ColumnFamilyHandle *cf,
+ bool read_flushed_data,
+ const std::string &key,
+ std::string *value)
+{
rocksdb::ReadOptions rd_opts;
if (read_flushed_data) {
// only read 'flushed' data, mainly to read 'last_flushed_decree'
rd_opts.read_tier = rocksdb::kPersistedTier;
}
- auto status = db->Get(rd_opts, cf, key, &data);
+ auto status = db->Get(rd_opts, cf, key, value);
if (status.ok()) {
- dassert_f(dsn::buf2uint64(data, *value),
- "rocksdb {} get {} from meta column family got error value {}",
- db->GetName(),
- key,
- data);
return ::dsn::ERR_OK;
}
@@ -97,7 +129,13 @@ uint64_t meta_store::get_decree_from_readonly_db(rocksdb::DB *db,
::dsn::error_code meta_store::set_value_to_meta_cf(const std::string &key, uint64_t value) const
{
- auto status = _db->Put(_wt_opts, _meta_cf, key, std::to_string(value));
+ return set_string_value_to_meta_cf(key, std::to_string(value));
+}
+
+::dsn::error_code meta_store::set_string_value_to_meta_cf(const std::string &key,
+ const std::string &value) const
+{
+ auto status = _db->Put(_wt_opts, _meta_cf, key, value);
if (!status.ok()) {
derror_replica(
"Put {}={} to meta column family failed, status {}", key, value, status.ToString());
@@ -124,5 +162,11 @@ void meta_store::set_last_manual_compact_finish_time(uint64_t last_manual_compac
set_value_to_meta_cf(LAST_MANUAL_COMPACT_FINISH_TIME, last_manual_compact_finish_time));
}
+void meta_store::set_usage_scenario(const std::string &usage_scenario) const
+{
+ dcheck_eq_replica(::dsn::ERR_OK,
+ set_string_value_to_meta_cf(ROCKSDB_ENV_USAGE_SCENARIO_KEY, usage_scenario));
+}
+
} // namespace server
} // namespace pegasus
diff --git a/src/server/meta_store.h b/src/server/meta_store.h
index 322971e..6e7a7b8 100644
--- a/src/server/meta_store.h
+++ b/src/server/meta_store.h
@@ -30,21 +30,33 @@ public:
rocksdb::ColumnFamilyHandle *meta_cf) const;
uint32_t get_data_version() const;
uint64_t get_last_manual_compact_finish_time() const;
+ std::string get_usage_scenario() const;
void set_last_flushed_decree(uint64_t decree) const;
void set_data_version(uint32_t version) const;
void set_last_manual_compact_finish_time(uint64_t last_manual_compact_finish_time) const;
+ void set_usage_scenario(const std::string &usage_scenario) const;
private:
::dsn::error_code
get_value_from_meta_cf(bool read_flushed_data, const std::string &key, uint64_t *value) const;
+ ::dsn::error_code get_string_value_from_meta_cf(bool read_flushed_data,
+ const std::string &key,
+ std::string *value) const;
::dsn::error_code set_value_to_meta_cf(const std::string &key, uint64_t value) const;
+ ::dsn::error_code set_string_value_to_meta_cf(const std::string &key,
+ const std::string &value) const;
static ::dsn::error_code get_value_from_meta_cf(rocksdb::DB *db,
rocksdb::ColumnFamilyHandle *cf,
bool read_flushed_data,
const std::string &key,
uint64_t *value);
+ static ::dsn::error_code get_string_value_from_meta_cf(rocksdb::DB *db,
+ rocksdb::ColumnFamilyHandle *cf,
+ bool read_flushed_data,
+ const std::string &key,
+ std::string *value);
friend class pegasus_write_service;
diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
index b181c44..b61bf31 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -8,6 +8,7 @@
#include <boost/lexical_cast.hpp>
#include <rocksdb/convenience.h>
#include <rocksdb/utilities/checkpoint.h>
+#include <rocksdb/utilities/options_util.h>
#include <dsn/utility/chrono_literals.h>
#include <dsn/utility/utils.h>
#include <dsn/utility/filesystem.h>
@@ -1326,14 +1327,45 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
ddebug("%s: start to open rocksDB's rdb(%s)", replica_name(), path.c_str());
+ // Here we create a `tmp_data_cf_opts` because we don't want to modify `_data_cf_opts`, which
+ // will be used elsewhere.
+ rocksdb::ColumnFamilyOptions tmp_data_cf_opts = _data_cf_opts;
if (db_exist) {
- // When DB exist, meta CF must be present.
bool missing_meta_cf = true;
- if (check_meta_cf(path, &missing_meta_cf) != ::dsn::ERR_OK) {
- derror_replica("check meta column family failed");
+ bool missing_data_cf = true;
+ // Load latest options from option file stored in the db directory.
+ rocksdb::DBOptions loaded_db_opt;
+ std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
+ rocksdb::ColumnFamilyOptions loaded_data_cf_opts;
+ // Set `ignore_unknown_options` true for forward compatibility.
+ auto status = rocksdb::LoadLatestOptions(path,
+ rocksdb::Env::Default(),
+ &loaded_db_opt,
+ &loaded_cf_descs,
+ /*ignore_unknown_options=*/true);
+ if (!status.ok()) {
+ derror_replica("load latest option file failed.");
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
+ for (int i = 0; i < loaded_cf_descs.size(); ++i) {
+ if (loaded_cf_descs[i].name == META_COLUMN_FAMILY_NAME) {
+ missing_meta_cf = false;
+ } else if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
+ missing_data_cf = false;
+ loaded_data_cf_opts = loaded_cf_descs[i].options;
+ } else {
+ derror_replica("unknown column family name.");
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+ }
+ // When DB exists, meta CF and data CF must be present.
dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0");
+ dassert_replica(!missing_data_cf, "Missing data column family");
+ // Reset usage scenario related options according to loaded_data_cf_opts.
+ // We don't use `loaded_data_cf_opts` directly because pointer-typed options will only be
+ // initialized with default values when calling 'LoadLatestOptions', see
+ // 'rocksdb/utilities/options_util.h'.
+ reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts);
} else {
// When create new DB, we have to create a new column family to store meta data (meta column
// family).
@@ -1341,7 +1373,13 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
}
std::vector<rocksdb::ColumnFamilyDescriptor> column_families(
- {{DATA_COLUMN_FAMILY_NAME, _data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
+ {{DATA_COLUMN_FAMILY_NAME, tmp_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
+ auto s = rocksdb::CheckOptionsCompatibility(
+ path, rocksdb::Env::Default(), _db_opts, column_families, /*ignore_unknown_options=*/true);
+ if (!s.ok() && !s.IsNotFound()) {
+ derror_replica("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString());
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
std::vector<rocksdb::ColumnFamilyHandle *> handles_opened;
auto status = rocksdb::DB::Open(_db_opts, path, column_families, &handles_opened, &_db);
if (!status.ok()) {
@@ -1360,6 +1398,7 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
if (db_exist) {
_last_committed_decree = _meta_store->get_last_flushed_decree();
_pegasus_data_version = _meta_store->get_data_version();
+ _usage_scenario = _meta_store->get_usage_scenario();
uint64_t last_manual_compact_finish_time =
_meta_store->get_last_manual_compact_finish_time();
if (_pegasus_data_version > PEGASUS_DATA_VERSION_MAX) {
@@ -1407,8 +1446,10 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
_is_open = true;
- // set default usage scenario after db opened.
- set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL);
+ if (!db_exist) {
+ // When create a new db, update usage scenario according to app envs.
+ update_usage_scenario(envs);
+ }
dinfo_replica("start the update rocksdb statistics timer task");
_update_replica_rdb_stat =
@@ -2472,6 +2513,7 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario)
return false;
}
if (set_options(new_options)) {
+ _meta_store->set_usage_scenario(usage_scenario);
_usage_scenario = usage_scenario;
ddebug_replica(
"set usage scenario from \"{}\" to \"{}\" succeed", old_usage_scenario, usage_scenario);
@@ -2483,6 +2525,23 @@ bool pegasus_server_impl::set_usage_scenario(const std::string &usage_scenario)
}
}
+void pegasus_server_impl::reset_usage_scenario_options(
+ const rocksdb::ColumnFamilyOptions &base_opts, rocksdb::ColumnFamilyOptions *target_opts)
+{
+ // reset usage scenario related options, refer to options set in 'set_usage_scenario' function.
+ target_opts->level0_file_num_compaction_trigger = base_opts.level0_file_num_compaction_trigger;
+ target_opts->level0_slowdown_writes_trigger = base_opts.level0_slowdown_writes_trigger;
+ target_opts->level0_stop_writes_trigger = base_opts.level0_stop_writes_trigger;
+ target_opts->soft_pending_compaction_bytes_limit =
+ base_opts.soft_pending_compaction_bytes_limit;
+ target_opts->hard_pending_compaction_bytes_limit =
+ base_opts.hard_pending_compaction_bytes_limit;
+ target_opts->disable_auto_compactions = base_opts.disable_auto_compactions;
+ target_opts->max_compaction_bytes = base_opts.max_compaction_bytes;
+ target_opts->write_buffer_size = base_opts.write_buffer_size;
+ target_opts->max_write_buffer_number = base_opts.max_write_buffer_number;
+}
+
bool pegasus_server_impl::set_options(
const std::unordered_map<std::string, std::string> &new_options)
{
@@ -2607,25 +2666,6 @@ void pegasus_server_impl::set_partition_version(int32_t partition_version)
// TODO(heyuchen): set filter _partition_version in further pr
}
-::dsn::error_code pegasus_server_impl::check_meta_cf(const std::string &path, bool *missing_meta_cf)
-{
- *missing_meta_cf = true;
- std::vector<std::string> column_families;
- auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families);
- if (!s.ok()) {
- derror_replica("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString());
- return ::dsn::ERR_LOCAL_APP_FAILURE;
- }
-
- for (const auto &column_family : column_families) {
- if (column_family == META_COLUMN_FAMILY_NAME) {
- *missing_meta_cf = false;
- break;
- }
- }
- return ::dsn::ERR_OK;
-}
-
::dsn::error_code pegasus_server_impl::flush_all_family_columns(bool wait)
{
rocksdb::FlushOptions options;
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 2ce6015..7165f76 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -166,6 +166,8 @@ private:
friend class pegasus_compression_options_test;
friend class pegasus_server_impl_test;
FRIEND_TEST(pegasus_server_impl_test, default_data_version);
+ FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_latest_options);
+ FRIEND_TEST(pegasus_server_impl_test, test_open_db_with_app_envs);
friend class pegasus_manual_compact_service;
friend class pegasus_write_service;
@@ -260,6 +262,9 @@ private:
// return true if successfully changed
bool set_usage_scenario(const std::string &usage_scenario);
+ void reset_usage_scenario_options(const rocksdb::ColumnFamilyOptions &base_opts,
+ rocksdb::ColumnFamilyOptions *target_opts);
+
// return true if successfully set
bool set_options(const std::unordered_map<std::string, std::string> &new_options);
@@ -305,8 +310,6 @@ private:
return false;
}
- ::dsn::error_code check_meta_cf(const std::string &path, bool *missing_meta_cf);
-
void release_db();
void release_db(rocksdb::DB *db, const std::vector<rocksdb::ColumnFamilyHandle *> &handles);
diff --git a/src/server/test/pegasus_server_impl_test.cpp b/src/server/test/pegasus_server_impl_test.cpp
index c8c16b4..d9d161a 100644
--- a/src/server/test/pegasus_server_impl_test.cpp
+++ b/src/server/test/pegasus_server_impl_test.cpp
@@ -11,7 +11,7 @@ namespace server {
class pegasus_server_impl_test : public pegasus_server_test_base
{
public:
- pegasus_server_impl_test() : pegasus_server_test_base() { start(); }
+ pegasus_server_impl_test() : pegasus_server_test_base() {}
void test_table_level_slow_query()
{
@@ -59,12 +59,45 @@ public:
}
};
-TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); }
+TEST_F(pegasus_server_impl_test, test_table_level_slow_query)
+{
+ start();
+ test_table_level_slow_query();
+}
TEST_F(pegasus_server_impl_test, default_data_version)
{
+ start();
ASSERT_EQ(_server->_pegasus_data_version, 1);
}
+TEST_F(pegasus_server_impl_test, test_open_db_with_latest_options)
+{
+ // open a new db with no app env.
+ start();
+ ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_NORMAL, _server->_usage_scenario);
+ // set bulk_load scenario for the db.
+ ASSERT_TRUE(_server->set_usage_scenario(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD));
+ ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario);
+ rocksdb::Options opts = _server->_db->GetOptions();
+ ASSERT_EQ(1000000000, opts.level0_file_num_compaction_trigger);
+ ASSERT_EQ(true, opts.disable_auto_compactions);
+ // reopen the db.
+ _server->stop(false);
+ start();
+ ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario);
+ ASSERT_EQ(opts.level0_file_num_compaction_trigger,
+ _server->_db->GetOptions().level0_file_num_compaction_trigger);
+ ASSERT_EQ(opts.disable_auto_compactions, _server->_db->GetOptions().disable_auto_compactions);
+}
+
+TEST_F(pegasus_server_impl_test, test_open_db_with_app_envs)
+{
+ std::map<std::string, std::string> envs;
+ envs[ROCKSDB_ENV_USAGE_SCENARIO_KEY] = ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD;
+ start(envs);
+ ASSERT_EQ(ROCKSDB_ENV_USAGE_SCENARIO_BULK_LOAD, _server->_usage_scenario);
+}
+
} // namespace server
} // namespace pegasus
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
[incubator-pegasus] 02/05: fix: Use default options to open db when
latest option file has incompatible db options (#593)
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit a83343d4a54d835d41180e5a6e1ca93591a76c8b
Author: Zhang Yifan <ch...@163.com>
AuthorDate: Mon Sep 7 09:50:28 2020 +0800
fix: Use default options to open db when latest option file has incompatible db options (#593)
---
src/server/pegasus_server_impl.cpp | 78 ++++++++++++++++++++++++++++----------
src/server/pegasus_server_impl.h | 3 ++
2 files changed, 61 insertions(+), 20 deletions(-)
diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
index b61bf31..1889e98 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -1330,9 +1330,18 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
// Here we create a `tmp_data_cf_opts` because we don't want to modify `_data_cf_opts`, which
// will be used elsewhere.
rocksdb::ColumnFamilyOptions tmp_data_cf_opts = _data_cf_opts;
+ bool has_incompatible_db_options = false;
if (db_exist) {
+ // When DB exists, meta CF and data CF must be present.
bool missing_meta_cf = true;
bool missing_data_cf = true;
+ if (check_column_families(path, &missing_meta_cf, &missing_data_cf) != ::dsn::ERR_OK) {
+ derror_replica("check column families failed");
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+ dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0");
+ dassert_replica(!missing_data_cf, "Missing data column family");
+
// Load latest options from option file stored in the db directory.
rocksdb::DBOptions loaded_db_opt;
std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
@@ -1344,28 +1353,31 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
&loaded_cf_descs,
/*ignore_unknown_options=*/true);
if (!status.ok()) {
- derror_replica("load latest option file failed.");
- return ::dsn::ERR_LOCAL_APP_FAILURE;
- }
- for (int i = 0; i < loaded_cf_descs.size(); ++i) {
- if (loaded_cf_descs[i].name == META_COLUMN_FAMILY_NAME) {
- missing_meta_cf = false;
- } else if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
- missing_data_cf = false;
- loaded_data_cf_opts = loaded_cf_descs[i].options;
- } else {
- derror_replica("unknown column family name.");
+ // Here we ignore an invalid argument error related to `pegasus_data_version` and
+ // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0).
+ if (status.code() != rocksdb::Status::kInvalidArgument ||
+ status.ToString().find("pegasus_data") == std::string::npos) {
+ derror_replica("load latest option file failed: {}.", status.ToString());
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
+ has_incompatible_db_options = true;
+ dwarn_replica("The latest option file has incompatible db options: {}, use default "
+ "options to open db.",
+ status.ToString());
+ }
+
+ if (!has_incompatible_db_options) {
+ for (int i = 0; i < loaded_cf_descs.size(); ++i) {
+ if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
+ loaded_data_cf_opts = loaded_cf_descs[i].options;
+ }
+ }
+ // Reset usage scenario related options according to loaded_data_cf_opts.
+ // We don't use `loaded_data_cf_opts` directly because pointer-typed options will
+ // only be initialized with default values when calling 'LoadLatestOptions', see
+ // 'rocksdb/utilities/options_util.h'.
+ reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts);
}
- // When DB exists, meta CF and data CF must be present.
- dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0");
- dassert_replica(!missing_data_cf, "Missing data column family");
- // Reset usage scenario related options according to loaded_data_cf_opts.
- // We don't use `loaded_data_cf_opts` directly because pointer-typed options will only be
- // initialized with default values when calling 'LoadLatestOptions', see
- // 'rocksdb/utilities/options_util.h'.
- reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts);
} else {
// When create new DB, we have to create a new column family to store meta data (meta column
// family).
@@ -1376,7 +1388,7 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
{{DATA_COLUMN_FAMILY_NAME, tmp_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
auto s = rocksdb::CheckOptionsCompatibility(
path, rocksdb::Env::Default(), _db_opts, column_families, /*ignore_unknown_options=*/true);
- if (!s.ok() && !s.IsNotFound()) {
+ if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) {
derror_replica("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString());
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
@@ -2574,6 +2586,32 @@ bool pegasus_server_impl::set_options(
}
}
+::dsn::error_code pegasus_server_impl::check_column_families(const std::string &path,
+ bool *missing_meta_cf,
+ bool *missing_data_cf)
+{
+ *missing_meta_cf = true;
+ *missing_data_cf = true;
+ std::vector<std::string> column_families;
+ auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families);
+ if (!s.ok()) {
+ derror_replica("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString());
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+
+ for (const auto &column_family : column_families) {
+ if (column_family == META_COLUMN_FAMILY_NAME) {
+ *missing_meta_cf = false;
+ } else if (column_family == DATA_COLUMN_FAMILY_NAME) {
+ *missing_data_cf = false;
+ } else {
+ derror_replica("unknown column family name: {}", column_family);
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+ }
+ return ::dsn::ERR_OK;
+}
+
uint64_t pegasus_server_impl::do_manual_compact(const rocksdb::CompactRangeOptions &options)
{
// wait flush before compact to make all data compacted.
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 7165f76..009f97f 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -310,6 +310,9 @@ private:
return false;
}
+ ::dsn::error_code
+ check_column_families(const std::string &path, bool *missing_meta_cf, bool *miss_data_cf);
+
void release_db();
void release_db(rocksdb::DB *db, const std::vector<rocksdb::ColumnFamilyHandle *> &handles);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org
[incubator-pegasus] 05/05: Release pegasus 2.1.0-RC1
Posted by yu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch v2.1
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit 4f8db17808470a999b9a7c49d451ff6e957f15e6
Author: heyuchen <he...@xiaomi.com>
AuthorDate: Mon Sep 14 14:44:56 2020 +0800
Release pegasus 2.1.0-RC1
---
src/include/pegasus/version.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/include/pegasus/version.h b/src/include/pegasus/version.h
index f9a1d63..d2814e0 100644
--- a/src/include/pegasus/version.h
+++ b/src/include/pegasus/version.h
@@ -3,4 +3,4 @@
// can be found in the LICENSE file in the root directory of this source tree.
#pragma once
-#define PEGASUS_VERSION "1.13.SNAPSHOT"
+#define PEGASUS_VERSION "2.1.0-RC1"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org