You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by yu...@apache.org on 2020/09/07 01:50:39 UTC
[incubator-pegasus] branch master updated: fix: Use default options
to open db when latest option file has incompatible db options (#593)
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 4b247ab fix: Use default options to open db when latest option file has incompatible db options (#593)
4b247ab is described below
commit 4b247ab1ed6fba88b78f905d7616df9a7ad6465e
Author: Zhang Yifan <ch...@163.com>
AuthorDate: Mon Sep 7 09:50:28 2020 +0800
fix: Use default options to open db when latest option file has incompatible db options (#593)
---
src/server/pegasus_server_impl.cpp | 78 ++++++++++++++++++++++++++++----------
src/server/pegasus_server_impl.h | 3 ++
2 files changed, 61 insertions(+), 20 deletions(-)
diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp
index b61bf31..1889e98 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -1330,9 +1330,18 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
// Here we create a `tmp_data_cf_opts` because we don't want to modify `_data_cf_opts`, which
// will be used elsewhere.
rocksdb::ColumnFamilyOptions tmp_data_cf_opts = _data_cf_opts;
+ bool has_incompatible_db_options = false;
if (db_exist) {
+ // When DB exists, meta CF and data CF must be present.
bool missing_meta_cf = true;
bool missing_data_cf = true;
+ if (check_column_families(path, &missing_meta_cf, &missing_data_cf) != ::dsn::ERR_OK) {
+ derror_replica("check column families failed");
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+ dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0");
+ dassert_replica(!missing_data_cf, "Missing data column family");
+
// Load latest options from option file stored in the db directory.
rocksdb::DBOptions loaded_db_opt;
std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs;
@@ -1344,28 +1353,31 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
&loaded_cf_descs,
/*ignore_unknown_options=*/true);
if (!status.ok()) {
- derror_replica("load latest option file failed.");
- return ::dsn::ERR_LOCAL_APP_FAILURE;
- }
- for (int i = 0; i < loaded_cf_descs.size(); ++i) {
- if (loaded_cf_descs[i].name == META_COLUMN_FAMILY_NAME) {
- missing_meta_cf = false;
- } else if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
- missing_data_cf = false;
- loaded_data_cf_opts = loaded_cf_descs[i].options;
- } else {
- derror_replica("unknown column family name.");
+ // Here we ignore an invalid argument error related to `pegasus_data_version` and
+ // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0).
+ if (status.code() != rocksdb::Status::kInvalidArgument ||
+ status.ToString().find("pegasus_data") == std::string::npos) {
+ derror_replica("load latest option file failed: {}.", status.ToString());
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
+ has_incompatible_db_options = true;
+ dwarn_replica("The latest option file has incompatible db options: {}, use default "
+ "options to open db.",
+ status.ToString());
+ }
+
+ if (!has_incompatible_db_options) {
+ for (int i = 0; i < loaded_cf_descs.size(); ++i) {
+ if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) {
+ loaded_data_cf_opts = loaded_cf_descs[i].options;
+ }
+ }
+ // Reset usage scenario related options according to loaded_data_cf_opts.
+ // We don't use `loaded_data_cf_opts` directly because pointer-typed options will
+ // only be initialized with default values when calling 'LoadLatestOptions', see
+ // 'rocksdb/utilities/options_util.h'.
+ reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts);
}
- // When DB exists, meta CF and data CF must be present.
- dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0");
- dassert_replica(!missing_data_cf, "Missing data column family");
- // Reset usage scenario related options according to loaded_data_cf_opts.
- // We don't use `loaded_data_cf_opts` directly because pointer-typed options will only be
- // initialized with default values when calling 'LoadLatestOptions', see
- // 'rocksdb/utilities/options_util.h'.
- reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts);
} else {
// When create new DB, we have to create a new column family to store meta data (meta column
// family).
@@ -1376,7 +1388,7 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache
{{DATA_COLUMN_FAMILY_NAME, tmp_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}});
auto s = rocksdb::CheckOptionsCompatibility(
path, rocksdb::Env::Default(), _db_opts, column_families, /*ignore_unknown_options=*/true);
- if (!s.ok() && !s.IsNotFound()) {
+ if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) {
derror_replica("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString());
return ::dsn::ERR_LOCAL_APP_FAILURE;
}
@@ -2574,6 +2586,32 @@ bool pegasus_server_impl::set_options(
}
}
+::dsn::error_code pegasus_server_impl::check_column_families(const std::string &path,
+ bool *missing_meta_cf,
+ bool *missing_data_cf)
+{
+ *missing_meta_cf = true;
+ *missing_data_cf = true;
+ std::vector<std::string> column_families;
+ auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families);
+ if (!s.ok()) {
+ derror_replica("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString());
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+
+ for (const auto &column_family : column_families) {
+ if (column_family == META_COLUMN_FAMILY_NAME) {
+ *missing_meta_cf = false;
+ } else if (column_family == DATA_COLUMN_FAMILY_NAME) {
+ *missing_data_cf = false;
+ } else {
+ derror_replica("unknown column family name: {}", column_family);
+ return ::dsn::ERR_LOCAL_APP_FAILURE;
+ }
+ }
+ return ::dsn::ERR_OK;
+}
+
uint64_t pegasus_server_impl::do_manual_compact(const rocksdb::CompactRangeOptions &options)
{
// wait flush before compact to make all data compacted.
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 7165f76..009f97f 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -310,6 +310,9 @@ private:
return false;
}
+ ::dsn::error_code
+ check_column_families(const std::string &path, bool *missing_meta_cf, bool *miss_data_cf);
+
void release_db();
void release_db(rocksdb::DB *db, const std::vector<rocksdb::ColumnFamilyHandle *> &handles);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org