You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2020/04/09 16:32:32 UTC
[incubator-doris] branch master updated: [refactor] A small
refactor on class DataDir (#3276)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new f39c8b1 [refactor] A small refactor on class DataDir (#3276)
f39c8b1 is described below
commit f39c8b156dd74b2eb5d1ed6fa03ff9477e4ddae3
Author: Yingchun Lai <40...@qq.com>
AuthorDate: Thu Apr 9 11:32:22 2020 -0500
[refactor] A small refactor on class DataDir (#3276)
main refactor points are:
- Use a single get_absolute_tablet_path function instead of 3
independent functions
- Remove meaningless return value of register_tablet and deregister_tablet
- Some typo and format
---
be/src/http/action/restore_tablet_action.cpp | 94 ++++++++++++++-------------
be/src/olap/data_dir.cpp | 82 ++++++++---------------
be/src/olap/data_dir.h | 41 +++++-------
be/src/olap/olap_snapshot_converter.cpp | 6 +-
be/src/olap/tablet.h | 12 ++--
be/src/olap/tablet_manager.cpp | 8 +--
be/test/olap/olap_snapshot_converter_test.cpp | 68 +++++++++----------
gensrc/proto/olap_file.proto | 2 +-
8 files changed, 139 insertions(+), 174 deletions(-)
diff --git a/be/src/http/action/restore_tablet_action.cpp b/be/src/http/action/restore_tablet_action.cpp
index bb24881..53f9f17 100644
--- a/be/src/http/action/restore_tablet_action.cpp
+++ b/be/src/http/action/restore_tablet_action.cpp
@@ -17,28 +17,28 @@
#include "http/action/restore_tablet_action.h"
-#include <string>
-#include <sstream>
#include <unistd.h>
-#include "boost/lexical_cast.hpp"
+#include <sstream>
+#include <string>
#include "agent/cgroups_mgr.h"
+#include "boost/lexical_cast.hpp"
#include "env/env.h"
+#include "gutil/strings/substitute.h" // for Substitute
#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
#include "http/http_response.h"
#include "http/http_status.h"
-#include "util/file_utils.h"
-#include "olap/utils.h"
-#include "olap/tablet_meta.h"
-#include "util/json_util.h"
+#include "olap/data_dir.h"
#include "olap/olap_define.h"
#include "olap/storage_engine.h"
-#include "olap/data_dir.h"
+#include "olap/tablet_meta.h"
+#include "olap/utils.h"
#include "runtime/exec_env.h"
-#include "gutil/strings/substitute.h" // for Substitute
+#include "util/file_utils.h"
+#include "util/json_util.h"
using boost::filesystem::path;
@@ -47,10 +47,9 @@ namespace doris {
const std::string TABLET_ID = "tablet_id";
const std::string SCHEMA_HASH = "schema_hash";
-RestoreTabletAction::RestoreTabletAction(ExecEnv* exec_env) : _exec_env(exec_env) {
-}
+RestoreTabletAction::RestoreTabletAction(ExecEnv* exec_env) : _exec_env(exec_env) {}
-void RestoreTabletAction::handle(HttpRequest *req) {
+void RestoreTabletAction::handle(HttpRequest* req) {
LOG(INFO) << "accept one request " << req->debug_string();
// add tid to cgroup in order to limit read bandwidth
CgroupsMgr::apply_system_cgroup();
@@ -64,20 +63,18 @@ void RestoreTabletAction::handle(HttpRequest *req) {
}
}
-Status RestoreTabletAction::_handle(HttpRequest *req) {
+Status RestoreTabletAction::_handle(HttpRequest* req) {
// Get tablet id
const std::string& tablet_id_str = req->param(TABLET_ID);
if (tablet_id_str.empty()) {
- std::string error_msg = std::string(
- "parameter " + TABLET_ID + " not specified in url.");
+ std::string error_msg = std::string("parameter " + TABLET_ID + " not specified in url.");
return Status::InternalError(error_msg);
}
// Get schema hash
const std::string& schema_hash_str = req->param(SCHEMA_HASH);
if (schema_hash_str.empty()) {
- std::string error_msg = std::string(
- "parameter " + SCHEMA_HASH + " not specified in url.");
+ std::string error_msg = std::string("parameter " + SCHEMA_HASH + " not specified in url.");
return Status::InternalError(error_msg);
}
@@ -97,33 +94,36 @@ Status RestoreTabletAction::_handle(HttpRequest *req) {
// check tablet_id + schema_hash already is restoring
std::lock_guard<std::mutex> l(_tablet_restore_lock);
if (_tablet_path_map.find(key) != _tablet_path_map.end()) {
- LOG(INFO) << "tablet_id:" << tablet_id << " schema_hash:" << schema_hash << " is restoring.";
+ LOG(INFO) << "tablet_id:" << tablet_id << " schema_hash:" << schema_hash
+ << " is restoring.";
return Status::InternalError("tablet is already restoring");
} else {
// set key in map and initialize value as ""
_tablet_path_map[key] = "";
- LOG(INFO) << "start to restore tablet_id:" << tablet_id << " schema_hash:" << schema_hash;
+ LOG(INFO) << "start to restore tablet_id:" << tablet_id
+ << " schema_hash:" << schema_hash;
}
}
Status status = _restore(key, tablet_id, schema_hash);
_clear_key(key);
- LOG(INFO) << "deal with restore tablet request finished! tablet id: " << tablet_id << "-" << schema_hash;
+ LOG(INFO) << "deal with restore tablet request finished! tablet id: " << tablet_id << "-"
+ << schema_hash;
return status;
}
-Status RestoreTabletAction::_reload_tablet(
- const std::string& key, const std::string& shard_path, int64_t tablet_id, int32_t schema_hash) {
+Status RestoreTabletAction::_reload_tablet(const std::string& key, const std::string& shard_path,
+ int64_t tablet_id, int32_t schema_hash) {
TCloneReq clone_req;
clone_req.__set_tablet_id(tablet_id);
clone_req.__set_schema_hash(schema_hash);
OLAPStatus res = OLAPStatus::OLAP_SUCCESS;
res = _exec_env->storage_engine()->load_header(shard_path, clone_req);
if (res != OLAPStatus::OLAP_SUCCESS) {
- LOG(WARNING) << "load header failed. status: " << res
- << ", signature: " << tablet_id;
+ LOG(WARNING) << "load header failed. status: " << res << ", signature: " << tablet_id;
// remove tablet data path in data path
// path: /roo_path/data/shard/tablet_id
- std::string tablet_path = strings::Substitute("$0/$1/$2", shard_path, tablet_id, schema_hash);
+ std::string tablet_path =
+ strings::Substitute("$0/$1/$2", shard_path, tablet_id, schema_hash);
LOG(INFO) << "remove tablet_path:" << tablet_path;
Status s = FileUtils::remove_all(tablet_path);
if (!s.ok()) {
@@ -137,25 +137,26 @@ Status RestoreTabletAction::_reload_tablet(
std::lock_guard<std::mutex> l(_tablet_restore_lock);
trash_tablet_schema_hash_dir = _tablet_path_map[key];
}
- LOG(INFO) << "load header success. status: " << res
- << ", signature: " << tablet_id << ", from trash path:" << trash_tablet_schema_hash_dir
+ LOG(INFO) << "load header success. status: " << res << ", signature: " << tablet_id
+ << ", from trash path:" << trash_tablet_schema_hash_dir
<< " to shard path:" << shard_path;
return Status::OK();
}
}
-Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id, int32_t schema_hash) {
+Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
+ int32_t schema_hash) {
// get latest tablet path in trash
std::string latest_tablet_path;
bool ret = _get_latest_tablet_path_from_trash(tablet_id, schema_hash, &latest_tablet_path);
if (!ret) {
- LOG(WARNING) << "can not find tablet:" << tablet_id
- << ", schema hash:" << schema_hash;
+ LOG(WARNING) << "can not find tablet:" << tablet_id << ", schema hash:" << schema_hash;
return Status::InternalError("can find tablet path in trash");
}
LOG(INFO) << "tablet path in trash:" << latest_tablet_path;
- std::string original_header_path = latest_tablet_path + "/" + std::to_string(tablet_id) +".hdr";
+ std::string original_header_path =
+ latest_tablet_path + "/" + std::to_string(tablet_id) + ".hdr";
TabletMeta tablet_meta;
OLAPStatus load_status = tablet_meta.create_from_file(original_header_path);
if (load_status != OLAP_SUCCESS) {
@@ -169,9 +170,11 @@ Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
_tablet_path_map[key] = latest_tablet_path;
}
- std::string root_path = DataDir::get_root_path_from_schema_hash_path_in_trash(latest_tablet_path);
+ std::string root_path =
+ DataDir::get_root_path_from_schema_hash_path_in_trash(latest_tablet_path);
DataDir* store = StorageEngine::instance()->get_store(root_path);
- std::string restore_schema_hash_path = store->get_absolute_tablet_path(&tablet_meta, true);
+ std::string restore_schema_hash_path = store->get_absolute_tablet_path(
+ tablet_meta.shard_id(), tablet_meta.tablet_id(), tablet_meta.schema_hash());
Status s = FileUtils::create_dir(restore_schema_hash_path);
if (!s.ok()) {
LOG(WARNING) << "create tablet path failed:" << restore_schema_hash_path;
@@ -183,12 +186,13 @@ Status RestoreTabletAction::_restore(const std::string& key, int64_t tablet_id,
RETURN_IF_ERROR(FileUtils::remove_all(restore_schema_hash_path));
return s;
}
- std::string restore_shard_path = store->get_absolute_shard_path(std::to_string(tablet_meta.shard_id()));
+ std::string restore_shard_path = store->get_absolute_shard_path(tablet_meta.shard_id());
Status status = _reload_tablet(key, restore_shard_path, tablet_id, schema_hash);
return status;
}
-Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src, const std::string& dst) {
+Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
+ const std::string& dst) {
std::vector<std::string> files;
RETURN_IF_ERROR(FileUtils::list_files(Env::Default(), src, &files));
for (auto& file : files) {
@@ -200,8 +204,8 @@ Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
} else {
int link_ret = link(from.c_str(), to.c_str());
if (link_ret != 0) {
- LOG(WARNING) << "link from:" << from
- << " to:" << to << " failed, link ret:" << link_ret;
+ LOG(WARNING) << "link from:" << from << " to:" << to
+ << " failed, link ret:" << link_ret;
return Status::InternalError("create link path failed");
}
}
@@ -209,8 +213,8 @@ Status RestoreTabletAction::_create_hard_link_recursive(const std::string& src,
return Status::OK();
}
-bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
- int64_t tablet_id, int32_t schema_hash, std::string* path) {
+bool RestoreTabletAction::_get_latest_tablet_path_from_trash(int64_t tablet_id, int32_t schema_hash,
+ std::string* path) {
std::vector<std::string> tablet_paths;
std::vector<DataDir*> stores = StorageEngine::instance()->get_stores();
for (auto& store : stores) {
@@ -229,8 +233,7 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
}
}
if (schema_hash_paths.size() == 0) {
- LOG(WARNING) << "can not find tablet_id:" << tablet_id
- << ", schema_hash:" << schema_hash;;
+ LOG(WARNING) << "can not find tablet_id:" << tablet_id << ", schema_hash:" << schema_hash;
return false;
} else if (schema_hash_paths.size() == 1) {
*path = schema_hash_paths[0];
@@ -240,9 +243,10 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
uint64_t max_timestamp = 0;
uint64_t max_counter = 0;
*path = schema_hash_paths[start_index];
- if (!_get_timestamp_and_count_from_schema_hash_path(
- schema_hash_paths[start_index], &max_timestamp, &max_counter)) {
- LOG(WARNING) << "schema hash paths are invalid, path:" << schema_hash_paths[start_index];
+ if (!_get_timestamp_and_count_from_schema_hash_path(schema_hash_paths[start_index],
+ &max_timestamp, &max_counter)) {
+ LOG(WARNING) << "schema hash paths are invalid, path:"
+ << schema_hash_paths[start_index];
return false;
}
// find latest path
@@ -250,7 +254,7 @@ bool RestoreTabletAction::_get_latest_tablet_path_from_trash(
uint64_t current_timestamp = 0;
uint64_t current_counter = 0;
if (!_get_timestamp_and_count_from_schema_hash_path(
- schema_hash_paths[i], ¤t_timestamp, ¤t_counter)) {
+ schema_hash_paths[i], ¤t_timestamp, ¤t_counter)) {
LOG(WARNING) << "schema hash path:" << schema_hash_paths[i] << " is invalid";
continue;
}
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 59cc5c3..fbcab3b 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -35,6 +35,7 @@
#include <sstream>
#include "env/env.h"
+#include "gutil/strings/substitute.h"
#include "olap/file_helper.h"
#include "olap/olap_define.h"
#include "olap/olap_snapshot_converter.h"
@@ -397,68 +398,35 @@ OLAPStatus DataDir::get_shard(uint64_t* shard) {
return OLAP_SUCCESS;
}
-OLAPStatus DataDir::register_tablet(Tablet* tablet) {
- std::lock_guard<std::mutex> l(_mutex);
-
+void DataDir::register_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(), tablet->tablet_uid());
- _tablet_set.insert(tablet_info);
- return OLAP_SUCCESS;
-}
-OLAPStatus DataDir::deregister_tablet(Tablet* tablet) {
std::lock_guard<std::mutex> l(_mutex);
+ _tablet_set.emplace(std::move(tablet_info));
+}
+void DataDir::deregister_tablet(Tablet* tablet) {
TabletInfo tablet_info(tablet->tablet_id(), tablet->schema_hash(), tablet->tablet_uid());
+
+ std::lock_guard<std::mutex> l(_mutex);
_tablet_set.erase(tablet_info);
- return OLAP_SUCCESS;
}
void DataDir::clear_tablets(std::vector<TabletInfo>* tablet_infos) {
- for (auto& tablet : _tablet_set) {
- tablet_infos->push_back(tablet);
- }
- _tablet_set.clear();
-}
-
-std::string DataDir::get_absolute_shard_path(const std::string& shard_string) {
- return _path + DATA_PREFIX + "/" + shard_string;
-}
-
-std::string DataDir::get_absolute_tablet_path(TabletMeta* tablet_meta, bool with_schema_hash) {
- if (with_schema_hash) {
- return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
- std::to_string(tablet_meta->tablet_id()) + "/" +
- std::to_string(tablet_meta->schema_hash());
+ std::lock_guard<std::mutex> l(_mutex);
- } else {
- return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
- std::to_string(tablet_meta->tablet_id());
- }
+ tablet_infos->insert(tablet_infos->end(), _tablet_set.begin(), _tablet_set.end());
+ _tablet_set.clear();
}
-std::string DataDir::get_absolute_tablet_path(TabletMetaPB* tablet_meta, bool with_schema_hash) {
- if (with_schema_hash) {
- return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
- std::to_string(tablet_meta->tablet_id()) + "/" +
- std::to_string(tablet_meta->schema_hash());
-
- } else {
- return _path + DATA_PREFIX + "/" + std::to_string(tablet_meta->shard_id()) + "/" +
- std::to_string(tablet_meta->tablet_id());
- }
+std::string DataDir::get_absolute_shard_path(int64_t shard_id) {
+ return strings::Substitute("$0$1/$2", _path, DATA_PREFIX, shard_id);
}
-std::string DataDir::get_absolute_tablet_path(OLAPHeaderMessage& olap_header_msg,
- bool with_schema_hash) {
- if (with_schema_hash) {
- return _path + DATA_PREFIX + "/" + std::to_string(olap_header_msg.shard()) + "/" +
- std::to_string(olap_header_msg.tablet_id()) + "/" +
- std::to_string(olap_header_msg.schema_hash());
-
- } else {
- return _path + DATA_PREFIX + "/" + std::to_string(olap_header_msg.shard()) + "/" +
- std::to_string(olap_header_msg.tablet_id());
- }
+std::string DataDir::get_absolute_tablet_path(int64_t shard_id, int64_t tablet_id,
+ int32_t schema_hash) {
+ return strings::Substitute("$0/$1/$2", get_absolute_shard_path(shard_id), tablet_id,
+ schema_hash);
}
void DataDir::find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths) {
@@ -540,7 +508,9 @@ OLAPStatus DataDir::_convert_old_tablet() {
<< tablet_id << "." << schema_hash;
return false;
}
- string old_data_path_prefix = get_absolute_tablet_path(olap_header_msg, true);
+ string old_data_path_prefix =
+ get_absolute_tablet_path(olap_header_msg.shard_id(), olap_header_msg.tablet_id(),
+ olap_header_msg.schema_hash());
OLAPStatus status = converter.to_new_snapshot(olap_header_msg, old_data_path_prefix,
old_data_path_prefix, &tablet_meta_pb,
&pending_rowsets, true);
@@ -617,7 +587,9 @@ OLAPStatus DataDir::remove_old_meta_and_files() {
TabletSchema tablet_schema;
tablet_schema.init_from_pb(tablet_meta_pb.schema());
- string data_path_prefix = get_absolute_tablet_path(&tablet_meta_pb, true);
+ string data_path_prefix =
+ get_absolute_tablet_path(tablet_meta_pb.shard_id(), tablet_meta_pb.tablet_id(),
+ tablet_meta_pb.schema_hash());
// convert visible pdelta file to rowsets and remove old files
for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
@@ -697,8 +669,8 @@ OLAPStatus DataDir::load() {
// if one rowset load failed, then the total data dir will not be loaded
std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
LOG(INFO) << "begin loading rowset from meta";
- auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
- const std::string& meta_str) -> bool {
+ auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
+ const std::string& meta_str) -> bool {
RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta());
bool parsed = rowset_meta->init(meta_str);
if (!parsed) {
@@ -741,7 +713,7 @@ OLAPStatus DataDir::load() {
LOG(INFO) << "load rowset from meta finished, data dir: " << _path;
}
- // tranverse rowset
+ // traverse rowset
// 1. add committed rowset to txn map
// 2. add visible rowset to tablet
// ignore any errors when load tablet or rowset, because fe will repair them after report
@@ -843,7 +815,7 @@ void DataDir::perform_path_gc_by_rowsetid() {
// gc thread should get tablet include deleted tablet
// or it will delete rowset file before tablet is garbage collected
RowsetId rowset_id;
- bool is_rowset_file = _tablet_manager->get_rowset_id_from_path(path, &rowset_id);
+ bool is_rowset_file = TabletManager::get_rowset_id_from_path(path, &rowset_id);
if (is_rowset_file) {
TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash);
if (tablet != nullptr) {
@@ -863,7 +835,7 @@ void DataDir::perform_path_gc_by_rowsetid() {
void DataDir::perform_path_scan() {
{
std::unique_lock<std::mutex> lck(_check_path_mutex);
- if (_all_check_paths.size() > 0) {
+ if (!_all_check_paths.empty()) {
LOG(INFO) << "_all_check_paths is not empty when path scan.";
return;
}
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index f1c0a8a..c9d2606 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -17,11 +17,11 @@
#pragma once
+#include <condition_variable>
#include <cstdint>
+#include <mutex>
#include <set>
#include <string>
-#include <mutex>
-#include <condition_variable>
#include "common/status.h"
#include "gen_cpp/Types_types.h"
@@ -37,15 +37,13 @@ class TabletManager;
class TabletMeta;
class TxnManager;
-// A DataDir used to manange data in same path.
+// A DataDir used to manage data in same path.
// Now, After DataDir was created, it will never be deleted for easy implementation.
class DataDir {
public:
- DataDir(const std::string& path,
- int64_t capacity_bytes = -1,
+ DataDir(const std::string& path, int64_t capacity_bytes = -1,
TStorageMedium::type storage_medium = TStorageMedium::HDD,
- TabletManager* tablet_manager = nullptr,
- TxnManager* txn_manager = nullptr);
+ TabletManager* tablet_manager = nullptr, TxnManager* txn_manager = nullptr);
~DataDir();
Status init();
@@ -78,27 +76,21 @@ public:
OlapMeta* get_meta() { return _meta; }
- bool is_ssd_disk() const {
- return _storage_medium == TStorageMedium::SSD;
- }
+ bool is_ssd_disk() const { return _storage_medium == TStorageMedium::SSD; }
TStorageMedium::type storage_medium() const { return _storage_medium; }
- OLAPStatus register_tablet(Tablet* tablet);
- OLAPStatus deregister_tablet(Tablet* tablet);
+ void register_tablet(Tablet* tablet);
+ void deregister_tablet(Tablet* tablet);
void clear_tablets(std::vector<TabletInfo>* tablet_infos);
- std::string get_absolute_tablet_path(TabletMeta* tablet_meta, bool with_schema_hash);
-
- std::string get_absolute_tablet_path(OLAPHeaderMessage& olap_header_msg, bool with_schema_hash);
-
- std::string get_absolute_tablet_path(TabletMetaPB* tablet_meta, bool with_schema_hash);
-
- std::string get_absolute_shard_path(const std::string& shard_string);
+ std::string get_absolute_shard_path(int64_t shard_id);
+ std::string get_absolute_tablet_path(int64_t shard_id, int64_t tablet_id, int32_t schema_hash);
void find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths);
- static std::string get_root_path_from_schema_hash_path_in_trash(const std::string& schema_hash_dir_in_trash);
+ static std::string get_root_path_from_schema_hash_path_in_trash(
+ const std::string& schema_hash_dir_in_trash);
// load data from meta and data files
OLAPStatus load();
@@ -122,8 +114,8 @@ public:
// check if the capacity reach the limit after adding the incoming data
// return true if limit reached, otherwise, return false.
// TODO(cmy): for now we can not precisely calculate the capacity Doris used,
- // so in order to avoid running out of disk capacity, we currenty use the actual
- // disk avaiable capacity and total capacity to do the calculation.
+ // so in order to avoid running out of disk capacity, we currently use the actual
+ // disk available capacity and total capacity to do the calculation.
// So that the capacity Doris actually used may exceeds the user specified capacity.
bool reach_capacity_limit(int64_t incoming_data_size);
@@ -186,15 +178,16 @@ private:
static const uint32_t MAX_SHARD_NUM = 1024;
char* _test_file_read_buf;
char* _test_file_write_buf;
+
OlapMeta* _meta = nullptr;
RowsetIdGenerator* _id_generator = nullptr;
- std::set<std::string> _all_check_paths;
std::mutex _check_path_mutex;
std::condition_variable _cv;
+ std::set<std::string> _all_check_paths;
- std::set<std::string> _pending_path_ids;
RWMutex _pending_path_mutex;
+ std::set<std::string> _pending_path_ids;
// used in convert process
bool _convert_old_data_success;
diff --git a/be/src/olap/olap_snapshot_converter.cpp b/be/src/olap/olap_snapshot_converter.cpp
index 2986f16..77c4788 100755
--- a/be/src/olap/olap_snapshot_converter.cpp
+++ b/be/src/olap/olap_snapshot_converter.cpp
@@ -89,7 +89,7 @@ OLAPStatus OlapSnapshotConverter::to_olap_header(const TabletMetaPB& tablet_meta
olap_header->set_schema_hash(tablet_meta_pb.schema_hash());
}
if (tablet_meta_pb.has_shard_id()) {
- olap_header->set_shard(tablet_meta_pb.shard_id());
+ olap_header->set_shard_id(tablet_meta_pb.shard_id());
}
return OLAP_SUCCESS;
}
@@ -102,8 +102,8 @@ OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage& ola
if (olap_header.has_schema_hash()) {
tablet_meta_pb->set_schema_hash(olap_header.schema_hash());
}
- if (olap_header.has_shard()) {
- tablet_meta_pb->set_shard_id(olap_header.shard());
+ if (olap_header.has_shard_id()) {
+ tablet_meta_pb->set_shard_id(olap_header.shard_id());
}
tablet_meta_pb->set_creation_time(olap_header.creation_time());
tablet_meta_pb->set_cumulative_layer_point(-1);
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index f192181..bb64f0f 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -59,8 +59,8 @@ public:
bool is_used();
inline DataDir* data_dir() const;
- OLAPStatus register_tablet_into_dir();
- OLAPStatus deregister_tablet_from_dir();
+ void register_tablet_into_dir();
+ void deregister_tablet_from_dir();
std::string tablet_path() const;
@@ -324,12 +324,12 @@ inline DataDir* Tablet::data_dir() const {
return _data_dir;
}
-inline OLAPStatus Tablet::register_tablet_into_dir() {
- return _data_dir->register_tablet(this);
+inline void Tablet::register_tablet_into_dir() {
+ _data_dir->register_tablet(this);
}
-inline OLAPStatus Tablet::deregister_tablet_from_dir() {
- return _data_dir->deregister_tablet(this);
+inline void Tablet::deregister_tablet_from_dir() {
+ _data_dir->deregister_tablet(this);
}
inline string Tablet::tablet_path() const {
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index c31ec5f..7367a20 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -184,9 +184,7 @@ OLAPStatus TabletManager::_add_tablet_to_map_unlocked(TTabletId tablet_id, Schem
// Register tablet into DataDir, so that we can manage tablet from
// the perspective of root path.
// Example: unregister all tables when a bad disk found.
- RETURN_NOT_OK_LOG(tablet->register_tablet_into_dir(), Substitute(
- "fail to register tablet into StorageEngine. data_dir=$0",
- tablet->data_dir()->path()));
+ tablet->register_tablet_into_dir();
tablet_map_t& tablet_map = _get_tablet_map(tablet_id);
tablet_map[tablet_id].table_arr.push_back(tablet);
tablet_map[tablet_id].table_arr.sort(_cmp_tablet_by_create_time);
@@ -1337,9 +1335,7 @@ OLAPStatus TabletManager::_drop_tablet_directly_unlocked(
}
}
- RETURN_NOT_OK_LOG(dropped_tablet->deregister_tablet_from_dir(), Substitute(
- "fail to unregister from root path. tablet=$0, schema_hash=$1",
- tablet_id, schema_hash));
+ dropped_tablet->deregister_tablet_from_dir();
return OLAP_SUCCESS;
}
diff --git a/be/test/olap/olap_snapshot_converter_test.cpp b/be/test/olap/olap_snapshot_converter_test.cpp
index f554499..40abeb2 100644
--- a/be/test/olap/olap_snapshot_converter_test.cpp
+++ b/be/test/olap/olap_snapshot_converter_test.cpp
@@ -15,24 +15,25 @@
// specific language governing permissions and limitations
// under the License.
+#include "olap/olap_snapshot_converter.h"
+
+#include <boost/algorithm/string.hpp>
+#include <fstream>
#include <iostream>
-#include <string>
#include <sstream>
-#include <fstream>
+#include <string>
-#include "gtest/gtest.h"
+#include "boost/filesystem.hpp"
#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "json2pb/json_to_pb.h"
#include "olap/lru_cache.h"
#include "olap/olap_meta.h"
-#include "olap/olap_snapshot_converter.h"
-#include "olap/rowset/rowset_meta_manager.h"
#include "olap/rowset/alpha_rowset.h"
#include "olap/rowset/alpha_rowset_meta.h"
+#include "olap/rowset/rowset_meta_manager.h"
#include "olap/storage_engine.h"
#include "olap/txn_manager.h"
-#include <boost/algorithm/string.hpp>
-#include "boost/filesystem.hpp"
-#include "json2pb/json_to_pb.h"
#include "util/file_utils.h"
#ifndef BE_TEST
@@ -77,16 +78,14 @@ public:
copy_dir(test_engine_data_path, tmp_data_path);
_tablet_id = 15007;
_schema_hash = 368169781;
- _tablet_data_path = tmp_data_path
- + "/" + std::to_string(0)
- + "/" + std::to_string(_tablet_id)
- + "/" + std::to_string(_schema_hash);
+ _tablet_data_path = tmp_data_path + "/" + std::to_string(0) + "/" +
+ std::to_string(_tablet_id) + "/" + std::to_string(_schema_hash);
if (boost::filesystem::exists(_meta_path)) {
boost::filesystem::remove_all(_meta_path);
}
ASSERT_TRUE(boost::filesystem::create_directory(_meta_path));
ASSERT_TRUE(boost::filesystem::exists(_meta_path));
- _meta = new(std::nothrow) OlapMeta(_meta_path);
+ _meta = new (std::nothrow) OlapMeta(_meta_path);
ASSERT_NE(nullptr, _meta);
OLAPStatus st = _meta->init();
ASSERT_TRUE(st == OLAP_SUCCESS);
@@ -131,12 +130,13 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
TabletMetaPB tablet_meta_pb;
vector<RowsetMetaPB> pending_rowsets;
OLAPStatus status = converter.to_new_snapshot(header_msg, _tablet_data_path, _tablet_data_path,
- &tablet_meta_pb, &pending_rowsets, true);
+ &tablet_meta_pb, &pending_rowsets, true);
ASSERT_TRUE(status == OLAP_SUCCESS);
TabletSchema tablet_schema;
tablet_schema.init_from_pb(tablet_meta_pb.schema());
- string data_path_prefix = _data_dir->get_absolute_tablet_path(&tablet_meta_pb, true);
+ string data_path_prefix = _data_dir->get_absolute_tablet_path(
+ tablet_meta_pb.shard_id(), tablet_meta_pb.tablet_id(), tablet_meta_pb.schema_hash());
// check converted new tabletmeta pb and its files
// check visible delta
ASSERT_TRUE(tablet_meta_pb.rs_metas().size() == header_msg.delta().size());
@@ -145,8 +145,8 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t end_version = pdelta.end_version();
bool found = false;
for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
- if (visible_rowset.start_version() == start_version
- && visible_rowset.end_version() == end_version) {
+ if (visible_rowset.start_version() == start_version &&
+ visible_rowset.end_version() == end_version) {
found = true;
}
}
@@ -169,9 +169,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t version_hash = pdelta.version_hash();
bool found = false;
for (auto& inc_rowset : tablet_meta_pb.inc_rs_metas()) {
- if (inc_rowset.start_version() == start_version
- && inc_rowset.end_version() == end_version
- && inc_rowset.version_hash() == version_hash) {
+ if (inc_rowset.start_version() == start_version &&
+ inc_rowset.end_version() == end_version &&
+ inc_rowset.version_hash() == version_hash) {
found = true;
}
}
@@ -184,7 +184,7 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
ASSERT_TRUE(rowset.init() == OLAP_SUCCESS);
ASSERT_TRUE(rowset.load() == OLAP_SUCCESS);
AlphaRowset tmp_rowset(&tablet_schema, data_path_prefix + "/incremental_delta",
- alpha_rowset_meta);
+ alpha_rowset_meta);
ASSERT_TRUE(tmp_rowset.init() == OLAP_SUCCESS);
std::vector<std::string> old_files;
tmp_rowset.remove_old_files(&old_files);
@@ -196,10 +196,10 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
int64_t transaction_id = pdelta.transaction_id();
bool found = false;
for (auto& pending_rowset : pending_rowsets) {
- if (pending_rowset.partition_id() == partition_id
- && pending_rowset.txn_id() == transaction_id
- && pending_rowset.tablet_uid().hi() == tablet_meta_pb.tablet_uid().hi()
- && pending_rowset.tablet_uid().lo() == tablet_meta_pb.tablet_uid().lo()) {
+ if (pending_rowset.partition_id() == partition_id &&
+ pending_rowset.txn_id() == transaction_id &&
+ pending_rowset.tablet_uid().hi() == tablet_meta_pb.tablet_uid().hi() &&
+ pending_rowset.tablet_uid().lo() == tablet_meta_pb.tablet_uid().lo()) {
found = true;
}
}
@@ -217,14 +217,14 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
// old files are removed, then convert new snapshot to old snapshot
OLAPHeaderMessage old_header_msg;
- status = converter.to_old_snapshot(tablet_meta_pb, _tablet_data_path,
- _tablet_data_path, &old_header_msg);
+ status = converter.to_old_snapshot(tablet_meta_pb, _tablet_data_path, _tablet_data_path,
+ &old_header_msg);
ASSERT_TRUE(status == OLAP_SUCCESS);
for (auto& pdelta : header_msg.delta()) {
bool found = false;
for (auto& converted_pdelta : old_header_msg.delta()) {
- if (converted_pdelta.start_version() == pdelta.start_version()
- && converted_pdelta.end_version() == pdelta.end_version()) {
+ if (converted_pdelta.start_version() == pdelta.start_version() &&
+ converted_pdelta.end_version() == pdelta.end_version()) {
found = true;
}
}
@@ -233,9 +233,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
for (auto& pdelta : header_msg.incremental_delta()) {
bool found = false;
for (auto& converted_pdelta : old_header_msg.incremental_delta()) {
- if (converted_pdelta.start_version() == pdelta.start_version()
- && converted_pdelta.end_version() == pdelta.end_version()
- && converted_pdelta.version_hash() == pdelta.version_hash()) {
+ if (converted_pdelta.start_version() == pdelta.start_version() &&
+ converted_pdelta.end_version() == pdelta.end_version() &&
+ converted_pdelta.version_hash() == pdelta.version_hash()) {
found = true;
}
}
@@ -243,9 +243,9 @@ TEST_F(OlapSnapshotConverterTest, ToNewAndToOldSnapshot) {
}
}
-} // namespace doris
+} // namespace doris
-int main(int argc, char **argv) {
+int main(int argc, char** argv) {
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index e95ef77..e58e67f 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -211,7 +211,7 @@ message OLAPHeaderMessage {
optional bool in_restore_mode = 19 [default = false]; // TabletMetaPB.is_restore_mode
optional int64 tablet_id = 20; // TabletMetaPB.tablet_id
optional int32 schema_hash = 21; // TabletMetaPB.schema_hash? int32 vs int64
- optional uint64 shard = 22; // TabletMetaPB.shard_id? int64 vs int32
+ optional uint64 shard_id = 22; // TabletMetaPB.shard_id? int64 vs int32
}
enum AlterTabletState {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org