You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by zh...@apache.org on 2019/10/11 09:21:45 UTC
[incubator-doris] branch master updated: Fix some warning when
compile type is debug using -Werror flag(because of use deprecated
funcrtions) (#1953)
This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 948f497 Fix some warning when compile type is debug using -Werror flag(because of use deprecated funcrtions) (#1953)
948f497 is described below
commit 948f497dd408b891aa0baa7964ab834d6fa4747d
Author: yangzhg <78...@qq.com>
AuthorDate: Fri Oct 11 17:21:41 2019 +0800
Fix some warning when compile type is debug using -Werror flag(because of use deprecated funcrtions) (#1953)
---
be/src/agent/cgroups_mgr.cpp | 173 ++++++++++++++++++++++---------------------
be/src/olap/data_dir.cpp | 63 ++++++----------
be/src/olap/data_dir.h | 5 +-
be/src/util/disk_info.cpp | 1 +
4 files changed, 112 insertions(+), 130 deletions(-)
diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp
index 8a2faac..9495271 100644
--- a/be/src/agent/cgroups_mgr.cpp
+++ b/be/src/agent/cgroups_mgr.cpp
@@ -18,12 +18,13 @@
#include "agent/cgroups_mgr.h"
#include <fstream>
#include <future>
-#include <linux/magic.h>
+#include <linux/magic.h>
#include <map>
#include <unistd.h>
#include <asm/unistd.h>
#include <sstream>
#include <sys/stat.h>
+#include <sys/sysmacros.h>
#include <sys/vfs.h>
#include "boost/filesystem.hpp"
#include "common/logging.h"
@@ -46,12 +47,12 @@ static CgroupsMgr *s_global_cg_mgr;
const std::string CgroupsMgr::_s_system_user = "system";
const std::string CgroupsMgr::_s_system_group = "normal";
-std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups =
- {{TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"},
+std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups =
+ {{TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"},
{TResourceType::type::TRESOURCE_IO_SHARE, "blkio.weight"}};
-CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path)
- : _exec_env(exec_env),
+CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path)
+ : _exec_env(exec_env),
_root_cgroups_path(root_cgroups_path),
_is_cgroups_init_success(false),
_cur_version(-1) {
@@ -64,31 +65,31 @@ CgroupsMgr::~CgroupsMgr() {
}
AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) {
-
+
std::lock_guard<std::mutex> lck(_update_cgroups_mtx);
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
}
-
-
+
+
if (_cur_version >= new_fetched_resource.resourceVersion) {
return AgentStatus::DORIS_SUCCESS;
}
-
- const std::map<std::string, TUserResource>& new_user_resource
+
+ const std::map<std::string, TUserResource>& new_user_resource
= new_fetched_resource.resourceByUser;
-
+
if (!_local_users.empty()) {
std::set<std::string>::const_iterator old_it = _local_users.begin();
for (; old_it != _local_users.end(); ++old_it) {
if (new_user_resource.count(*old_it) == 0) {
this->delete_user_cgroups(*old_it);
- }
+ }
}
}
// Clear local users set, because it will be inserted again
- _local_users.clear();
+ _local_users.clear();
std::map<std::string, TUserResource>::const_iterator new_it = new_user_resource.begin();
for (; new_it != new_user_resource.end(); ++new_it) {
@@ -102,7 +103,7 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe
if (_s_resource_cgroups.count(resource_it->first) > 0) {
user_share[_s_resource_cgroups[resource_it->first]] =
resource_it->second;
- }
+ }
}
modify_user_cgroups(user_name, user_share, level_share);
@@ -111,46 +112,46 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe
// Insert user to local user's set
_local_users.insert(user_name);
}
-
+
// Using resource version, not subscribe version
_cur_version = new_fetched_resource.resourceVersion;
return AgentStatus::DORIS_SUCCESS;
}
-void CgroupsMgr::_config_user_disk_throttle(std::string user_name,
+void CgroupsMgr::_config_user_disk_throttle(std::string user_name,
const std::map<TResourceType::type, int32_t>& resource_share) {
- int64_t hdd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS,
+ int64_t hdd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS,
resource_share);
- int64_t hdd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS,
+ int64_t hdd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS,
resource_share);
- int64_t hdd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS,
+ int64_t hdd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS,
resource_share);
- int64_t hdd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS,
+ int64_t hdd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS,
resource_share);
- int64_t ssd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS,
+ int64_t ssd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS,
resource_share);
- int64_t ssd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS,
+ int64_t ssd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS,
resource_share);
- int64_t ssd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS,
+ int64_t ssd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS,
resource_share);
- int64_t ssd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS,
+ int64_t ssd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS,
resource_share);
-
- _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
+
+ _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops,
+ hdd_read_mbps, hdd_write_mbps,
+ ssd_read_iops, ssd_write_iops,
ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
+ _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops,
+ hdd_read_mbps, hdd_write_mbps,
+ ssd_read_iops, ssd_write_iops,
ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
+ _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops,
+ hdd_read_mbps, hdd_write_mbps,
+ ssd_read_iops, ssd_write_iops,
ssd_read_mbps, ssd_write_mbps);
- _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops,
- hdd_read_mbps, hdd_write_mbps,
- ssd_read_iops, ssd_write_iops,
+ _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops,
+ hdd_read_mbps, hdd_write_mbps,
+ ssd_read_iops, ssd_write_iops,
ssd_read_mbps, ssd_write_mbps);
}
@@ -166,32 +167,32 @@ int64_t CgroupsMgr::_get_resource_value(const TResourceType::type resource_type,
AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
std::string level,
- int64_t hdd_read_iops,
+ int64_t hdd_read_iops,
int64_t hdd_write_iops,
- int64_t hdd_read_mbps,
- int64_t hdd_write_mbps,
- int64_t ssd_read_iops,
- int64_t ssd_write_iops,
- int64_t ssd_read_mbps,
+ int64_t hdd_read_mbps,
+ int64_t hdd_write_mbps,
+ int64_t ssd_read_iops,
+ int64_t ssd_write_iops,
+ int64_t ssd_read_mbps,
int64_t ssd_write_mbps) {
- string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level;
+ string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level;
string read_bps_path = cgroups_path + "/blkio.throttle.read_bps_device";
string write_bps_path = cgroups_path + "/blkio.throttle.write_bps_device";
string read_iops_path = cgroups_path + "/blkio.throttle.read_iops_device";
string write_iops_path = cgroups_path + "/blkio.throttle.write_iops_device";
- if (!is_file_exist(cgroups_path.c_str())) {
+ if (!is_file_exist(cgroups_path.c_str())) {
if (!boost::filesystem::create_directory(cgroups_path)) {
LOG(ERROR) << "Create cgroups: " << cgroups_path << " failed";
return AgentStatus::DORIS_ERROR;
}
}
-
+
// add olap engine data path here
auto stores = StorageEngine::instance()->get_stores();
// buld load data path, it is alreay in data path
// _exec_env->load_path_mgr()->get_load_data_path(&data_paths);
-
+
stringstream ctrl_cmd;
for (auto store : stores) {
// check disk type
@@ -215,7 +216,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
minor_number = (minor_number / 16) * 16;
if (read_iops != -1) {
ctrl_cmd << major_number << ":"
- << minor_number << " "
+ << minor_number << " "
<< read_iops;
_echo_cmd_to_cgroup(ctrl_cmd, read_iops_path);
ctrl_cmd.clear();
@@ -223,7 +224,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
}
if (write_iops != -1) {
ctrl_cmd << major_number << ":"
- << minor_number << " "
+ << minor_number << " "
<< write_iops;
_echo_cmd_to_cgroup(ctrl_cmd, write_iops_path);
ctrl_cmd.clear();
@@ -231,7 +232,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
}
if (read_mbps != -1) {
ctrl_cmd << major_number << ":"
- << minor_number << " "
+ << minor_number << " "
<< (read_mbps << 20);
_echo_cmd_to_cgroup(ctrl_cmd, read_bps_path);
ctrl_cmd.clear();
@@ -239,7 +240,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
}
if (write_mbps != -1) {
ctrl_cmd << major_number << ":"
- << minor_number << " "
+ << minor_number << " "
<< (write_mbps << 20);
_echo_cmd_to_cgroup(ctrl_cmd, write_bps_path);
ctrl_cmd.clear();
@@ -249,12 +250,12 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name,
return AgentStatus::DORIS_SUCCESS;
}
-AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
- const map<string, int32_t>& user_share,
+AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
+ const map<string, int32_t>& user_share,
const map<string, int32_t>& level_share) {
// Check if the user's cgroups exists, if not create it
- string user_cgroups_path = this->_root_cgroups_path + "/" + user_name;
- if (!is_file_exist(user_cgroups_path.c_str())) {
+ string user_cgroups_path = this->_root_cgroups_path + "/" + user_name;
+ if (!is_file_exist(user_cgroups_path.c_str())) {
if (!boost::filesystem::create_directory(user_cgroups_path)) {
LOG(ERROR) << "Create cgroups for user " << user_name << " failed";
return AgentStatus::DORIS_ERROR;
@@ -262,7 +263,7 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
}
// Traverse the user resource share map to append share value to cgroup's file
- for (map<string, int32_t>::const_iterator user_resource = user_share.begin();
+ for (map<string, int32_t>::const_iterator user_resource = user_share.begin();
user_resource != user_share.end(); ++user_resource){
string resource_file_name = user_resource->first;
int32_t user_share_weight = user_resource->second;
@@ -271,11 +272,11 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
std::ofstream user_cgroups(user_resource_path.c_str(), std::ios::out | std::ios::app);
if (!user_cgroups.is_open()) {
return AgentStatus::DORIS_ERROR;
- }
+ }
user_cgroups << user_share_weight << std::endl;
user_cgroups.close();
LOG(INFO) << "Append " << user_share_weight << " to " << user_resource_path;
- for (map<string, int32_t>::const_iterator level_resource = level_share.begin();
+ for (map<string, int32_t>::const_iterator level_resource = level_share.begin();
level_resource != level_share.end(); ++level_resource){
// Append resource share to level shares
string level_name = level_resource->first;
@@ -288,9 +289,9 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
}
}
- // Append the share_weight value to the file
+ // Append the share_weight value to the file
string level_resource_path = level_cgroups_path + "/" + resource_file_name;
- std::ofstream level_cgroups(level_resource_path.c_str(),
+ std::ofstream level_cgroups(level_resource_path.c_str(),
std::ios::out | std::ios::app);
if (!level_cgroups.is_open()) {
return AgentStatus::DORIS_ERROR;
@@ -299,7 +300,7 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name,
level_cgroups.close();
LOG(INFO) << "Append " << level_share_weight << " to " << level_resource_path;
- }
+ }
}
return AgentStatus::DORIS_SUCCESS;
}
@@ -316,14 +317,14 @@ AgentStatus CgroupsMgr::init_cgroups() {
if (fs_type.f_type != CGROUP_SUPER_MAGIC) {
LOG(ERROR) << _root_cgroups_path << " is not a cgroups file system.";
_is_cgroups_init_success = false;
- return AgentStatus::DORIS_ERROR;
+ return AgentStatus::DORIS_ERROR;
}
#endif
// Check if current user have write permission to cgroup folder
if (access(_root_cgroups_path.c_str(), W_OK) != 0) {
- LOG(ERROR) << "Doris does not have write permission to "
+ LOG(ERROR) << "Doris does not have write permission to "
<< _root_cgroups_path;
- _is_cgroups_init_success = false;
+ _is_cgroups_init_success = false;
return AgentStatus::DORIS_ERROR;
}
// If root folder exists, then delete all subfolders under it
@@ -332,10 +333,10 @@ AgentStatus CgroupsMgr::init_cgroups() {
for (; item_begin != item_end; item_begin++) {
if (is_directory(item_begin->path().string().c_str())) {
// Delete the sub folder
- if (delete_user_cgroups(item_begin->path().filename().string())
+ if (delete_user_cgroups(item_begin->path().filename().string())
!= AgentStatus::DORIS_SUCCESS) {
- LOG(ERROR) << "Could not clean subfolder "
- << item_begin->path().string();
+ LOG(ERROR) << "Could not clean subfolder "
+ << item_begin->path().string();
_is_cgroups_init_success = false;
return AgentStatus::DORIS_ERROR;
}
@@ -346,11 +347,11 @@ AgentStatus CgroupsMgr::init_cgroups() {
_is_cgroups_init_success = true;
return AgentStatus::DORIS_SUCCESS;
} else {
- LOG(ERROR) << "Could not find a valid cgroups path for resource isolation,"
+ LOG(ERROR) << "Could not find a valid cgroups path for resource isolation,"
<< "current value is " << _root_cgroups_path;
_is_cgroups_init_success = false;
return AgentStatus::DORIS_ERROR;
- }
+ }
}
#define gettid() syscall(__NR_gettid)
@@ -361,7 +362,7 @@ void CgroupsMgr::apply_cgroup(const string& user_name, const string& level) {
s_global_cg_mgr->assign_to_cgroups(user_name, level);
}
-AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name,
+AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name,
const string& level) {
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
@@ -370,16 +371,16 @@ AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name,
return assign_thread_to_cgroups(tid, user_name, level);
}
-AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id,
- const string& user_name,
+AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id,
+ const string& user_name,
const string& level) {
if (!_is_cgroups_init_success) {
return AgentStatus::DORIS_ERROR;
}
string tasks_path = _root_cgroups_path + "/" + user_name + "/" + level + "/tasks";
if (!is_file_exist(_root_cgroups_path + "/" + user_name)) {
- tasks_path = this->_root_cgroups_path + "/"
- + _default_user_name + "/"
+ tasks_path = this->_root_cgroups_path + "/"
+ + _default_user_name + "/"
+ _default_level + "/tasks";
} else if (!is_file_exist(_root_cgroups_path + "/" + user_name + "/" + level)) {
tasks_path = this->_root_cgroups_path + "/" + user_name + "/tasks";
@@ -387,7 +388,7 @@ AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id,
if (!is_file_exist(tasks_path.c_str())) {
LOG(ERROR) << "Cgroups path " << tasks_path << " not exist!";
return AgentStatus::DORIS_ERROR;
- }
+ }
std::ofstream tasks(tasks_path.c_str(), std::ios::out | std::ios::app);
if (!tasks.is_open()) {
// This means doris could not open this file. May be it does not have access to it
@@ -419,25 +420,25 @@ AgentStatus CgroupsMgr::delete_user_cgroups(const string& user_name) {
if (this->drop_cgroups(user_cgroups_path) < 0) {
return AgentStatus::DORIS_ERROR;
}
- }
+ }
return AgentStatus::DORIS_SUCCESS;
}
AgentStatus CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) {
// Try to delete the cgroups folder
- // If failed then there maybe exist active tasks under it and try to relocate them
- // Currently, try 10 times to relocate and delete the cgroups.
- int32_t i = 0;
- while (is_file_exist(deleted_cgroups_path)
- && rmdir(deleted_cgroups_path.c_str()) < 0
- && i < this->_drop_retry_times) {
+ // If failed then there maybe exist active tasks under it and try to relocate them
+ // Currently, try 10 times to relocate and delete the cgroups.
+ int32_t i = 0;
+ while (is_file_exist(deleted_cgroups_path)
+ && rmdir(deleted_cgroups_path.c_str()) < 0
+ && i < this->_drop_retry_times) {
this->relocate_tasks(deleted_cgroups_path, this->_root_cgroups_path);
++i;
#ifdef BE_TEST
boost::filesystem::remove_all(deleted_cgroups_path);
#endif
if (i == this->_drop_retry_times){
- LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path
+ LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path
<< " failed.";
return AgentStatus::DORIS_ERROR;
}
@@ -454,7 +455,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string&
}
std::ofstream dest_tasks(dest_tasks_path.c_str(), std::ios::out | std::ios::app);
if (!dest_tasks) {
- return AgentStatus::DORIS_ERROR;
+ return AgentStatus::DORIS_ERROR;
}
int64_t taskid;
while (src_tasks >> taskid) {
@@ -469,7 +470,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string&
}
void CgroupsMgr::_echo_cmd_to_cgroup(stringstream& ctrl_cmd, string& cgroups_path) {
- std::ofstream cgroups_stream(cgroups_path.c_str(),
+ std::ofstream cgroups_stream(cgroups_path.c_str(),
std::ios::out | std::ios::app);
if (cgroups_stream.is_open()) {
cgroups_stream << ctrl_cmd.str() << std::endl;
@@ -489,7 +490,7 @@ bool CgroupsMgr::is_directory(const char* file_path) {
}
}
-bool CgroupsMgr::is_file_exist(const char* file_path) {
+bool CgroupsMgr::is_file_exist(const char* file_path) {
struct stat file_stat;
if (stat(file_path, &file_stat) != 0) {
return false;
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index f86f21f..dba6d76 100755
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -92,7 +92,10 @@ Status DataDir::init() {
LOG(WARNING) << "fail to allocate memory. size=" << TEST_FILE_BUF_SIZE;
return Status::InternalError("No memory");
}
- RETURN_IF_ERROR(_check_path_exist());
+ if (!check_dir_existed(_path)) {
+ LOG(WARNING) << "opendir failed, path=" << _path;
+ return Status::InternalError("opendir failed");
+ }
std::string align_tag_path = _path + ALIGN_TAG_PREFIX;
if (access(align_tag_path.c_str(), F_OK) == 0) {
LOG(WARNING) << "align tag was found, path=" << _path;
@@ -109,28 +112,6 @@ Status DataDir::init() {
return Status::OK();
}
-Status DataDir::_check_path_exist() {
- DIR* dirp = opendir(_path.c_str());
- if (dirp == nullptr) {
- char buf[64];
- LOG(WARNING) << "opendir failed, path=" << _path
- << ", errno=" << errno << ", errmsg=" << strerror_r(errno, buf, 64);
- return Status::InternalError("opendir failed");
- }
- struct dirent dirent;
- struct dirent* result = nullptr;
- if (readdir_r(dirp, &dirent, &result) != 0) {
- char buf[64];
- LOG(WARNING) << "readdir failed, path=" << _path
- << ", errno=" << errno << ", errmsg=" << strerror_r(errno, buf, 64);
- closedir(dirp);
- return Status::InternalError("readdir failed");
- }
- // opendir and closedir should be called both or not.
- closedir(dirp);
- return Status::OK();
-}
-
Status DataDir::_init_cluster_id() {
std::string cluster_id_path = _path + CLUSTER_ID_PREFIX;
if (access(cluster_id_path.c_str(), F_OK) != 0) {
@@ -566,7 +547,7 @@ OLAPStatus DataDir::_convert_old_tablet() {
OLAPStatus status = converter.to_new_snapshot(olap_header_msg, old_data_path_prefix,
old_data_path_prefix, *this, &tablet_meta_pb, &pending_rowsets, true);
if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet="
+ LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet="
<< tablet_id << "." << schema_hash;
return false;
}
@@ -577,7 +558,7 @@ OLAPStatus DataDir::_convert_old_tablet() {
rowset_id.init(rowset_pb.rowset_id_v2());
status = RowsetMetaManager::save(_meta, rowset_pb.tablet_uid(), rowset_id, rowset_pb);
if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when save rowset meta tablet="
+ LOG(FATAL) << "convert olap header to tablet meta failed when save rowset meta tablet="
<< tablet_id << "." << schema_hash;
return false;
}
@@ -588,16 +569,16 @@ OLAPStatus DataDir::_convert_old_tablet() {
tablet_meta_pb.SerializeToString(&meta_binary);
status = TabletMetaManager::save(this, tablet_meta_pb.tablet_id(), tablet_meta_pb.schema_hash(), meta_binary);
if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when save tablet meta tablet="
+ LOG(FATAL) << "convert olap header to tablet meta failed when save tablet meta tablet="
<< tablet_id << "." << schema_hash;
return false;
} else {
- LOG(INFO) << "convert olap header to tablet meta successfully and save tablet meta to meta tablet="
+ LOG(INFO) << "convert olap header to tablet meta successfully and save tablet meta to meta tablet="
<< tablet_id << "." << schema_hash;
}
return true;
};
- OLAPStatus convert_tablet_status = TabletMetaManager::traverse_headers(_meta,
+ OLAPStatus convert_tablet_status = TabletMetaManager::traverse_headers(_meta,
convert_tablet_func, OLD_HEADER_PREFIX);
if (convert_tablet_status != OLAP_SUCCESS) {
LOG(FATAL) << "there is failure when convert old tablet, data dir:" << _path;
@@ -609,7 +590,7 @@ OLAPStatus DataDir::_convert_old_tablet() {
}
OLAPStatus DataDir::remove_old_meta_and_files() {
- // clean old meta(olap header message)
+ // clean old meta(olap header message)
auto clean_old_meta_files_func = [this](int64_t tablet_id,
int32_t schema_hash, const std::string& value) -> bool {
// convert olap header and files
@@ -625,7 +606,7 @@ OLAPStatus DataDir::remove_old_meta_and_files() {
OlapSnapshotConverter converter;
OLAPStatus status = converter.to_tablet_meta_pb(olap_header_msg, &tablet_meta_pb, &pending_rowsets);
if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet="
+ LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet="
<< tablet_id << "." << schema_hash;
return true;
}
@@ -672,7 +653,7 @@ OLAPStatus DataDir::remove_old_meta_and_files() {
}
TabletMetaManager::remove(this, tablet_id, schema_hash, OLD_HEADER_PREFIX);
- LOG(INFO) << "successfully clean old tablet meta(olap header) for tablet="
+ LOG(INFO) << "successfully clean old tablet meta(olap header) for tablet="
<< tablet_id << "." << schema_hash
<< " tablet_path=" << data_path_prefix;
@@ -733,7 +714,7 @@ OLAPStatus DataDir::load() {
// if one rowset load failed, then the total data dir will not be loaded
std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
LOG(INFO) << "begin loading rowset from meta";
- auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
+ auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id,
const std::string& meta_str) -> bool {
RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta());
@@ -777,7 +758,7 @@ OLAPStatus DataDir::load() {
LOG(INFO) << "load rowset from meta finished, data dir: " << _path;
}
- // tranverse rowset
+ // tranverse rowset
// 1. add committed rowset to txn map
// 2. add visible rowset to tablet
// ignore any errors when load tablet or rowset, because fe will repair them after report
@@ -808,20 +789,20 @@ OLAPStatus DataDir::load() {
&& rowset_meta->tablet_uid() == tablet->tablet_uid()) {
OLAPStatus commit_txn_status = _txn_manager->commit_txn(
_meta,
- rowset_meta->partition_id(), rowset_meta->txn_id(),
- rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(),
+ rowset_meta->partition_id(), rowset_meta->txn_id(),
+ rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(),
rowset_meta->tablet_uid(), rowset_meta->load_id(), rowset, true);
if (commit_txn_status != OLAP_SUCCESS && commit_txn_status != OLAP_ERR_PUSH_TRANSACTION_ALREADY_EXIST) {
LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id()
- << " to tablet: " << rowset_meta->tablet_id()
+ << " to tablet: " << rowset_meta->tablet_id()
<< " for txn: " << rowset_meta->txn_id();
} else {
LOG(INFO) << "successfully to add committed rowset: " << rowset_meta->rowset_id()
- << " to tablet: " << rowset_meta->tablet_id()
+ << " to tablet: " << rowset_meta->tablet_id()
<< " schema hash: " << rowset_meta->tablet_schema_hash()
<< " for txn: " << rowset_meta->txn_id();
}
- } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE
+ } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE
&& rowset_meta->tablet_uid() == tablet->tablet_uid()) {
OLAPStatus publish_status = tablet->add_rowset(rowset, false);
if (publish_status != OLAP_SUCCESS && publish_status != OLAP_ERR_PUSH_VERSION_ALREADY_EXIST) {
@@ -833,7 +814,7 @@ OLAPStatus DataDir::load() {
}
} else {
LOG(WARNING) << "find invalid rowset: " << rowset_meta->rowset_id()
- << " with tablet id: " << rowset_meta->tablet_id()
+ << " with tablet id: " << rowset_meta->tablet_id()
<< " tablet uid: " << rowset_meta->tablet_uid()
<< " schema hash: " << rowset_meta->tablet_schema_hash()
<< " txn: " << rowset_meta->txn_id()
@@ -949,7 +930,7 @@ void DataDir::perform_path_gc_by_rowsetid() {
if (is_rowset_file) {
TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash);
if (tablet != nullptr) {
- if (!tablet->check_rowset_id(rowset_id)
+ if (!tablet->check_rowset_id(rowset_id)
&& !StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id)) {
_process_garbage_path(path);
}
@@ -1057,7 +1038,7 @@ Status DataDir::update_capacity() {
bool DataDir::reach_capacity_limit(int64_t incoming_data_size) {
double used_pct = (_disk_capacity_bytes - _available_bytes + incoming_data_size) / (double) _disk_capacity_bytes;
int64_t left_bytes = _disk_capacity_bytes - _available_bytes - incoming_data_size;
-
+
if (used_pct >= config::storage_flood_stage_usage_percent / 100.0
&& left_bytes <= config::storage_flood_stage_left_capacity_bytes) {
LOG(WARNING) << "reach capacity limit. used pct: " << used_pct << ", left bytes: " << left_bytes
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index 8316811..429f0de 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -92,7 +92,7 @@ public:
void find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths);
static std::string get_root_path_from_schema_hash_path_in_trash(const std::string& schema_hash_dir_in_trash);
-
+
// load data from meta and data files
OLAPStatus load();
@@ -129,7 +129,6 @@ public:
private:
std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; }
Status _init_cluster_id();
- Status _check_path_exist();
Status _init_extension_and_capacity();
Status _init_file_system();
Status _init_meta();
@@ -137,7 +136,7 @@ private:
Status _check_disk();
OLAPStatus _read_and_write_test_file();
Status _read_cluster_id(const std::string& path, int32_t* cluster_id);
- Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id);
+ Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id);
OLAPStatus _clean_unfinished_converting_data();
OLAPStatus _convert_old_tablet();
diff --git a/be/src/util/disk_info.cpp b/be/src/util/disk_info.cpp
index 5767de3..5628740 100644
--- a/be/src/util/disk_info.cpp
+++ b/be/src/util/disk_info.cpp
@@ -24,6 +24,7 @@
#include <sys/vfs.h>
#include <sys/types.h>
#include <sys/stat.h>
+#include <sys/sysmacros.h>
#include <boost/algorithm/string.hpp>
#include <boost/algorithm/string/join.hpp>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org