You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/01 05:39:14 UTC
(doris) branch master updated: [Enhancement](group commit) Optimize WAL insufficient space error messages (#30614)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7c133689455 [Enhancement](group commit) Optimize WAL insufficient space error messages (#30614)
7c133689455 is described below
commit 7c1336894553a762c83cf41c4085e86fda2ddd54
Author: abmdocrt <Yu...@gmail.com>
AuthorDate: Thu Feb 1 13:39:06 2024 +0800
[Enhancement](group commit) Optimize WAL insufficient space error messages (#30614)
---
be/src/http/action/http_stream.cpp | 17 +++++------------
be/src/http/action/stream_load.cpp | 17 +++++------------
be/src/olap/wal/wal_dirs_info.cpp | 16 ++++++++++++++++
be/src/olap/wal/wal_dirs_info.h | 2 ++
be/src/olap/wal/wal_manager.cpp | 4 ++++
be/src/olap/wal/wal_manager.h | 1 +
6 files changed, 33 insertions(+), 24 deletions(-)
diff --git a/be/src/http/action/http_stream.cpp b/be/src/http/action/http_stream.cpp
index efa5c10df45..090519ef577 100644
--- a/be/src/http/action/http_stream.cpp
+++ b/be/src/http/action/http_stream.cpp
@@ -398,18 +398,11 @@ Status HttpStreamAction::_handle_group_commit(HttpRequest* req,
if (iequal(group_commit_mode, "async_mode")) {
group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
if (iequal(group_commit_mode, "sync_mode")) {
- size_t max_available_size =
- ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
- LOG(INFO) << "When enable group commit, the data size can't be too large or "
- "unknown. The data size for this stream load("
- << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
- ? 0
- : req->header(HttpHeaders::CONTENT_LENGTH))
- << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
- << max_available_size
- << " Bytes). So we set this load to \"group commit\"=sync_mode\" "
- "automatically.";
- return Status::Error<EXCEEDED_LIMIT>("Http load size too large.");
+ std::stringstream ss;
+ ss << "There is no space for group commit http load async WAL. WAL dir info: "
+ << ExecEnv::GetInstance()->wal_mgr()->get_wal_dirs_info_string();
+ LOG(WARNING) << ss.str();
+ return Status::Error<EXCEEDED_LIMIT>(ss.str());
}
}
}
diff --git a/be/src/http/action/stream_load.cpp b/be/src/http/action/stream_load.cpp
index 2aa230530ae..7b5b6b10419 100644
--- a/be/src/http/action/stream_load.cpp
+++ b/be/src/http/action/stream_load.cpp
@@ -718,18 +718,11 @@ Status StreamLoadAction::_handle_group_commit(HttpRequest* req,
if (iequal(group_commit_mode, "async_mode")) {
group_commit_mode = load_size_smaller_than_wal_limit(req) ? "async_mode" : "sync_mode";
if (iequal(group_commit_mode, "sync_mode")) {
- size_t max_available_size =
- ExecEnv::GetInstance()->wal_mgr()->get_max_available_size();
- LOG(INFO) << "When enable group commit, the data size can't be too large or "
- "unknown. The data size for this stream load("
- << (req->header(HttpHeaders::CONTENT_LENGTH).empty()
- ? 0
- : req->header(HttpHeaders::CONTENT_LENGTH))
- << " Bytes) exceeds the WAL (Write-Ahead Log) limit ("
- << max_available_size
- << " Bytes). So we set this load to \"group commit\"=sync_mode\" "
- "automatically.";
- return Status::Error<EXCEEDED_LIMIT>("Stream load size too large.");
+ std::stringstream ss;
+ ss << "There is no space for group commit stream load async WAL. WAL dir info: "
+ << ExecEnv::GetInstance()->wal_mgr()->get_wal_dirs_info_string();
+ LOG(WARNING) << ss.str();
+ return Status::Error<EXCEEDED_LIMIT>(ss.str());
}
}
}
diff --git a/be/src/olap/wal/wal_dirs_info.cpp b/be/src/olap/wal/wal_dirs_info.cpp
index e451146106c..4e54a543980 100644
--- a/be/src/olap/wal/wal_dirs_info.cpp
+++ b/be/src/olap/wal/wal_dirs_info.cpp
@@ -17,6 +17,8 @@
#include "olap/wal/wal_dirs_info.h"
+#include <string>
+
#include "common/config.h"
#include "common/status.h"
#include "io/fs/local_file_system.h"
@@ -105,6 +107,12 @@ void WalDirInfo::update_wal_dir_pre_allocated(size_t increase_pre_allocated,
set_pre_allocated(increase_pre_allocated, decrease_pre_allocated);
}
+std::string WalDirInfo::get_wal_dir_info_string() {
+ return "[" + _wal_dir + ": limit " + std::to_string(_limit) + " Bytes, used " +
+ std::to_string(_used) + " Bytes, pre allocated " + std::to_string(_pre_allocated) +
+ " Bytes, available " + std::to_string(available()) + "Bytes.]";
+}
+
Status WalDirsInfo::add(const std::string& wal_dir, size_t limit, size_t used,
size_t pre_allocated) {
for (const auto& it : _wal_dirs_info_vec) {
@@ -155,6 +163,14 @@ size_t WalDirsInfo::get_max_available_size() {
->available();
}
+std::string WalDirsInfo::get_wal_dirs_info_string() {
+ std::string wal_dirs_info_string;
+ for (const auto& wal_dir_info : _wal_dirs_info_vec) {
+ wal_dirs_info_string += wal_dir_info->get_wal_dir_info_string() + "\n";
+ }
+ return wal_dirs_info_string;
+}
+
Status WalDirsInfo::update_wal_dir_limit(const std::string& wal_dir, size_t limit) {
for (const auto& wal_dir_info : _wal_dirs_info_vec) {
LOG(INFO) << "wal_dir_info:" << wal_dir_info->get_wal_dir();
diff --git a/be/src/olap/wal/wal_dirs_info.h b/be/src/olap/wal/wal_dirs_info.h
index 6eeb75cdb86..cfd34733007 100644
--- a/be/src/olap/wal/wal_dirs_info.h
+++ b/be/src/olap/wal/wal_dirs_info.h
@@ -49,6 +49,7 @@ public:
Status update_wal_dir_limit(size_t limit = -1);
Status update_wal_dir_used(size_t used = -1);
void update_wal_dir_pre_allocated(size_t increase_pre_allocated, size_t decrease_pre_allocated);
+ std::string get_wal_dir_info_string();
private:
std::string _wal_dir;
@@ -75,6 +76,7 @@ public:
size_t decrease_pre_allocated);
Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
Status get_wal_dir_info(const std::string& wal_dir, std::shared_ptr<WalDirInfo>& wal_dir_info);
+ std::string get_wal_dirs_info_string();
private:
std::vector<std::shared_ptr<WalDirInfo>> _wal_dirs_info_vec;
diff --git a/be/src/olap/wal/wal_manager.cpp b/be/src/olap/wal/wal_manager.cpp
index 45a9b58ff2d..6d59c6df686 100644
--- a/be/src/olap/wal/wal_manager.cpp
+++ b/be/src/olap/wal/wal_manager.cpp
@@ -381,6 +381,10 @@ size_t WalManager::get_max_available_size() {
return _wal_dirs_info->get_max_available_size();
}
+std::string WalManager::get_wal_dirs_info_string() {
+ return _wal_dirs_info->get_wal_dirs_info_string();
+}
+
Status WalManager::update_wal_dir_limit(const std::string& wal_dir, size_t limit) {
return _wal_dirs_info->update_wal_dir_limit(wal_dir, limit);
}
diff --git a/be/src/olap/wal/wal_manager.h b/be/src/olap/wal/wal_manager.h
index 44fdef2e6c0..dfa2859cbe3 100644
--- a/be/src/olap/wal/wal_manager.h
+++ b/be/src/olap/wal/wal_manager.h
@@ -61,6 +61,7 @@ public:
size_t decrease_pre_allocated);
Status get_wal_dir_available_size(const std::string& wal_dir, size_t* available_bytes);
size_t get_max_available_size();
+ std::string get_wal_dirs_info_string();
// replay wal
Status create_wal_path(int64_t db_id, int64_t table_id, int64_t wal_id,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org