You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/06 11:47:22 UTC
(doris) branch master updated: [fix](tool) Fix meta_tool coredump and dead loop issues (#61509)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 35c6887064f [fix](tool) Fix meta_tool coredump and dead loop issues (#61509)
35c6887064f is described below
commit 35c6887064fc855ba5f03c68dcdd0cb730dca005
Author: Wen Zhenghu <we...@gmail.com>
AuthorDate: Wed May 6 19:47:13 2026 +0800
[fix](tool) Fix meta_tool coredump and dead loop issues (#61509)
### What problem does this PR solve?
related issue：https://github.com/apache/doris/issues/61447
## Proposed changes
This PR fixes several bugs in the offline tool `meta_tool`, including
coredumps caused by uninitialized ExecEnv components and a dead loop
when loading tablet meta.
1. **Fix coredump in offline commands**:
- For lightweight commands (`show_meta`, `batch_delete_meta`,
`show_segment_footer`, `show_segment_data`): These do not require a full
StorageEngine environment but still rely on basic components like
`TabletSchemaCache`, `TabletColumnObjectPool`, and `MemTracker`. This PR
introduces a lightweight initialization function
`init_common_components()` for them to prevent coredumps caused by null
pointer dereferences.
- For engine-level commands (`get_meta`, `load_meta`, `delete_meta`):
These commands initialize a `StorageEngine` and `DataDir` instance to
read/write RocksDB. The PR ensures that they correctly load
configurations from `be.conf` (via `DORIS_HOME`) and also call
`init_common_components()` to guarantee that both the global settings
(e.g., memory limits, thread pools) and the ExecEnv components are
properly set up before the engine starts.
2. **Fix dead loop in `load_meta`**:
The `TabletMetaManager::load_json_meta` function previously used
`infile.getline()` in a `while(!infile.eof())` loop. If the file cannot
be opened or an error occurs, the `failbit` is set but not the `eofbit`,
causing an infinite loop. This has been fixed by reading the file
content safely using `std::istreambuf_iterator` and adding an
`is_open()` check.
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [x] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [x] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/src/runtime/exec_env.h | 14 +++--
be/src/storage/tablet/tablet_meta_manager.cpp | 10 ++--
be/src/tools/meta_tool.cpp | 69 ++++++++++++++++++++--
.../storage/tablet/tablet_meta_manager_test.cpp | 6 ++
4 files changed, 83 insertions(+), 16 deletions(-)
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 0daba28e494..d915fbc5721 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -331,13 +331,8 @@ public:
segment_v2::InvertedIndexQueryCache* inverted_index_query_cache) {
_inverted_index_query_cache = inverted_index_query_cache;
}
- void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; }
void set_process_profile(ProcessProfile* pp) { this->_process_profile = pp; }
- void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; }
void set_delete_bitmap_agg_cache(DeleteBitmapAggCache* c) { _delete_bitmap_agg_cache = c; }
- void set_tablet_column_object_pool(TabletColumnObjectPool* c) {
- this->_tablet_column_object_pool = c;
- }
void set_storage_page_cache(StoragePageCache* c) { this->_storage_page_cache = c; }
void set_ann_index_ivf_list_cache(AnnIndexIVFListCache* c) {
this->_ann_index_ivf_list_cache = c;
@@ -365,6 +360,15 @@ public:
_file_cache_open_fd_cache = std::move(fd_cache);
}
#endif
+ // WARN: The following setter methods are intended for use in test code and
+ // offline tools (like meta_tool) ONLY. They should NOT be called in the
+ // production environment to avoid thread safety issues and undefined behaviors.
+ void set_cache_manager(CacheManager* cm) { this->_cache_manager = cm; }
+ void set_tablet_schema_cache(TabletSchemaCache* c) { this->_tablet_schema_cache = c; }
+ void set_tablet_column_object_pool(TabletColumnObjectPool* c) {
+ this->_tablet_column_object_pool = c;
+ }
+
LoadStreamMapPool* load_stream_map_pool() { return _load_stream_map_pool.get(); }
DeltaWriterV2Pool* delta_writer_v2_pool() { return _delta_writer_v2_pool.get(); }
diff --git a/be/src/storage/tablet/tablet_meta_manager.cpp b/be/src/storage/tablet/tablet_meta_manager.cpp
index 8de25fa645f..e6d10c28665 100644
--- a/be/src/storage/tablet/tablet_meta_manager.cpp
+++ b/be/src/storage/tablet/tablet_meta_manager.cpp
@@ -147,12 +147,12 @@ Status TabletMetaManager::traverse_headers(
Status TabletMetaManager::load_json_meta(DataDir* store, const std::string& meta_path) {
std::ifstream infile(meta_path);
- char buffer[102400];
- std::string json_meta;
- while (!infile.eof()) {
- infile.getline(buffer, 102400);
- json_meta = json_meta + buffer;
+ if (!infile.is_open()) {
+ return Status::InternalError("Failed to open file {}", meta_path);
}
+
+ std::string json_meta((std::istreambuf_iterator<char>(infile)),
+ std::istreambuf_iterator<char>());
boost::algorithm::trim(json_meta);
TabletMetaPB tablet_meta_pb;
std::string error;
diff --git a/be/src/tools/meta_tool.cpp b/be/src/tools/meta_tool.cpp
index c4f7595678c..6f3a21688d1 100644
--- a/be/src/tools/meta_tool.cpp
+++ b/be/src/tools/meta_tool.cpp
@@ -30,6 +30,7 @@
#include <sstream>
#include <string>
+#include "common/logging.h"
#include "common/status.h"
#include "core/assert_cast.h"
#include "core/column/column.h"
@@ -46,6 +47,7 @@
#include "io/fs/local_file_system.h"
#include "json2pb/pb_to_json.h"
#include "runtime/exec_env.h"
+#include "runtime/memory/cache_manager.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "storage/data_dir.h"
#include "storage/olap_common.h"
@@ -54,9 +56,11 @@
#include "storage/segment/encoding_info.h"
#include "storage/segment/page_pointer.h"
#include "storage/storage_engine.h"
+#include "storage/tablet/tablet_column_object_pool.h"
#include "storage/tablet/tablet_meta.h"
#include "storage/tablet/tablet_meta_manager.h"
#include "storage/tablet/tablet_schema.h"
+#include "storage/tablet/tablet_schema_cache.h"
#include "storage/types.h"
#include "util/coding.h"
@@ -101,7 +105,7 @@ std::string get_usage(const std::string& progname) {
ss << "./meta_tool --operation=delete_meta "
"--root_path=/path/to/storage/path --tablet_id=tabletid "
"--schema_hash=schemahash\n";
- ss << "./meta_tool --operation=delete_meta --tablet_file=file_path\n";
+ ss << "./meta_tool --operation=batch_delete_meta --tablet_file=file_path\n";
ss << "./meta_tool --operation=show_meta --pb_meta_path=path\n";
ss << "./meta_tool --operation=show_segment_footer --file=/path/to/segment/file\n";
ss << "./meta_tool --operation=show_segment_data --file=/path/to/segment/file\n";
@@ -132,9 +136,13 @@ void get_meta(DataDir* data_dir) {
std::string value;
Status s =
TabletMetaManager::get_json_meta(data_dir, FLAGS_tablet_id, FLAGS_schema_hash, &value);
- if (s.is<doris::ErrorCode::META_KEY_NOT_FOUND>()) {
- std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id
- << ", schema_hash:" << FLAGS_schema_hash << std::endl;
+ if (!s.ok()) {
+ if (s.is<doris::ErrorCode::META_KEY_NOT_FOUND>()) {
+ std::cout << "no tablet meta for tablet_id:" << FLAGS_tablet_id
+ << ", schema_hash:" << FLAGS_schema_hash << std::endl;
+ } else {
+ std::cout << "get meta failed: " << s.to_string() << std::endl;
+ }
return;
}
std::cout << value << std::endl;
@@ -177,8 +185,8 @@ Status init_data_dir(StorageEngine& engine, const std::string& dir, std::unique_
}
res = p->init();
if (!res.ok()) {
- std::cout << "data_dir load failed" << std::endl;
- return Status::InternalError("data_dir load failed");
+ std::cout << "data_dir load failed: " << res.to_string() << std::endl;
+ return res;
}
p.swap(*ret);
@@ -908,6 +916,32 @@ void show_segment_data(const std::string& file_name) {
}
}
+void init_common_components() {
+ // init meta_tool.log to current dir
+ if (doris::config::sys_log_dir == "") {
+ doris::config::sys_log_dir = ".";
+ }
+ if (doris::config::sys_log_level == "") {
+ doris::config::sys_log_level = "INFO";
+ }
+ if (doris::config::sys_log_roll_mode == "") {
+ doris::config::sys_log_roll_mode = "SIZE-MB-1024";
+ }
+ FLAGS_log_dir = doris::config::sys_log_dir;
+ if (!doris::init_glog("meta_tool")) {
+ fprintf(stderr, "init glog failed.\n");
+ }
+
+ doris::ExecEnv::GetInstance()->init_mem_tracker();
+ doris::ExecEnv::GetInstance()->set_cache_manager(doris::CacheManager::create_global_instance());
+ doris::ExecEnv::GetInstance()->set_tablet_schema_cache(
+ doris::TabletSchemaCache::create_global_schema_cache(
+ doris::config::tablet_schema_cache_capacity));
+ doris::ExecEnv::GetInstance()->set_tablet_column_object_pool(
+ doris::TabletColumnObjectPool::create_global_column_cache(
+ doris::config::tablet_schema_cache_capacity));
+}
+
void gen_empty_segment() {
std::string output_path = FLAGS_output_path.empty() ? "." : FLAGS_output_path;
@@ -1029,6 +1063,7 @@ int main(int argc, char** argv) {
google::ParseCommandLineFlags(&argc, &argv, true);
if (FLAGS_operation == "show_meta") {
+ init_common_components();
show_meta();
} else if (FLAGS_operation == "batch_delete_meta") {
std::string tablet_file;
@@ -1040,18 +1075,21 @@ int main(int argc, char** argv) {
return -1;
}
+ init_common_components();
batch_delete_meta(tablet_file);
} else if (FLAGS_operation == "show_segment_footer") {
if (FLAGS_file == "") {
std::cout << "no file flag for show dict" << std::endl;
return -1;
}
+ init_common_components();
show_segment_footer(FLAGS_file);
} else if (FLAGS_operation == "show_segment_data") {
if (FLAGS_file == "") {
std::cout << "no file flag for show_segment_data" << std::endl;
return -1;
}
+ init_common_components();
show_segment_data(FLAGS_file);
} else if (FLAGS_operation == "gen_empty_segment") {
gen_empty_segment();
@@ -1063,6 +1101,25 @@ int main(int argc, char** argv) {
return -1;
}
+ if (getenv("DORIS_HOME") == nullptr) {
+ fprintf(stderr, "you need set DORIS_HOME environment variable.\n");
+ exit(-1);
+ }
+
+ std::string conffile = std::string(getenv("DORIS_HOME")) + "/conf/be.conf";
+ if (!doris::config::init(conffile.c_str(), true, true, true)) {
+ fprintf(stderr, "error read config file. \n");
+ return -1;
+ }
+
+ std::string custom_conffile = doris::config::custom_config_dir + "/be_custom.conf";
+ if (!doris::config::init(custom_conffile.c_str(), true, false, false)) {
+ fprintf(stderr, "error read custom config file. \n");
+ return -1;
+ }
+
+ init_common_components();
+
StorageEngine engine(doris::EngineOptions {});
std::unique_ptr<DataDir> data_dir;
Status st = init_data_dir(engine, FLAGS_root_path, &data_dir);
diff --git a/be/test/storage/tablet/tablet_meta_manager_test.cpp b/be/test/storage/tablet/tablet_meta_manager_test.cpp
index f5fee8a21cc..34f9641aebe 100644
--- a/be/test/storage/tablet/tablet_meta_manager_test.cpp
+++ b/be/test/storage/tablet/tablet_meta_manager_test.cpp
@@ -113,6 +113,12 @@ TEST_F(TabletMetaManagerTest, TestLoad) {
// EXPECT_EQ(_json_header, json_meta_read);
}
+TEST_F(TabletMetaManagerTest, TestLoadFailed) {
+ Status s = TabletMetaManager::load_json_meta(_data_dir, "/path/to/non_existent_file.json");
+ EXPECT_FALSE(s.ok());
+ EXPECT_TRUE(s.is<ErrorCode::INTERNAL_ERROR>());
+}
+
TEST_F(TabletMetaManagerTest, TestDeleteBitmapEncode) {
TTabletId tablet_id = 1234;
int64_t version = 456;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org