You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/12 17:03:13 UTC
[doris] 29/33: [bug](GC)the issue of incorrect disk usage (#18397)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch doris-for-zhongjin
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1af97cda039967ec66903fe7d29735da2d2a1ac9
Author: chenlinzhong <49...@qq.com>
AuthorDate: Sat Apr 8 09:32:36 2023 +0800
[bug](GC)the issue of incorrect disk usage (#18397)
---
be/src/io/fs/local_file_system.cpp | 16 ++++++++++++++++
be/src/io/fs/local_file_system.h | 2 ++
be/src/olap/data_dir.cpp | 6 +++---
be/src/olap/rowset/rowset_meta_manager.cpp | 7 +++++++
be/src/olap/rowset/rowset_meta_manager.h | 1 +
be/src/olap/tablet.cpp | 3 ++-
6 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp
index 5ffcabca5e..cd08928ba9 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -115,6 +115,22 @@ Status LocalFileSystem::delete_directory_impl(const Path& dir) {
return Status::OK();
}
+Status LocalFileSystem::delete_directory_or_file(const Path& path) {
+ bool is_dir;
+ Status ret = is_directory(path, &is_dir);
+ if (ret.ok()) {
+ Status s;
+ if (is_dir) {
+ s = delete_directory_impl(path);
+ } else {
+ s = delete_file_impl(path);
+ }
+ return s;
+ } else {
+ return ret;
+ }
+}
+
Status LocalFileSystem::batch_delete_impl(const std::vector<Path>& files) {
for (auto& file : files) {
RETURN_IF_ERROR(delete_file_impl(file));
diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h
index da1704f381..71f17dbe28 100644
--- a/be/src/io/fs/local_file_system.h
+++ b/be/src/io/fs/local_file_system.h
@@ -54,6 +54,8 @@ public:
Status copy_dirs(const Path& src, const Path& dest);
// return true if parent path contain sub path
static bool contain_path(const Path& parent, const Path& sub);
+ // delete dir or file
+ Status delete_directory_or_file(const Path& path);
protected:
Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 3fe2dcd7cc..59a03d5471 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -723,9 +723,9 @@ void DataDir::_process_garbage_path(const std::string& path) {
return;
}
if (exists) {
- LOG(INFO) << "collect garbage dir path: " << path;
- WARN_IF_ERROR(io::global_local_filesystem()->delete_directory(path),
- "remove garbage dir failed");
+ LOG(INFO) << "collect garbage path: " << path;
+ WARN_IF_ERROR(io::global_local_filesystem()->delete_directory_or_file(path),
+ "remove garbage failed");
}
}
diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp b/be/src/olap/rowset/rowset_meta_manager.cpp
index 7a58fa9cb6..ac9ae5764e 100644
--- a/be/src/olap/rowset/rowset_meta_manager.cpp
+++ b/be/src/olap/rowset/rowset_meta_manager.cpp
@@ -42,6 +42,13 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid,
return meta->key_may_exist(META_COLUMN_FAMILY_INDEX, key, &value);
}
+Status RowsetMetaManager::exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id) {
+ std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + rowset_id.to_string();
+ std::string value;
+ Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value);
+ return s;
+}
+
Status RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid,
const RowsetId& rowset_id,
RowsetMetaSharedPtr rowset_meta) {
diff --git a/be/src/olap/rowset/rowset_meta_manager.h b/be/src/olap/rowset/rowset_meta_manager.h
index 8c8f3144e0..cca2304ce3 100644
--- a/be/src/olap/rowset/rowset_meta_manager.h
+++ b/be/src/olap/rowset/rowset_meta_manager.h
@@ -31,6 +31,7 @@ namespace doris {
class RowsetMetaManager {
public:
static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id);
+ static Status exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id);
static Status get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& rowset_id,
RowsetMetaSharedPtr rowset_meta);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 2bf536468f..8dd0e83c6f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1160,7 +1160,8 @@ bool Tablet::check_rowset_id(const RowsetId& rowset_id) {
return true;
}
}
- if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), tablet_uid(), rowset_id)) {
+ Status s = RowsetMetaManager::exists(_data_dir->get_meta(), tablet_uid(), rowset_id);
+ if (!s.is<META_KEY_NOT_FOUND>()) {
return true;
}
return false;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org