You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2023/06/12 13:05:18 UTC
[doris] branch master updated: [bug](cooldown) Fix the issue of unused remote files not being deleted (#19785)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 283c55720d [bug](cooldown) Fix the issue of unused remote files not being deleted (#19785)
283c55720d is described below
commit 283c55720db4ee890db395ea33b7efd36126e5b6
Author: plat1ko <pl...@gmail.com>
AuthorDate: Mon Jun 12 21:05:09 2023 +0800
[bug](cooldown) Fix the issue of unused remote files not being deleted (#19785)
---
be/src/io/fs/s3_file_system.cpp | 4 +---
be/src/olap/tablet.cpp | 16 ++++++++--------
be/src/olap/tablet_manager.cpp | 2 +-
3 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index 51cd474238..312799d140 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -320,13 +320,11 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vector<File
}
for (const auto& obj : outcome.GetResult().GetContents()) {
std::string key = obj.GetKey();
- bool is_dir = (key.at(key.size() - 1) == '/');
+ bool is_dir = (key.back() == '/');
if (only_file && is_dir) {
continue;
}
FileInfo file_info;
- // note: if full path is s3://bucket/path/to/file.txt
- // obj.GetKey() will be /path/to/file.txt
file_info.file_name = obj.GetKey().substr(prefix.size());
file_info.file_size = obj.GetSize();
file_info.is_file = !is_dir;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 224823deca..f0c64e2c06 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -2485,23 +2485,23 @@ void Tablet::remove_unused_remote_files() {
if (auto it = buffer.find(id); LIKELY(it != buffer.end())) {
auto& fs = it->second.first;
auto& files = it->second.second;
+ std::vector<io::Path> paths;
+ paths.reserve(files.size());
// delete unused files
LOG(INFO) << "delete unused files. root_path=" << fs->root_path()
<< " tablet_id=" << id;
- io::Path dir("data/" + std::to_string(id));
+ io::Path dir = remote_tablet_path(id);
for (auto& file : files) {
- auto delete_path = dir / io::Path(file.file_name);
- LOG(INFO) << "delete unused file: " << delete_path.native();
+ auto file_path = dir / file.file_name;
+ LOG(INFO) << "delete unused file: " << file_path.native();
+ paths.push_back(std::move(file_path));
}
- std::vector<io::Path> file_names;
- for (auto& info : files) {
- file_names.emplace_back(info.file_name);
- }
- st = fs->batch_delete(file_names);
+ st = fs->batch_delete(paths);
if (!st.ok()) {
LOG(WARNING) << "failed to delete unused files, tablet_id=" << id << " : "
<< st;
}
+ buffer.erase(it);
}
}
};
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 085159b7ae..5d2bdee60d 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -856,7 +856,7 @@ Status TabletManager::load_tablet_from_dir(DataDir* store, TTabletId tablet_id,
}
TabletMetaSharedPtr tablet_meta(new TabletMeta());
- if (tablet_meta->create_from_file(header_path) != Status::OK()) {
+ if (!tablet_meta->create_from_file(header_path).ok()) {
LOG(WARNING) << "fail to load tablet_meta. file_path=" << header_path;
return Status::Error<ENGINE_LOAD_INDEX_TABLE_ERROR>();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org