You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by GitBox <gi...@apache.org> on 2022/07/18 04:23:11 UTC
[GitHub] [doris] pengxiangyu commented on a diff in pull request #10731: [Enhancement] Garbage collection of unused data on remote storage backend
pengxiangyu commented on code in PR #10731:
URL: https://github.com/apache/doris/pull/10731#discussion_r922954365
##########
be/src/olap/base_tablet.h:
##########
@@ -59,10 +60,10 @@ class BaseTablet : public std::enable_shared_from_this<BaseTablet> {
int16_t shard_id() const;
bool equal(int64_t tablet_id, int32_t schema_hash);
- const io::ResourceId& cooldown_resource() const { return _tablet_meta->cooldown_resource(); }
+ const std::string& storage_policy() const { return _tablet_meta->storage_policy(); }
- void set_cooldown_resource(io::ResourceId resource) {
- _tablet_meta->set_cooldown_resource(std::move(resource));
+ void set_storage_policy(std::string policy) {
Review Comment:
const std::string& is better.
##########
be/src/olap/data_dir.cpp:
##########
@@ -804,4 +805,41 @@ Status DataDir::move_to_trash(const std::string& tablet_path) {
return Status::OK();
}
+void DataDir::perform_remote_gc() {
+ std::vector<std::pair<std::string, std::string>> gc_kvs;
+ auto traverse_remote_rowset_func = [&gc_kvs](const std::string& key,
+ const std::string& value) -> bool {
+ gc_kvs.emplace_back(key, value);
+ return true;
+ };
+ _meta->iterate(META_COLUMN_FAMILY_INDEX, REMOTE_ROWSET_GC_PREFIX, traverse_remote_rowset_func);
+ std::vector<std::string> deleted_keys;
+ for (auto& [key, val] : gc_kvs) {
+ auto rowset_id = key.substr(REMOTE_ROWSET_GC_PREFIX.size());
+ RemoteRowsetGcPB gc_pb;
+ gc_pb.ParseFromString(val);
+ auto fs = io::FileSystemMap::instance()->get(gc_pb.resource_id());
+ if (!fs) {
+ LOG(WARNING) << "Cannot get file system: " << gc_pb.resource_id();
+ continue;
+ }
+ DCHECK(fs->type() != io::FileSystemType::LOCAL);
+ Status st;
+ for (int i = 0; i < gc_pb.num_segments(); ++i) {
+ auto seg_path = BetaRowset::remote_segment_path(gc_pb.tablet_id(), rowset_id, i);
Review Comment:
delete file directly？It is dangerous, move it to trash is better。
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org