You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/01/12 01:59:51 UTC
(doris) branch branch-4.0 updated: branch-4.0: [feat](cloud) Cherry pick packed file prs (#59693)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 5ca01b6f479 branch-4.0: [feat](cloud) Cherry pick packed file prs (#59693)
5ca01b6f479 is described below
commit 5ca01b6f47986bbdc4bbff31612e1db9e1dabf1c
Author: Xin Liao <li...@selectdb.com>
AuthorDate: Mon Jan 12 09:59:44 2026 +0800
branch-4.0: [feat](cloud) Cherry pick packed file prs (#59693)
pick:
https://github.com/apache/doris/issues?q=state%3Aclosed%20label%3Apacked-file
---------
Co-authored-by: Luwei <lu...@selectdb.com>
Co-authored-by: Luwei <81...@qq.com>
---
be/src/cloud/cloud_meta_mgr.cpp | 23 +
be/src/cloud/cloud_meta_mgr.h | 3 +
be/src/cloud/cloud_rowset_writer.cpp | 75 +-
be/src/cloud/cloud_rowset_writer.h | 12 +-
be/src/cloud/cloud_tablet.cpp | 2 +
be/src/cloud/config.cpp | 11 +
be/src/cloud/config.h | 10 +
be/src/cloud/injection_point_action.cpp | 36 +-
be/src/cloud/pb_convert.cpp | 12 +
be/src/index-tools/index_tool.cpp | 7 +-
be/src/io/fs/packed_file_manager.cpp | 915 ++++++++++++++++
be/src/io/fs/packed_file_manager.h | 207 ++++
be/src/io/fs/packed_file_reader.cpp | 87 ++
be/src/io/fs/packed_file_reader.h | 63 ++
be/src/io/fs/packed_file_system.cpp | 114 ++
be/src/io/fs/packed_file_system.h | 97 ++
be/src/io/fs/packed_file_trailer.cpp | 152 +++
.../fs/packed_file_trailer.h} | 23 +-
be/src/io/fs/packed_file_writer.cpp | 233 ++++
be/src/io/fs/packed_file_writer.h | 91 ++
be/src/io/fs/s3_file_writer.cpp | 64 +-
be/src/io/fs/s3_file_writer.h | 4 +
be/src/olap/calc_delete_bitmap_executor.h | 21 +
be/src/olap/compaction.cpp | 2 +
be/src/olap/rowset/beta_rowset_writer.cpp | 124 ++-
be/src/olap/rowset/beta_rowset_writer.h | 11 +-
be/src/olap/rowset/rowset_factory.cpp | 4 +-
be/src/olap/rowset/rowset_meta.cpp | 30 +-
be/src/olap/rowset/rowset_meta.h | 11 +
be/src/olap/rowset/rowset_writer_context.h | 74 +-
be/src/olap/rowset/segment_creator.cpp | 4 +-
.../olap/rowset/segment_v2/index_file_writer.cpp | 32 +-
be/src/olap/rowset/segment_v2/index_file_writer.h | 11 +-
.../segment_v2/inverted_index_fs_directory.cpp | 42 +-
be/src/olap/rowset/segment_v2/segment_writer.h | 2 +-
.../rowset/segment_v2/vertical_segment_writer.h | 2 +-
be/src/olap/schema_change.cpp | 3 +
be/src/olap/task/index_builder.cpp | 18 +-
be/src/runtime/exec_env.h | 3 +
be/src/runtime/exec_env_init.cpp | 13 +
be/src/service/http_service.cpp | 1 +
be/src/tools/CMakeLists.txt | 29 +
be/src/tools/packed_file_tool.cpp | 56 +
be/src/util/s3_util.cpp | 22 +
be/src/util/s3_util.h | 11 +
be/test/io/fs/packed_file_concurrency_test.cpp | 753 +++++++++++++
be/test/io/fs/packed_file_manager_test.cpp | 1129 ++++++++++++++++++++
be/test/io/fs/packed_file_reader_test.cpp | 271 +++++
be/test/io/fs/packed_file_system_test.cpp | 439 ++++++++
be/test/io/fs/packed_file_writer_test.cpp | 259 +++++
be/test/io/fs/s3_file_writer_test.cpp | 3 +-
be/test/io/packed_file_trailer_test.cpp | 116 ++
.../inverted_index/empty_index_file_test.cpp | 3 +-
.../query/phrase_edge_query_test.cpp | 4 +-
.../query/phrase_prefix_query_test.cpp | 4 +-
.../inverted_index/query/phrase_query_test.cpp | 4 +-
.../segment_v2/inverted_index_array_test.cpp | 21 +-
.../inverted_index_compound_reader_test.cpp | 8 +-
.../segment_v2/inverted_index_file_reader_test.cpp | 6 +-
.../segment_v2/inverted_index_file_writer_test.cpp | 92 +-
.../segment_v2/inverted_index_reader_test.cpp | 24 +-
.../segment_v2/inverted_index_writer_test.cpp | 48 +-
.../rowset/segment_v2/segment_corruption_test.cpp | 4 +-
be/test/olap/test_data/rowset_meta.json | 3 +-
cloud/src/common/bvars.cpp | 17 +
cloud/src/common/bvars.h | 9 +
cloud/src/common/config.h | 9 +
cloud/src/meta-service/meta_service.cpp | 74 ++
cloud/src/meta-service/meta_service.h | 13 +
cloud/src/meta-store/keys.cpp | 12 +-
cloud/src/meta-store/keys.h | 10 +
cloud/src/recycler/checker.cpp | 418 +++++++-
cloud/src/recycler/checker.h | 5 +
cloud/src/recycler/recycler.cpp | 1037 +++++++++++++++++-
cloud/src/recycler/recycler.h | 73 ++
cloud/test/meta_service_test.cpp | 156 +++
cloud/test/recycler_test.cpp | 1019 ++++++++++++++++++
gensrc/proto/cloud.proto | 46 +
gensrc/proto/olap_file.proto | 9 +
.../packed_file/merge_file_stream_load.csv | 200 ++++
.../test_packed_file_concurrent_load.groovy | 324 ++++++
.../packed_file/test_packed_file_mixed_load.groovy | 558 ++++++++++
.../test_packed_file_query_with_table_drop.groovy | 181 ++++
.../test_packed_file_with_group_commit.groovy | 160 +++
.../suites/cloud_p0/recycler/test_recycler.groovy | 2 +
.../test_packed_file_stream_load_case1.groovy | 79 ++
.../test_packed_file_stream_load_case2.groovy | 80 ++
.../test_packed_file_stream_load_case3.groovy | 89 ++
.../test_packed_file_stream_load_case4.groovy | 91 ++
.../test_packed_file_stream_load_case5.groovy | 147 +++
.../test_packed_file_stream_load_case6.groovy | 148 +++
.../test_packed_file_stream_load_case7.groovy | 157 +++
.../test_packed_file_stream_load_case8.groovy | 159 +++
93 files changed, 11064 insertions(+), 176 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 27c40853245..84420a67a3e 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -367,6 +367,8 @@ static std::string debug_info(const Request& req) {
return fmt::format(" index_id={}", req.index_id());
} else if constexpr (is_any_v<Request, RestoreJobRequest>) {
return fmt::format(" tablet_id={}", req.tablet_id());
+ } else if constexpr (is_any_v<Request, UpdatePackedFileInfoRequest>) {
+ return fmt::format(" packed_file_path={}", req.packed_file_path());
} else {
static_assert(!sizeof(Request));
}
@@ -2213,5 +2215,26 @@ Status CloudMetaMgr::get_snapshot_properties(SnapshotSwitchStatus& switch_status
: 3600;
return Status::OK();
}
+
+Status CloudMetaMgr::update_packed_file_info(const std::string& packed_file_path,
+ const cloud::PackedFileInfoPB& packed_file_info) {
+ VLOG_DEBUG << "Updating meta service for packed file: " << packed_file_path << " with "
+ << packed_file_info.total_slice_num() << " small files"
+ << ", total bytes: " << packed_file_info.total_slice_bytes();
+
+ // Create request
+ cloud::UpdatePackedFileInfoRequest req;
+ cloud::UpdatePackedFileInfoResponse resp;
+
+ // Set required fields
+ req.set_cloud_unique_id(config::cloud_unique_id);
+ req.set_packed_file_path(packed_file_path);
+ *req.mutable_packed_file_info() = packed_file_info;
+
+ // Make RPC call using retry pattern
+ return retry_rpc("update packed file info", req, &resp,
+ &cloud::MetaService_Stub::update_packed_file_info);
+}
+
#include "common/compile_check_end.h"
} // namespace doris::cloud
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index a815c4dc5e5..ea13069ef20 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -86,6 +86,9 @@ public:
Status update_tmp_rowset(const RowsetMeta& rs_meta);
+ Status update_packed_file_info(const std::string& packed_file_path,
+ const cloud::PackedFileInfoPB& packed_file_info);
+
Status commit_txn(const StreamLoadContext& ctx, bool is_2pc);
Status abort_txn(const StreamLoadContext& ctx);
diff --git a/be/src/cloud/cloud_rowset_writer.cpp b/be/src/cloud/cloud_rowset_writer.cpp
index 5b77495f8c2..66067bb9428 100644
--- a/be/src/cloud/cloud_rowset_writer.cpp
+++ b/be/src/cloud/cloud_rowset_writer.cpp
@@ -19,12 +19,12 @@
#include "common/status.h"
#include "io/cache/block_file_cache_factory.h"
-#include "io/fs/file_system.h"
+#include "io/fs/packed_file_writer.h"
#include "olap/rowset/rowset_factory.h"
namespace doris {
-CloudRowsetWriter::CloudRowsetWriter() = default;
+CloudRowsetWriter::CloudRowsetWriter(CloudStorageEngine& engine) : _engine(engine) {}
CloudRowsetWriter::~CloudRowsetWriter() = default;
@@ -63,15 +63,31 @@ Status CloudRowsetWriter::init(const RowsetWriterContext& rowset_writer_context)
_rowset_meta->set_tablet_schema(_context.tablet_schema);
_context.segment_collector = std::make_shared<SegmentCollectorT<BaseBetaRowsetWriter>>(this);
_context.file_writer_creator = std::make_shared<FileWriterCreatorT<BaseBetaRowsetWriter>>(this);
+ if (_context.mow_context != nullptr) {
+ _calc_delete_bitmap_token = _engine.calc_delete_bitmap_executor_for_load()->create_token();
+ }
return Status::OK();
}
+Status CloudRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num) {
+ // Call base class implementation
+ RETURN_IF_ERROR(BaseBetaRowsetWriter::_build_rowset_meta(rowset_meta, check_segment_num));
+
+ // Collect packed file segment index information for interim rowsets as well.
+ return _collect_all_packed_slice_locations(rowset_meta);
+}
+
Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) {
+ if (_calc_delete_bitmap_token != nullptr) {
+ RETURN_IF_ERROR(_calc_delete_bitmap_token->wait());
+ }
RETURN_IF_ERROR(_close_file_writers());
// TODO(plat1ko): check_segment_footer
RETURN_IF_ERROR(_build_rowset_meta(_rowset_meta.get()));
+ // At this point all writers have been closed, so collecting packed file indices is safe.
+ RETURN_IF_ERROR(_collect_all_packed_slice_locations(_rowset_meta.get()));
// If the current load is a partial update, new segments may be appended to the tmp rowset after the tmp rowset
// has been committed if conflicts occur due to concurrent partial updates. However, when the recycler do recycling,
// it will generate the paths for the segments to be recycled on the object storage based on the number of segments
@@ -130,4 +146,59 @@ Status CloudRowsetWriter::build(RowsetSharedPtr& rowset) {
return Status::OK();
}
+Status CloudRowsetWriter::_collect_all_packed_slice_locations(RowsetMeta* rowset_meta) {
+ if (!_context.packed_file_active) {
+ return Status::OK();
+ }
+
+ // Collect segment file packed indices
+ const auto& file_writers = _seg_files.get_file_writers();
+ for (const auto& [seg_id, writer_ptr] : file_writers) {
+ auto segment_path = _context.segment_path(seg_id);
+ RETURN_IF_ERROR(
+ _collect_packed_slice_location(writer_ptr.get(), segment_path, rowset_meta));
+ }
+
+ // Collect inverted index file packed indices
+ const auto& idx_file_writers = _idx_files.get_file_writers();
+ for (const auto& [seg_id, idx_writer_ptr] : idx_file_writers) {
+ if (idx_writer_ptr != nullptr && idx_writer_ptr->get_file_writer() != nullptr) {
+ auto segment_path = _context.segment_path(seg_id);
+ auto index_prefix_view =
+ InvertedIndexDescriptor::get_index_file_path_prefix(segment_path);
+ std::string index_path =
+ InvertedIndexDescriptor::get_index_file_path_v2(std::string(index_prefix_view));
+ RETURN_IF_ERROR(_collect_packed_slice_location(idx_writer_ptr->get_file_writer(),
+ index_path, rowset_meta));
+ }
+ }
+
+ return Status::OK();
+}
+
+Status CloudRowsetWriter::_collect_packed_slice_location(io::FileWriter* file_writer,
+ const std::string& file_path,
+ RowsetMeta* rowset_meta) {
+ // At this point, we only call this when RowsetWriterContext::merge_file_active is true,
+ // and all writers should be MergeFileWriter. So we can safely cast without extra checks.
+ auto* packed_writer = static_cast<io::PackedFileWriter*>(file_writer);
+
+ if (packed_writer->state() != io::FileWriter::State::CLOSED) {
+ // Writer is still open; index will be collected after it is closed.
+ return Status::OK();
+ }
+
+ io::PackedSliceLocation index;
+ RETURN_IF_ERROR(packed_writer->get_packed_slice_location(&index));
+ if (index.packed_file_path.empty()) {
+ return Status::OK(); // File not in packed file, skip
+ }
+
+ rowset_meta->add_packed_slice_location(file_path, index.packed_file_path, index.offset,
+ index.size, index.packed_file_size);
+ LOG(INFO) << "collect packed file index: " << file_path << " -> " << index.packed_file_path
+ << ", offset: " << index.offset << ", size: " << index.size;
+ return Status::OK();
+}
+
} // namespace doris
diff --git a/be/src/cloud/cloud_rowset_writer.h b/be/src/cloud/cloud_rowset_writer.h
index 1bb2b3d38b8..56cbe4df249 100644
--- a/be/src/cloud/cloud_rowset_writer.h
+++ b/be/src/cloud/cloud_rowset_writer.h
@@ -17,19 +17,29 @@
#pragma once
+#include "cloud/cloud_storage_engine.h"
#include "olap/rowset/beta_rowset_writer.h"
namespace doris {
class CloudRowsetWriter : public BaseBetaRowsetWriter {
public:
- CloudRowsetWriter();
+ CloudRowsetWriter(CloudStorageEngine& engine);
~CloudRowsetWriter() override;
Status init(const RowsetWriterContext& rowset_writer_context) override;
Status build(RowsetSharedPtr& rowset) override;
+
+private:
+ Status _build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num = false) override;
+
+ Status _collect_all_packed_slice_locations(RowsetMeta* rowset_meta);
+
+ Status _collect_packed_slice_location(io::FileWriter* file_writer, const std::string& file_path,
+ RowsetMeta* rowset_meta);
+ CloudStorageEngine& _engine;
};
} // namespace doris
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index ea5f52366ab..91b5930b7c7 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -939,6 +939,8 @@ Result<std::unique_ptr<RowsetWriter>> CloudTablet::create_transient_rowset_write
context.enable_unique_key_merge_on_write = enable_unique_key_merge_on_write();
context.txn_expiration = txn_expiration;
context.encrypt_algorithm = tablet_meta()->encryption_algorithm();
+ // TODO(liaoxin) enable packed file for transient rowset
+ context.allow_packed_file = false;
auto storage_resource = rowset.rowset_meta()->remote_storage_resource();
if (!storage_resource) {
diff --git a/be/src/cloud/config.cpp b/be/src/cloud/config.cpp
index 30cecb7a405..009e6118621 100644
--- a/be/src/cloud/config.cpp
+++ b/be/src/cloud/config.cpp
@@ -133,6 +133,16 @@ DEFINE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms, "120000");
DEFINE_mBool(enable_warmup_immediately_on_new_rowset, "false");
+// Packed file manager config
+DEFINE_mBool(enable_packed_file, "true");
+DEFINE_mInt64(packed_file_size_threshold_bytes, "5242880"); // 5MB
+DEFINE_mInt64(packed_file_time_threshold_ms, "100"); // 100ms
+DEFINE_mInt64(packed_file_try_lock_timeout_ms, "5"); // 5ms
+DEFINE_mInt64(packed_file_small_file_count_threshold, "100");
+DEFINE_mInt64(small_file_threshold_bytes, "1048576"); // 1MB
+DEFINE_mInt64(uploaded_file_retention_seconds, "1800"); // 1 minute
+DEFINE_mInt64(packed_file_cleanup_interval_seconds, "60"); // 1 minute
+
DEFINE_mBool(enable_standby_passive_compaction, "true");
DEFINE_mDouble(standby_compaction_version_ratio, "0.8");
@@ -143,5 +153,6 @@ DEFINE_mBool(enable_cache_read_from_peer, "true");
// This can be configured to be less than the `rehash_tablet_after_be_dead_seconds` setting in the `fe` configuration.
// If the value is -1, use the `rehash_tablet_after_be_dead_seconds` setting in the `fe` configuration as the expiration time.
DEFINE_mInt64(cache_read_from_peer_expired_seconds, "-1");
+
#include "common/compile_check_end.h"
} // namespace doris::config
diff --git a/be/src/cloud/config.h b/be/src/cloud/config.h
index b12ff464700..80cfd6de2ed 100644
--- a/be/src/cloud/config.h
+++ b/be/src/cloud/config.h
@@ -176,6 +176,16 @@ DECLARE_mInt64(warm_up_rowset_sync_wait_max_timeout_ms);
DECLARE_mBool(enable_warmup_immediately_on_new_rowset);
+// Packed file manager config
+DECLARE_mBool(enable_packed_file);
+DECLARE_mInt64(packed_file_size_threshold_bytes);
+DECLARE_mInt64(packed_file_time_threshold_ms);
+DECLARE_mInt64(packed_file_try_lock_timeout_ms);
+DECLARE_mInt64(packed_file_small_file_count_threshold);
+DECLARE_mInt64(small_file_threshold_bytes);
+DECLARE_mInt64(uploaded_file_retention_seconds);
+DECLARE_mInt64(packed_file_cleanup_interval_seconds);
+
DECLARE_mBool(enable_standby_passive_compaction);
DECLARE_mDouble(standby_compaction_version_ratio);
diff --git a/be/src/cloud/injection_point_action.cpp b/be/src/cloud/injection_point_action.cpp
index 9f7809a8e79..521ff52ffc3 100644
--- a/be/src/cloud/injection_point_action.cpp
+++ b/be/src/cloud/injection_point_action.cpp
@@ -21,6 +21,7 @@
#include <chrono>
#include <mutex>
+#include <random>
#include "common/status.h"
#include "cpp/sync_point.h"
@@ -222,7 +223,9 @@ void set_return_ok(const std::string& point, HttpRequest* req) {
void set_return_error(const std::string& point, HttpRequest* req) {
const std::string CODE_PARAM = "code";
+ const std::string PROBABILITY_PARAM = "probability";
int code = ErrorCode::INTERNAL_ERROR;
+ double probability = 100.0;
auto& code_str = req->param(CODE_PARAM);
if (!code_str.empty()) {
try {
@@ -233,10 +236,37 @@ void set_return_error(const std::string& point, HttpRequest* req) {
return;
}
}
+ auto& probability_str = req->param(PROBABILITY_PARAM);
+ if (!probability_str.empty()) {
+ try {
+ probability = std::stod(probability_str);
+ } catch (const std::exception& e) {
+ HttpChannel::send_reply(
+ req, HttpStatus::BAD_REQUEST,
+ fmt::format("invalid probability: {}, {}", probability_str, e.what()));
+ return;
+ }
+ if (probability < 0.0 || probability > 100.0) {
+ HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST,
+ "probability should be between 0 and 100");
+ return;
+ }
+ }
auto sp = SyncPoint::get_instance();
- sp->set_call_back(point, [code, point](auto&& args) {
+ sp->set_call_back(point, [code, point, probability](auto&& args) {
try {
+ if (probability < 100.0) {
+ static thread_local std::mt19937 gen(std::random_device {}());
+ std::uniform_real_distribution<double> dist(0.0, 100.0);
+ double dice = dist(gen);
+ if (dice >= probability) {
+ LOG(INFO) << "injection point hit, point=" << point
+ << " skip error injection, probability=" << probability
+ << "%, random=" << dice;
+ return;
+ }
+ }
LOG(INFO) << "injection point hit, point=" << point << " return error code=" << code;
auto* pair = try_any_cast_ret<Status>(args);
pair->first = Status::Error<false>(code, "injected error");
@@ -345,13 +375,15 @@ InjectionPointAction::InjectionPointAction() = default;
// * return_ok: for injection point with return value, always return Status::OK
// * return_error: for injection point with return value, accepted param is `code`,
// which is an int, valid values can be found in status.h, e.g. -235 or -230,
-// if `code` is not present return Status::InternalError
+// if `code` is not present return Status::InternalError. Optional `probability`
+// determines the percentage of times to inject the error (default 100).
// ```
// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=sleep&duration=${x_millsec}" # sleep x millisecs
// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return" # return void
// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_ok" # return ok
// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error" # internal error
// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error&code=${code}" # -235
+// curl "be_ip:http_port/api/injection_point/set?name=${injection_point_name}&behavior=return_error&code=${code}&probability=50" # inject with 50% probability
// ```
void InjectionPointAction::handle(HttpRequest* req) {
LOG(INFO) << "handle InjectionPointAction " << req->debug_string();
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index e0d974dbaed..096f072c2bc 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -102,6 +102,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ auto* slice_locations = out->mutable_packed_slice_locations();
+ slice_locations->clear();
+ slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
}
void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
@@ -171,6 +174,9 @@ void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ auto* slice_locations = out->mutable_packed_slice_locations();
+ slice_locations->clear();
+ slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
}
RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in) {
@@ -250,6 +256,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB& in)
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ auto* slice_locations = out->mutable_packed_slice_locations();
+ slice_locations->clear();
+ slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
}
void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
@@ -318,6 +327,9 @@ void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
if (in.has_reference_instance_id()) {
out->set_reference_instance_id(in.reference_instance_id());
}
+ auto* slice_locations = out->mutable_packed_slice_locations();
+ slice_locations->clear();
+ slice_locations->insert(in.packed_slice_locations().begin(), in.packed_slice_locations().end());
}
TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) {
diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp
index 15f18dac972..9bd63601af9 100644
--- a/be/src/index-tools/index_tool.cpp
+++ b/be/src/index-tools/index_tool.cpp
@@ -621,11 +621,16 @@ int main(int argc, char** argv) {
_CLLDELETE(analyzer);
_CLLDELETE(char_string_reader);
- auto ret = index_file_writer->close();
+ auto ret = index_file_writer->begin_close();
if (!ret.ok()) {
std::cerr << "IndexFileWriter close error:" << ret.msg() << std::endl;
return -1;
}
+ ret = index_file_writer->finish_close();
+ if (!ret.ok()) {
+ std::cerr << "IndexFileWriter wait close error:" << ret.msg() << std::endl;
+ return -1;
+ }
} else if (FLAGS_operation == "show_nested_files_v2") {
if (FLAGS_idx_file_path == "") {
std::cout << "no file flag for show " << std::endl;
diff --git a/be/src/io/fs/packed_file_manager.cpp b/be/src/io/fs/packed_file_manager.cpp
new file mode 100644
index 00000000000..3cf65f12e49
--- /dev/null
+++ b/be/src/io/fs/packed_file_manager.cpp
@@ -0,0 +1,915 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_manager.h"
+
+#include <bvar/bvar.h>
+#include <bvar/recorder.h>
+#include <bvar/window.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstdint>
+#include <ctime>
+#include <functional>
+#include <limits>
+#include <optional>
+#include <random>
+#include <sstream>
+#include <unordered_set>
+
+#ifdef BE_TEST
+#include "cpp/sync_point.h"
+#endif
+
+#include "cloud/cloud_meta_mgr.h"
+#include "cloud/cloud_storage_engine.h"
+#include "cloud/config.h"
+#include "common/config.h"
+#include "gen_cpp/cloud.pb.h"
+#include "io/fs/packed_file_trailer.h"
+#include "olap/storage_engine.h"
+#include "runtime/exec_env.h"
+#include "util/coding.h"
+#include "util/uid_util.h"
+
+namespace doris::io {
+
+namespace {
+
+bvar::Adder<int64_t> g_packed_file_total_count("packed_file", "total_count");
+bvar::Adder<int64_t> g_packed_file_total_small_file_count("packed_file", "total_small_file_num");
+bvar::Adder<int64_t> g_packed_file_total_size_bytes("packed_file", "total_size_bytes");
+bvar::IntRecorder g_packed_file_small_file_num_recorder;
+bvar::IntRecorder g_packed_file_file_size_recorder;
+bvar::Window<bvar::IntRecorder> g_packed_file_avg_small_file_num(
+ "packed_file_avg_small_file_num", &g_packed_file_small_file_num_recorder,
+ /*window_size=*/10);
+bvar::Window<bvar::IntRecorder> g_packed_file_avg_file_size_bytes("packed_file_avg_file_size_bytes",
+ &g_packed_file_file_size_recorder,
+ /*window_size=*/10);
+bvar::IntRecorder g_packed_file_active_to_ready_ms_recorder;
+bvar::IntRecorder g_packed_file_ready_to_upload_ms_recorder;
+bvar::IntRecorder g_packed_file_uploading_to_uploaded_ms_recorder;
+bvar::Window<bvar::IntRecorder> g_packed_file_active_to_ready_ms_window(
+ "packed_file_active_to_ready_ms", &g_packed_file_active_to_ready_ms_recorder,
+ /*window_size=*/10);
+bvar::Window<bvar::IntRecorder> g_packed_file_ready_to_upload_ms_window(
+ "packed_file_ready_to_upload_ms", &g_packed_file_ready_to_upload_ms_recorder,
+ /*window_size=*/10);
+bvar::Window<bvar::IntRecorder> g_packed_file_uploading_to_uploaded_ms_window(
+ "packed_file_uploading_to_uploaded_ms", &g_packed_file_uploading_to_uploaded_ms_recorder,
+ /*window_size=*/10);
+
+Status append_packed_info_trailer(FileWriter* writer, const std::string& packed_file_path,
+ const cloud::PackedFileInfoPB& packed_file_info) {
+ if (writer == nullptr) {
+ return Status::InternalError("File writer is null for packed file: {}", packed_file_path);
+ }
+ if (writer->state() == FileWriter::State::CLOSED) {
+ return Status::InternalError("File writer already closed for packed file: {}",
+ packed_file_path);
+ }
+
+ cloud::PackedFileFooterPB footer_pb;
+ footer_pb.mutable_packed_file_info()->CopyFrom(packed_file_info);
+
+ std::string serialized_footer;
+ if (!footer_pb.SerializeToString(&serialized_footer)) {
+ return Status::InternalError("Failed to serialize packed file footer info for {}",
+ packed_file_path);
+ }
+
+ if (serialized_footer.size() >
+ std::numeric_limits<uint32_t>::max() - kPackedFileTrailerSuffixSize) {
+ return Status::InternalError("PackedFileFooterPB too large for {}", packed_file_path);
+ }
+
+ std::string trailer;
+ trailer.reserve(serialized_footer.size() + kPackedFileTrailerSuffixSize);
+ trailer.append(serialized_footer);
+ put_fixed32_le(&trailer, static_cast<uint32_t>(serialized_footer.size()));
+ put_fixed32_le(&trailer, kPackedFileTrailerVersion);
+
+ return writer->append(Slice(trailer));
+}
+
+} // namespace
+
+PackedFileManager* PackedFileManager::instance() {
+ static PackedFileManager instance;
+ return &instance;
+}
+
+PackedFileManager::~PackedFileManager() {
+ stop_background_manager();
+}
+
+Status PackedFileManager::init() {
+ return Status::OK();
+}
+
+Status PackedFileManager::create_new_packed_file_context(
+ const std::string& resource_id, std::unique_ptr<PackedFileContext>& packed_file_ctx) {
+ FileSystemSPtr file_system;
+ RETURN_IF_ERROR(ensure_file_system(resource_id, &file_system));
+ if (file_system == nullptr) {
+ return Status::InternalError("File system is not available for packed file creation: " +
+ resource_id);
+ }
+
+ auto uuid = generate_uuid_string();
+ auto hash_val = std::hash<std::string> {}(uuid);
+ uint16_t path_bucket = hash_val % 4096 + 1;
+ std::stringstream path_stream;
+ path_stream << "data/packed_file/" << path_bucket << "/" << uuid << ".bin";
+
+ packed_file_ctx = std::make_unique<PackedFileContext>();
+ const std::string relative_path = path_stream.str();
+ packed_file_ctx->packed_file_path = relative_path;
+ packed_file_ctx->create_time = std::time(nullptr);
+ packed_file_ctx->create_timestamp = std::chrono::steady_clock::now();
+ packed_file_ctx->state = PackedFileState::INIT;
+ packed_file_ctx->resource_id = resource_id;
+ packed_file_ctx->file_system = std::move(file_system);
+
+ // Create file writer for the packed file
+ FileWriterPtr new_writer;
+ FileWriterOptions opts;
+ // enable write file cache for packed file
+ opts.write_file_cache = true;
+ RETURN_IF_ERROR(
+ packed_file_ctx->file_system->create_file(Path(relative_path), &new_writer, &opts));
+ packed_file_ctx->writer = std::move(new_writer);
+
+ return Status::OK();
+}
+
+Status PackedFileManager::ensure_file_system(const std::string& resource_id,
+ FileSystemSPtr* file_system) {
+ if (file_system == nullptr) {
+ return Status::InvalidArgument("file_system output parameter is null");
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(_file_system_mutex);
+ if (resource_id.empty()) {
+ if (_default_file_system != nullptr) {
+ *file_system = _default_file_system;
+ return Status::OK();
+ }
+ } else {
+ auto it = _file_systems.find(resource_id);
+ if (it != _file_systems.end()) {
+ *file_system = it->second;
+ return Status::OK();
+ }
+ }
+ }
+
+ if (!config::is_cloud_mode()) {
+ return Status::InternalError("Cloud file system is not available in local mode");
+ }
+
+ auto* exec_env = ExecEnv::GetInstance();
+ if (exec_env == nullptr) {
+ return Status::InternalError("ExecEnv instance is not initialized");
+ }
+
+ FileSystemSPtr resolved_fs;
+ if (resource_id.empty()) {
+ resolved_fs = exec_env->storage_engine().to_cloud().latest_fs();
+ if (resolved_fs == nullptr) {
+ return Status::InternalError("Cloud file system is not ready");
+ }
+ } else {
+ auto storage_resource =
+ exec_env->storage_engine().to_cloud().get_storage_resource(resource_id);
+ if (!storage_resource.has_value() || storage_resource->fs == nullptr) {
+ return Status::InternalError("Cloud file system is not ready for resource: " +
+ resource_id);
+ }
+ resolved_fs = storage_resource->fs;
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(_file_system_mutex);
+ if (resource_id.empty()) {
+ _default_file_system = resolved_fs;
+ *file_system = _default_file_system;
+ } else {
+ _file_systems[resource_id] = resolved_fs;
+ *file_system = resolved_fs;
+ }
+ }
+
+ return Status::OK();
+}
+
+Status PackedFileManager::append_small_file(const std::string& path, const Slice& data,
+ const PackedAppendContext& info) {
+ // Check if file is too large to be merged
+ if (data.get_size() > config::small_file_threshold_bytes) {
+ return Status::OK(); // Skip merging for large files
+ }
+
+ if (info.txn_id <= 0) {
+ return Status::InvalidArgument("Missing valid txn id for packed file append: " + path);
+ }
+
+ std::lock_guard<std::timed_mutex> lock(_current_packed_file_mutex);
+
+ auto& current_state = _current_packed_files[info.resource_id];
+ if (!current_state || !current_state->writer) {
+ RETURN_IF_ERROR(create_new_packed_file_context(info.resource_id, current_state));
+ }
+
+ // Check if we need to create a new packed file
+ if (current_state->total_size + data.get_size() >= config::packed_file_size_threshold_bytes) {
+ RETURN_IF_ERROR(mark_current_packed_file_for_upload_locked(info.resource_id));
+ auto it = _current_packed_files.find(info.resource_id);
+ if (it == _current_packed_files.end() || !it->second || !it->second->writer) {
+ RETURN_IF_ERROR(create_new_packed_file_context(
+ info.resource_id, _current_packed_files[info.resource_id]));
+ }
+ }
+
+ PackedFileContext* active_state = current_state.get();
+
+ // Write data to current packed file
+ RETURN_IF_ERROR(active_state->writer->append(data));
+
+ // Update index
+ PackedSliceLocation location;
+ location.packed_file_path = active_state->packed_file_path;
+ location.offset = active_state->current_offset;
+ location.size = data.get_size();
+ location.create_time = std::time(nullptr);
+ location.tablet_id = info.tablet_id;
+ location.rowset_id = info.rowset_id;
+ location.resource_id = info.resource_id;
+ location.txn_id = info.txn_id;
+
+ active_state->slice_locations[path] = location;
+ active_state->current_offset += data.get_size();
+ active_state->total_size += data.get_size();
+
+ // Rotate packed file when small file count reaches threshold
+ if (config::packed_file_small_file_count_threshold > 0 &&
+ static_cast<int64_t>(active_state->slice_locations.size()) >=
+ config::packed_file_small_file_count_threshold) {
+ RETURN_IF_ERROR(mark_current_packed_file_for_upload_locked(info.resource_id));
+ }
+
+ // Mark as active if this is the first write
+ if (!active_state->first_append_timestamp.has_value()) {
+ active_state->first_append_timestamp = std::chrono::steady_clock::now();
+ }
+ if (active_state->state == PackedFileState::INIT) {
+ active_state->state = PackedFileState::ACTIVE;
+ }
+
+ // Update global index
+ {
+ std::lock_guard<std::mutex> global_lock(_global_index_mutex);
+ _global_slice_locations[path] = location;
+ }
+
+ return Status::OK();
+}
+
+Status PackedFileManager::wait_for_packed_file_upload(PackedFileContext* packed_file_ptr) {
+ std::unique_lock<std::mutex> upload_lock(packed_file_ptr->upload_mutex);
+ packed_file_ptr->upload_cv.wait(upload_lock, [packed_file_ptr] {
+ auto state = packed_file_ptr->state.load(std::memory_order_acquire);
+ return state == PackedFileState::UPLOADED || state == PackedFileState::FAILED;
+ });
+ if (packed_file_ptr->state == PackedFileState::FAILED) {
+ std::string err = packed_file_ptr->last_error;
+ if (err.empty()) {
+ err = "Packed file upload failed";
+ }
+ return Status::InternalError(err);
+ }
+ return Status::OK();
+}
+
+Status PackedFileManager::wait_upload_done(const std::string& path) {
+ std::string packed_file_path;
+ {
+ std::lock_guard<std::mutex> global_lock(_global_index_mutex);
+ auto it = _global_slice_locations.find(path);
+ if (it == _global_slice_locations.end()) {
+ return Status::InternalError("File not found in global index: " + path);
+ }
+ packed_file_path = it->second.packed_file_path;
+ }
+
+ // Find the packed file in uploaded files first - if already uploaded, no need to wait
+ std::shared_ptr<PackedFileContext> managed_packed_file;
+ std::shared_ptr<PackedFileContext> failed_packed_file;
+ {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ auto uploaded_it = _uploaded_packed_files.find(packed_file_path);
+ if (uploaded_it != _uploaded_packed_files.end()) {
+ auto state = uploaded_it->second->state.load(std::memory_order_acquire);
+ if (state == PackedFileState::UPLOADED) {
+ return Status::OK(); // Already uploaded, no need to wait
+ }
+ if (state == PackedFileState::FAILED) {
+ failed_packed_file = uploaded_it->second;
+ } else {
+ managed_packed_file = uploaded_it->second;
+ }
+ }
+ }
+
+ if (failed_packed_file) {
+ std::lock_guard<std::mutex> upload_lock(failed_packed_file->upload_mutex);
+ std::string err = failed_packed_file->last_error;
+ if (err.empty()) {
+ err = "Merge file upload failed";
+ }
+ return Status::InternalError(err);
+ }
+
+ // Find the packed file in either current or uploading files
+ PackedFileContext* packed_file_ptr = nullptr;
+ {
+ std::lock_guard<std::timed_mutex> current_lock(_current_packed_file_mutex);
+ for (auto& [resource_id, state] : _current_packed_files) {
+ if (state && state->packed_file_path == packed_file_path) {
+ packed_file_ptr = state.get();
+ break;
+ }
+ }
+ }
+
+ if (!packed_file_ptr) {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ auto uploading_it = _uploading_packed_files.find(packed_file_path);
+ if (uploading_it != _uploading_packed_files.end()) {
+ managed_packed_file = uploading_it->second;
+ packed_file_ptr = managed_packed_file.get();
+ } else {
+ auto uploaded_it = _uploaded_packed_files.find(packed_file_path);
+ if (uploaded_it != _uploaded_packed_files.end()) {
+ managed_packed_file = uploaded_it->second;
+ packed_file_ptr = managed_packed_file.get();
+ }
+ }
+ }
+
+ if (!packed_file_ptr) {
+ // Packed file not found in any location, this is unexpected
+ return Status::InternalError("Packed file not found for path: " + path);
+ }
+
+ Status wait_status = wait_for_packed_file_upload(packed_file_ptr);
+ (void)managed_packed_file; // keep shared ownership alive during wait
+ return wait_status;
+}
+
+Status PackedFileManager::get_packed_slice_location(const std::string& path,
+ PackedSliceLocation* location) {
+ std::lock_guard<std::mutex> lock(_global_index_mutex);
+ auto it = _global_slice_locations.find(path);
+ if (it == _global_slice_locations.end()) {
+ return Status::NotFound("File not found in global packed index: {}", path);
+ }
+
+ *location = it->second;
+ return Status::OK();
+}
+
+void PackedFileManager::start_background_manager() {
+ if (_background_thread) {
+ return; // Already started
+ }
+
+ _stop_background_thread = false;
+ _background_thread = std::make_unique<std::thread>([this] { background_manager(); });
+}
+
+void PackedFileManager::stop_background_manager() {
+ _stop_background_thread = true;
+ if (_background_thread && _background_thread->joinable()) {
+ _background_thread->join();
+ }
+ _background_thread.reset();
+}
+
+Status PackedFileManager::mark_current_packed_file_for_upload_locked(
+ const std::string& resource_id) {
+ auto it = _current_packed_files.find(resource_id);
+ if (it == _current_packed_files.end() || !it->second || !it->second->writer) {
+ return Status::OK(); // Nothing to mark for upload
+ }
+
+ auto& current = it->second;
+
+ // Mark as ready for upload
+ current->state = PackedFileState::READY_TO_UPLOAD;
+ if (!current->ready_to_upload_timestamp.has_value()) {
+ auto now = std::chrono::steady_clock::now();
+ current->ready_to_upload_timestamp = now;
+ int64_t active_to_ready_ms = -1;
+ if (current->first_append_timestamp.has_value()) {
+ active_to_ready_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *current->first_append_timestamp)
+ .count();
+ g_packed_file_active_to_ready_ms_recorder << active_to_ready_ms;
+ if (auto* sampler = g_packed_file_active_to_ready_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ }
+ VLOG_DEBUG << "Packed file " << current->packed_file_path
+ << " transition ACTIVE->READY_TO_UPLOAD; active_to_ready_ms="
+ << active_to_ready_ms;
+ }
+
+ // Move to uploading files list
+ {
+ std::shared_ptr<PackedFileContext> uploading_ptr =
+ std::shared_ptr<PackedFileContext>(std::move(current));
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ _uploading_packed_files[uploading_ptr->packed_file_path] = uploading_ptr;
+ }
+
+ // Create new packed file
+ return create_new_packed_file_context(resource_id, it->second);
+}
+
+Status PackedFileManager::mark_current_packed_file_for_upload(const std::string& resource_id) {
+ std::lock_guard<std::timed_mutex> lock(_current_packed_file_mutex);
+ return mark_current_packed_file_for_upload_locked(resource_id);
+}
+
+void PackedFileManager::record_packed_file_metrics(const PackedFileContext& packed_file) {
+ g_packed_file_total_count << 1;
+ g_packed_file_total_small_file_count
+ << static_cast<int64_t>(packed_file.slice_locations.size());
+ g_packed_file_total_size_bytes << packed_file.total_size;
+ g_packed_file_small_file_num_recorder
+ << static_cast<int64_t>(packed_file.slice_locations.size());
+ g_packed_file_file_size_recorder << packed_file.total_size;
+ // Flush samplers immediately so the window bvar reflects the latest packed file.
+ if (auto* sampler = g_packed_file_small_file_num_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ if (auto* sampler = g_packed_file_file_size_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+}
+
+void PackedFileManager::background_manager() {
+ auto last_cleanup_time = std::chrono::steady_clock::now();
+
+ while (!_stop_background_thread.load()) {
+ int64_t check_interval_ms =
+ std::max<int64_t>(1, config::packed_file_time_threshold_ms / 10);
+ std::this_thread::sleep_for(std::chrono::milliseconds(check_interval_ms));
+
+ // Check if current packed file should be closed due to time threshold
+ std::vector<std::string> resources_to_mark;
+ {
+ std::unique_lock<std::timed_mutex> current_lock(_current_packed_file_mutex,
+ std::defer_lock);
+ int64_t lock_wait_ms = std::max<int64_t>(0, config::packed_file_try_lock_timeout_ms);
+ if (current_lock.try_lock_for(std::chrono::milliseconds(lock_wait_ms))) {
+ for (auto& [resource_id, state] : _current_packed_files) {
+ if (!state || state->state != PackedFileState::ACTIVE) {
+ continue;
+ }
+ if (!state->first_append_timestamp.has_value()) {
+ continue;
+ }
+ auto current_time = std::chrono::steady_clock::now();
+ auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ current_time - *(state->first_append_timestamp))
+ .count();
+ if (elapsed_ms >= config::packed_file_time_threshold_ms) {
+ resources_to_mark.push_back(resource_id);
+ }
+ }
+ }
+ }
+ for (const auto& resource_id : resources_to_mark) {
+ Status st = mark_current_packed_file_for_upload(resource_id);
+ if (!st.ok()) {
+ LOG(WARNING) << "Failed to close current packed file for resource " << resource_id
+ << ": " << st.to_string();
+ }
+ }
+
+ // Process uploading files
+ process_uploading_packed_files();
+
+ auto now = std::chrono::steady_clock::now();
+ int64_t cleanup_interval_sec =
+ std::max<int64_t>(1, config::packed_file_cleanup_interval_seconds);
+ auto cleanup_interval = std::chrono::seconds(cleanup_interval_sec);
+ if (now - last_cleanup_time >= cleanup_interval) {
+ cleanup_expired_data();
+ last_cleanup_time = now;
+ }
+ }
+}
+
+void PackedFileManager::process_uploading_packed_files() {
+ std::vector<std::shared_ptr<PackedFileContext>> files_ready;
+ std::vector<std::shared_ptr<PackedFileContext>> files_uploading;
+ auto record_ready_to_upload = [&](const std::shared_ptr<PackedFileContext>& packed_file) {
+ if (!packed_file->uploading_timestamp.has_value()) {
+ packed_file->uploading_timestamp = std::chrono::steady_clock::now();
+ int64_t duration_ms = -1;
+ if (packed_file->ready_to_upload_timestamp.has_value()) {
+ duration_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ *packed_file->uploading_timestamp -
+ *packed_file->ready_to_upload_timestamp)
+ .count();
+ g_packed_file_ready_to_upload_ms_recorder << duration_ms;
+ if (auto* sampler = g_packed_file_ready_to_upload_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ }
+ VLOG_DEBUG << "Packed file " << packed_file->packed_file_path
+ << " transition READY_TO_UPLOAD->UPLOADING; "
+ "ready_to_upload_ms="
+ << duration_ms;
+ }
+ };
+
+ {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ for (auto& [packed_file_path, packed_file] : _uploading_packed_files) {
+ auto state = packed_file->state.load(std::memory_order_acquire);
+ if (state != PackedFileState::READY_TO_UPLOAD && state != PackedFileState::UPLOADING) {
+ continue;
+ }
+ if (state == PackedFileState::READY_TO_UPLOAD) {
+ files_ready.emplace_back(packed_file);
+ } else {
+ files_uploading.emplace_back(packed_file);
+ }
+ }
+ }
+
+ auto handle_success = [&](const std::shared_ptr<PackedFileContext>& packed_file) {
+ auto now = std::chrono::steady_clock::now();
+ int64_t active_to_ready_ms = -1;
+ int64_t ready_to_upload_ms = -1;
+ int64_t uploading_to_uploaded_ms = -1;
+ if (packed_file->first_append_timestamp.has_value() &&
+ packed_file->ready_to_upload_timestamp.has_value()) {
+ active_to_ready_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ *packed_file->ready_to_upload_timestamp -
+ *packed_file->first_append_timestamp)
+ .count();
+ }
+ if (packed_file->ready_to_upload_timestamp.has_value() &&
+ packed_file->uploading_timestamp.has_value()) {
+ ready_to_upload_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ *packed_file->uploading_timestamp -
+ *packed_file->ready_to_upload_timestamp)
+ .count();
+ }
+ if (packed_file->uploading_timestamp.has_value()) {
+ uploading_to_uploaded_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *packed_file->uploading_timestamp)
+ .count();
+ g_packed_file_uploading_to_uploaded_ms_recorder << uploading_to_uploaded_ms;
+ if (auto* sampler = g_packed_file_uploading_to_uploaded_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ }
+ int64_t total_ms = -1;
+ if (packed_file->first_append_timestamp.has_value()) {
+ total_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *packed_file->first_append_timestamp)
+ .count();
+ }
+ std::ostringstream slices_stream;
+ bool first_slice = true;
+ for (const auto& [small_file_path, index] : packed_file->slice_locations) {
+ if (!first_slice) {
+ slices_stream << "; ";
+ }
+ first_slice = false;
+ slices_stream << small_file_path << "(txn=" << index.txn_id
+ << ", offset=" << index.offset << ", size=" << index.size << ")";
+
+ // Update packed_file_size in global index
+ {
+ std::lock_guard<std::mutex> global_lock(_global_index_mutex);
+ auto it = _global_slice_locations.find(small_file_path);
+ if (it != _global_slice_locations.end()) {
+ it->second.packed_file_size = packed_file->total_size;
+ }
+ }
+ }
+ LOG(INFO) << "Packed file " << packed_file->packed_file_path
+ << " uploaded; slices=" << packed_file->slice_locations.size()
+ << ", total_bytes=" << packed_file->total_size << ", slice_detail=["
+ << slices_stream.str() << "]"
+ << ", active_to_ready_ms=" << active_to_ready_ms
+ << ", ready_to_upload_ms=" << ready_to_upload_ms
+ << ", uploading_to_uploaded_ms=" << uploading_to_uploaded_ms
+ << ", total_ms=" << total_ms;
+ {
+ std::lock_guard<std::mutex> upload_lock(packed_file->upload_mutex);
+ packed_file->state = PackedFileState::UPLOADED;
+ packed_file->upload_time = std::time(nullptr);
+ }
+ packed_file->upload_cv.notify_all();
+ {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ _uploading_packed_files.erase(packed_file->packed_file_path);
+ _uploaded_packed_files[packed_file->packed_file_path] = packed_file;
+ }
+ };
+
+ auto handle_failure = [&](const std::shared_ptr<PackedFileContext>& packed_file,
+ const Status& status) {
+ LOG(WARNING) << "Failed to upload packed file: " << packed_file->packed_file_path
+ << ", error: " << status.to_string();
+ {
+ std::lock_guard<std::mutex> upload_lock(packed_file->upload_mutex);
+ packed_file->state = PackedFileState::FAILED;
+ packed_file->last_error = status.to_string();
+ packed_file->upload_time = std::time(nullptr);
+ }
+ packed_file->upload_cv.notify_all();
+ {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ _uploading_packed_files.erase(packed_file->packed_file_path);
+ _uploaded_packed_files[packed_file->packed_file_path] = packed_file;
+ }
+ };
+
+ for (auto& packed_file : files_ready) {
+ const std::string& packed_file_path = packed_file->packed_file_path;
+ cloud::PackedFileInfoPB packed_file_info;
+ packed_file_info.set_ref_cnt(packed_file->slice_locations.size());
+ packed_file_info.set_total_slice_num(packed_file->slice_locations.size());
+ packed_file_info.set_total_slice_bytes(packed_file->total_size);
+ packed_file_info.set_remaining_slice_bytes(packed_file->total_size);
+ packed_file_info.set_created_at_sec(packed_file->create_time);
+ packed_file_info.set_corrected(false);
+ packed_file_info.set_state(doris::cloud::PackedFileInfoPB::NORMAL);
+ packed_file_info.set_resource_id(packed_file->resource_id);
+
+ for (const auto& [small_file_path, index] : packed_file->slice_locations) {
+ auto* small_file = packed_file_info.add_slices();
+ small_file->set_path(small_file_path);
+ small_file->set_offset(index.offset);
+ small_file->set_size(index.size);
+ small_file->set_deleted(false);
+ if (index.tablet_id != 0) {
+ small_file->set_tablet_id(index.tablet_id);
+ }
+ if (!index.rowset_id.empty()) {
+ small_file->set_rowset_id(index.rowset_id);
+ }
+ if (index.txn_id != 0) {
+ small_file->set_txn_id(index.txn_id);
+ }
+ }
+
+ Status meta_status = update_meta_service(packed_file->packed_file_path, packed_file_info);
+ if (!meta_status.ok()) {
+ LOG(WARNING) << "Failed to update meta service for packed file: "
+ << packed_file->packed_file_path << ", error: " << meta_status.to_string();
+ handle_failure(packed_file, meta_status);
+ continue;
+ }
+
+ // Record stats once the packed file metadata is persisted.
+ record_packed_file_metrics(*packed_file);
+
+ Status trailer_status = append_packed_info_trailer(
+ packed_file->writer.get(), packed_file->packed_file_path, packed_file_info);
+ if (!trailer_status.ok()) {
+ handle_failure(packed_file, trailer_status);
+ continue;
+ }
+
+ // Now upload the file
+ if (!packed_file->slice_locations.empty()) {
+ std::ostringstream oss;
+ oss << "Uploading packed file " << packed_file_path << " with "
+ << packed_file->slice_locations.size() << " small files: ";
+ bool first = true;
+ for (const auto& [small_file_path, index] : packed_file->slice_locations) {
+ if (!first) {
+ oss << ", ";
+ }
+ first = false;
+ oss << "[" << small_file_path << ", offset=" << index.offset
+ << ", size=" << index.size << "]";
+ }
+ VLOG_DEBUG << oss.str();
+ } else {
+ VLOG_DEBUG << "Uploading packed file " << packed_file_path << " with no small files";
+ }
+
+ Status upload_status = finalize_packed_file_upload(packed_file->packed_file_path,
+ packed_file->writer.get());
+
+ if (upload_status.is<ErrorCode::ALREADY_CLOSED>()) {
+ record_ready_to_upload(packed_file);
+ handle_success(packed_file);
+ continue;
+ }
+ if (!upload_status.ok()) {
+ handle_failure(packed_file, upload_status);
+ continue;
+ }
+
+ record_ready_to_upload(packed_file);
+ packed_file->state = PackedFileState::UPLOADING;
+ }
+
+ for (auto& packed_file : files_uploading) {
+ if (!packed_file->writer) {
+ handle_failure(packed_file,
+ Status::InternalError("File writer is null for packed file: {}",
+ packed_file->packed_file_path));
+ continue;
+ }
+
+ if (packed_file->writer->state() != FileWriter::State::CLOSED) {
+ continue;
+ }
+
+ Status status = packed_file->writer->close(true);
+ if (status.is<ErrorCode::ALREADY_CLOSED>()) {
+ handle_success(packed_file);
+ continue;
+ }
+ if (status.ok()) {
+ continue;
+ }
+
+ handle_failure(packed_file, status);
+ }
+}
+
+Status PackedFileManager::finalize_packed_file_upload(const std::string& packed_file_path,
+ FileWriter* writer) {
+ if (writer == nullptr) {
+ return Status::InternalError("File writer is null for packed file: " + packed_file_path);
+ }
+
+ return writer->close(true);
+}
+
+Status PackedFileManager::update_meta_service(const std::string& packed_file_path,
+ const cloud::PackedFileInfoPB& packed_file_info) {
+#ifdef BE_TEST
+ TEST_SYNC_POINT_RETURN_WITH_VALUE("PackedFileManager::update_meta_service", Status::OK(),
+ packed_file_path, &packed_file_info);
+#endif
+ VLOG_DEBUG << "Updating meta service for packed file: " << packed_file_path << " with "
+ << packed_file_info.total_slice_num() << " small files"
+ << ", total bytes: " << packed_file_info.total_slice_bytes();
+
+ // Get CloudMetaMgr through StorageEngine
+ if (!config::is_cloud_mode()) {
+ return Status::InternalError("Storage engine is not cloud mode");
+ }
+
+ auto& storage_engine = ExecEnv::GetInstance()->storage_engine();
+ auto& cloud_meta_mgr = storage_engine.to_cloud().meta_mgr();
+ return cloud_meta_mgr.update_packed_file_info(packed_file_path, packed_file_info);
+}
+
+void PackedFileManager::cleanup_expired_data() {
+ auto current_time = std::time(nullptr);
+
+ // Clean up expired uploaded files
+ {
+ std::lock_guard<std::mutex> uploaded_lock(_packed_files_mutex);
+ auto it = _uploaded_packed_files.begin();
+ while (it != _uploaded_packed_files.end()) {
+ if (it->second->state == PackedFileState::UPLOADED &&
+ current_time - it->second->upload_time > config::uploaded_file_retention_seconds) {
+ it = _uploaded_packed_files.erase(it);
+ } else if (it->second->state == PackedFileState::FAILED &&
+ current_time - it->second->upload_time >
+ config::uploaded_file_retention_seconds) {
+ it = _uploaded_packed_files.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+
+ // Clean up expired global index entries
+ {
+ std::lock_guard<std::mutex> global_lock(_global_index_mutex);
+ auto it = _global_slice_locations.begin();
+ while (it != _global_slice_locations.end()) {
+ const auto& index = it->second;
+ if (index.create_time > 0 &&
+ current_time - index.create_time > config::uploaded_file_retention_seconds) {
+ it = _global_slice_locations.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ }
+}
+
+#ifdef BE_TEST
+namespace {
+void reset_adder(bvar::Adder<int64_t>& adder) {
+ auto current = adder.get_value();
+ if (current != 0) {
+ adder << (-current);
+ }
+}
+} // namespace
+
+void PackedFileManager::reset_packed_file_bvars_for_test() const {
+ reset_adder(g_packed_file_total_count);
+ reset_adder(g_packed_file_total_small_file_count);
+ reset_adder(g_packed_file_total_size_bytes);
+ g_packed_file_small_file_num_recorder.reset();
+ g_packed_file_file_size_recorder.reset();
+ g_packed_file_active_to_ready_ms_recorder.reset();
+ g_packed_file_ready_to_upload_ms_recorder.reset();
+ g_packed_file_uploading_to_uploaded_ms_recorder.reset();
+ if (auto* sampler = g_packed_file_small_file_num_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ if (auto* sampler = g_packed_file_file_size_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ if (auto* sampler = g_packed_file_active_to_ready_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ if (auto* sampler = g_packed_file_ready_to_upload_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ if (auto* sampler = g_packed_file_uploading_to_uploaded_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+}
+
+int64_t PackedFileManager::packed_file_total_count_for_test() const {
+ return g_packed_file_total_count.get_value();
+}
+
+int64_t PackedFileManager::packed_file_total_small_file_num_for_test() const {
+ return g_packed_file_total_small_file_count.get_value();
+}
+
+int64_t PackedFileManager::packed_file_total_size_bytes_for_test() const {
+ return g_packed_file_total_size_bytes.get_value();
+}
+
+double PackedFileManager::packed_file_avg_small_file_num_for_test() const {
+ return g_packed_file_avg_small_file_num.get_value().get_average_double();
+}
+
+double PackedFileManager::packed_file_avg_file_size_for_test() const {
+ return g_packed_file_avg_file_size_bytes.get_value().get_average_double();
+}
+
+void PackedFileManager::record_packed_file_metrics_for_test(
+ const PackedFileManager::PackedFileContext* packed_file) {
+ DCHECK(packed_file != nullptr);
+ record_packed_file_metrics(*packed_file);
+}
+
+void PackedFileManager::clear_state_for_test() {
+ std::lock_guard<std::timed_mutex> cur_lock(_current_packed_file_mutex);
+ _current_packed_files.clear();
+ {
+ std::lock_guard<std::mutex> lock(_packed_files_mutex);
+ _uploading_packed_files.clear();
+ _uploaded_packed_files.clear();
+ }
+ {
+ std::lock_guard<std::mutex> lock(_global_index_mutex);
+ _global_slice_locations.clear();
+ }
+}
+#endif
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_manager.h b/be/src/io/fs/packed_file_manager.h
new file mode 100644
index 00000000000..bfa83a240e0
--- /dev/null
+++ b/be/src/io/fs/packed_file_manager.h
@@ -0,0 +1,207 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <butil/macros.h>
+#include <gen_cpp/cloud.pb.h>
+#include <glog/logging.h>
+
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <string>
+#include <thread>
+#include <unordered_map>
+#include <vector>
+
+#include "common/status.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/path.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+struct PackedSliceLocation {
+ std::string packed_file_path;
+ int64_t offset;
+ int64_t size;
+ int64_t create_time = 0;
+ int64_t tablet_id = 0;
+ std::string rowset_id;
+ std::string resource_id;
+ int64_t txn_id = 0;
+ int64_t packed_file_size = -1; // Total size of the packed file, -1 means not set
+};
+
+struct PackedAppendContext {
+ std::string resource_id;
+ int64_t tablet_id = 0;
+ std::string rowset_id;
+ int64_t txn_id = 0;
+};
+
+// Global object that manages packing small files into larger files for S3 optimization
+class PackedFileManager {
+ struct PackedFileContext;
+
+public:
+ static PackedFileManager* instance();
+
+ // Initialize manager state; file system will be resolved lazily
+ Status init();
+
+ // Write a small file to the current packed file
+ Status append_small_file(const std::string& path, const Slice& data,
+ const PackedAppendContext& info);
+
+ // Block until the small file's packed file is uploaded to S3
+ Status wait_upload_done(const std::string& path);
+
+ // Get packed file index information for a small file
+ Status get_packed_slice_location(const std::string& path, PackedSliceLocation* location);
+
+ // Start the background management thread
+ void start_background_manager();
+
+ // Stop the background management thread
+ void stop_background_manager();
+
+ // Mark current packed file for upload and create new one
+ Status mark_current_packed_file_for_upload(const std::string& resource_id);
+
+ // Internal helper; expects caller holds _current_packed_file_mutex
+ Status mark_current_packed_file_for_upload_locked(const std::string& resource_id);
+
+ void record_packed_file_metrics(const PackedFileContext& packed_file);
+
+private:
+ PackedFileManager() = default;
+ ~PackedFileManager();
+
+ DISALLOW_COPY_AND_ASSIGN(PackedFileManager);
+
+ // Background thread function for managing packed file lifecycle
+ void background_manager();
+
+ // Upload packed file to S3 and update meta service
+ Status finalize_packed_file_upload(const std::string& packed_file_path, FileWriter* writer);
+
+ // Update meta service with packed file information
+ Status update_meta_service(const std::string& packed_file_path,
+ const cloud::PackedFileInfoPB& packed_file_info);
+
+ // Process uploading files
+ void process_uploading_packed_files();
+
+ // Clean up expired data
+ void cleanup_expired_data();
+
+ // Internal structure to track packed file state
+ enum class PackedFileState {
+ INIT, // Initial state, no files written yet
+ ACTIVE, // Has files but doesn't meet upload conditions
+ READY_TO_UPLOAD, // Ready for upload, metadata still being prepared
+ UPLOADING, // Upload triggered, waiting for writer close to finish
+ UPLOADED, // Upload completed
+ FAILED // Upload failed
+ };
+
+ struct PackedFileContext {
+ std::string packed_file_path;
+ std::unique_ptr<FileWriter> writer;
+ std::unordered_map<std::string, PackedSliceLocation> slice_locations;
+ int64_t current_offset = 0;
+ int64_t total_size = 0;
+ int64_t create_time;
+ int64_t upload_time = 0;
+ std::chrono::steady_clock::time_point create_timestamp;
+ std::optional<std::chrono::steady_clock::time_point> first_append_timestamp;
+ std::optional<std::chrono::steady_clock::time_point> ready_to_upload_timestamp;
+ std::optional<std::chrono::steady_clock::time_point> uploading_timestamp;
+ std::atomic<PackedFileState> state {PackedFileState::INIT};
+ std::condition_variable upload_cv;
+ std::mutex upload_mutex;
+ std::string last_error;
+ std::string resource_id;
+ FileSystemSPtr file_system;
+ };
+
+ // Create a new packed file state with file writer
+ Status create_new_packed_file_context(const std::string& resource_id,
+ std::unique_ptr<PackedFileContext>& packed_file_ctx);
+
+ Status ensure_file_system(const std::string& resource_id, FileSystemSPtr* file_system);
+
+ // Helper function to wait for packed file upload completion
+ Status wait_for_packed_file_upload(PackedFileContext* packed_file_ptr);
+
+ // Thread management
+ std::atomic<bool> _stop_background_thread {false};
+ std::unique_ptr<std::thread> _background_thread;
+
+ // File system
+ FileSystemSPtr _default_file_system;
+ std::unordered_map<std::string, FileSystemSPtr> _file_systems;
+ std::mutex _file_system_mutex;
+
+ // Current active packed file
+ std::unordered_map<std::string, std::unique_ptr<PackedFileContext>> _current_packed_files;
+ std::timed_mutex _current_packed_file_mutex;
+
+ // Merge files ready for upload or being processed
+ std::unordered_map<std::string, std::shared_ptr<PackedFileContext>> _uploading_packed_files;
+
+ // Uploaded packed files (kept for some time for wait_write_done)
+ std::unordered_map<std::string, std::shared_ptr<PackedFileContext>> _uploaded_packed_files;
+ std::mutex _packed_files_mutex;
+
+ // Global index mapping small file path to packed file index
+ std::unordered_map<std::string, PackedSliceLocation> _global_slice_locations;
+ std::mutex _global_index_mutex;
+
+#ifdef BE_TEST
+public:
+ void reset_packed_file_bvars_for_test() const;
+ int64_t packed_file_total_count_for_test() const;
+ int64_t packed_file_total_small_file_num_for_test() const;
+ int64_t packed_file_total_size_bytes_for_test() const;
+ double packed_file_avg_small_file_num_for_test() const;
+ double packed_file_avg_file_size_for_test() const;
+ void record_packed_file_metrics_for_test(const PackedFileContext* packed_file);
+
+ // Test-only helpers to introspect/clear internal state
+ void clear_state_for_test();
+ auto& current_packed_files_for_test() { return _current_packed_files; }
+ auto& uploading_packed_files_for_test() { return _uploading_packed_files; }
+ auto& uploaded_packed_files_for_test() { return _uploaded_packed_files; }
+ auto& global_slice_locations_for_test() { return _global_slice_locations; }
+ auto& file_systems_for_test() { return _file_systems; }
+ FileSystemSPtr& default_file_system_for_test() { return _default_file_system; }
+ Status create_new_packed_file_state_for_test(const std::string& resource_id,
+ std::unique_ptr<PackedFileContext>& ctx) {
+ return create_new_packed_file_context(resource_id, ctx);
+ }
+#endif
+};
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_reader.cpp b/be/src/io/fs/packed_file_reader.cpp
new file mode 100644
index 00000000000..ae8cc0322ad
--- /dev/null
+++ b/be/src/io/fs/packed_file_reader.cpp
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_reader.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "common/logging.h"
+#include "common/status.h"
+#include "io/fs/file_reader.h"
+
+namespace doris::io {
+
+PackedFileReader::PackedFileReader(FileReaderSPtr inner_reader, Path path, int64_t offset,
+ int64_t size)
+ : _inner_reader(std::move(inner_reader)),
+ _path(std::move(path)),
+ _packed_file_offset(offset),
+ _file_size(size) {
+ DCHECK(_inner_reader != nullptr);
+}
+
+PackedFileReader::~PackedFileReader() {
+ if (!_closed) {
+ static_cast<void>(close());
+ }
+}
+
+Status PackedFileReader::read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) {
+ if (_closed) {
+ return Status::InternalError("FileReader is already closed");
+ }
+
+ // Calculate the actual offset in packed file
+ size_t actual_offset = _packed_file_offset + offset;
+
+ // Calculate the maximum bytes we can read
+ size_t max_read = std::min(result.get_size(), static_cast<size_t>(_file_size - offset));
+
+ // Adjust result slice to the actual size we can read
+ Slice adjusted_result(result.get_data(), max_read);
+
+ // Read from packed file at the adjusted offset
+ auto s = _inner_reader->read_at(actual_offset, adjusted_result, bytes_read, io_ctx);
+ if (!s.ok()) {
+ LOG(WARNING) << "failed to read packed file: " << _path.native() << ", offset: " << offset
+ << ", actual offset: " << actual_offset
+ << ", result size: " << adjusted_result.get_size()
+ << ", error: " << s.to_string()
+ << ", packed file offset: " << _packed_file_offset
+ << ", file size: " << _file_size;
+ return s;
+ }
+
+ return Status::OK();
+}
+
+Status PackedFileReader::close() {
+ if (_closed) {
+ return Status::OK();
+ }
+
+ if (_inner_reader) {
+ RETURN_IF_ERROR(_inner_reader->close());
+ }
+
+ _closed = true;
+ return Status::OK();
+}
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_reader.h b/be/src/io/fs/packed_file_reader.h
new file mode 100644
index 00000000000..79cd23c8cd7
--- /dev/null
+++ b/be/src/io/fs/packed_file_reader.h
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+
+#include "butil/macros.h"
+#include "common/status.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/path.h"
+
+namespace doris::io {
+
+struct IOContext;
+
+// FileReader wrapper that reads data from packed file using offset and size
+// It wraps an inner reader that points to the packed file,
+// and adjusts read offsets based on the slice's position in the packed file
+class PackedFileReader final : public FileReader {
+public:
+ PackedFileReader(FileReaderSPtr inner_reader, Path path, int64_t offset, int64_t size);
+ ~PackedFileReader() override;
+
+ PackedFileReader(const PackedFileReader&) = delete;
+ const PackedFileReader& operator=(const PackedFileReader&) = delete;
+
+ Status close() override;
+
+ const Path& path() const override { return _path; }
+
+ size_t size() const override { return _file_size; }
+
+ bool closed() const override { return _closed; }
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* io_ctx) override;
+
+private:
+ FileReaderSPtr _inner_reader;
+ Path _path;
+ int64_t _packed_file_offset; // Offset in packed file where this slice starts
+ int64_t _file_size; // Size of the slice
+ bool _closed = false;
+};
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_system.cpp b/be/src/io/fs/packed_file_system.cpp
new file mode 100644
index 00000000000..dd5b136ba3b
--- /dev/null
+++ b/be/src/io/fs/packed_file_system.cpp
@@ -0,0 +1,114 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_system.h"
+
+#include <utility>
+
+#include "common/status.h"
+#include "io/fs/packed_file_reader.h"
+#include "io/fs/packed_file_writer.h"
+
+namespace doris::io {
+
+PackedFileSystem::PackedFileSystem(FileSystemSPtr inner_fs, PackedAppendContext append_info)
+ : FileSystem(inner_fs->id(), inner_fs->type()),
+ _inner_fs(std::move(inner_fs)),
+ _append_info(std::move(append_info)) {
+ if (_append_info.resource_id.empty() && _inner_fs != nullptr) {
+ _append_info.resource_id = _inner_fs->id();
+ }
+}
+
+PackedFileSystem::PackedFileSystem(FileSystemSPtr inner_fs,
+ std::unordered_map<std::string, PackedSliceLocation> index_map,
+ PackedAppendContext append_info)
+ : FileSystem(inner_fs->id(), inner_fs->type()),
+ _inner_fs(std::move(inner_fs)),
+ _index_map(std::move(index_map)),
+ _append_info(std::move(append_info)) {
+ if (_append_info.resource_id.empty() && _inner_fs != nullptr) {
+ _append_info.resource_id = _inner_fs->id();
+ }
+ _index_map_initialized = true;
+}
+
+Status PackedFileSystem::create_file_impl(const Path& file, FileWriterPtr* writer,
+ const FileWriterOptions* opts) {
+ // Create file using inner file system
+ FileWriterPtr inner_writer;
+ RETURN_IF_ERROR(_inner_fs->create_file(file, &inner_writer, opts));
+
+ // Wrap with PackedFileWriter
+ *writer = std::make_unique<PackedFileWriter>(std::move(inner_writer), file, _append_info);
+ return Status::OK();
+}
+
+Status PackedFileSystem::open_file_impl(const Path& file, FileReaderSPtr* reader,
+ const FileReaderOptions* opts) {
+ // Check if this file is in a packed file
+ std::string file_path = file.native();
+ auto it = _index_map.find(file_path);
+ bool is_packed_file = (it != _index_map.end());
+
+ if (is_packed_file) {
+ // File is in packed file, open packed file and wrap with PackedFileReader
+ const auto& index = it->second;
+ FileReaderSPtr inner_reader;
+ // Create a new FileReaderOptions with the correct file size
+ FileReaderOptions local_opts = opts ? *opts : FileReaderOptions();
+ // Set file_size to packed file size to avoid head object request
+ local_opts.file_size = index.packed_file_size;
+ LOG(INFO) << "open packed file: " << index.packed_file_path << ", file: " << file.native()
+ << ", offset: " << index.offset << ", size: " << index.size
+ << ", packed_file_size: " << index.packed_file_size;
+ RETURN_IF_ERROR(
+ _inner_fs->open_file(Path(index.packed_file_path), &inner_reader, &local_opts));
+
+ *reader = std::make_shared<PackedFileReader>(std::move(inner_reader), file, index.offset,
+ index.size);
+ } else {
+ RETURN_IF_ERROR(_inner_fs->open_file(file, reader, opts));
+ }
+ return Status::OK();
+}
+
+Status PackedFileSystem::exists_impl(const Path& path, bool* res) const {
+ LOG(INFO) << "packed file system exist, rowset id " << _append_info.rowset_id;
+ if (!_index_map_initialized) {
+ return Status::InternalError("PackedFileSystem index map is not initialized");
+ }
+ const auto it = _index_map.find(path.native());
+ if (it != _index_map.end()) {
+ return _inner_fs->exists(Path(it->second.packed_file_path), res);
+ }
+ return _inner_fs->exists(path, res);
+}
+
+Status PackedFileSystem::file_size_impl(const Path& file, int64_t* file_size) const {
+ if (!_index_map_initialized) {
+ return Status::InternalError("PackedFileSystem index map is not initialized");
+ }
+ const auto it = _index_map.find(file.native());
+ if (it != _index_map.end()) {
+ *file_size = it->second.size;
+ return Status::OK();
+ }
+ return _inner_fs->file_size(file, file_size);
+}
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_system.h b/be/src/io/fs/packed_file_system.h
new file mode 100644
index 00000000000..7ba5c030713
--- /dev/null
+++ b/be/src/io/fs/packed_file_system.h
@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include "common/status.h"
+#include "io/fs/file_system.h"
+#include "io/fs/packed_file_manager.h"
+
+namespace doris::io {
+
+// FileSystem wrapper that handles packed file logic for small files
+// When creating a file, it wraps the writer with PackedFileWriter
+// When opening a file, it checks if the file is in a packed file and wraps with PackedFileReader
+class PackedFileSystem final : public FileSystem {
+public:
+ PackedFileSystem(FileSystemSPtr inner_fs, PackedAppendContext append_info = {});
+
+ PackedFileSystem(FileSystemSPtr inner_fs,
+ std::unordered_map<std::string, PackedSliceLocation> index_map,
+ PackedAppendContext append_info = {});
+
+ ~PackedFileSystem() override = default;
+
+ PackedFileSystem(const PackedFileSystem&) = delete;
+ PackedFileSystem& operator=(const PackedFileSystem&) = delete;
+
+protected:
+ // Create file and wrap writer with PackedFileWriter
+ Status create_file_impl(const Path& file, FileWriterPtr* writer,
+ const FileWriterOptions* opts) override;
+
+ // Open file and wrap reader with PackedFileReader if file is in packed file
+ Status open_file_impl(const Path& file, FileReaderSPtr* reader,
+ const FileReaderOptions* opts) override;
+
+ // Other operations not supported for PackedFileSystem
+ Status create_directory_impl(const Path& dir, bool failed_if_exists = false) override {
+ return Status::InternalError("PackedFileSystem does not support create_directory");
+ }
+
+ Status delete_file_impl(const Path& file) override {
+ return Status::InternalError("PackedFileSystem does not support delete_file");
+ }
+
+ Status batch_delete_impl(const std::vector<Path>& files) override {
+ return Status::InternalError("PackedFileSystem does not support batch_delete");
+ }
+
+ Status delete_directory_impl(const Path& dir) override {
+ return Status::InternalError("PackedFileSystem does not support delete_directory");
+ }
+
+ Status exists_impl(const Path& path, bool* res) const override;
+
+ Status file_size_impl(const Path& file, int64_t* file_size) const override;
+
+ Status list_impl(const Path& dir, bool only_file, std::vector<FileInfo>* files,
+ bool* exists) override {
+ return Status::InternalError("PackedFileSystem does not support list");
+ }
+
+ Status rename_impl(const Path& orig_name, const Path& new_name) override {
+ return Status::InternalError("PackedFileSystem does not support rename");
+ }
+
+ Status absolute_path(const Path& path, Path& abs_path) const override {
+ abs_path = path;
+ return Status::OK();
+ }
+
+private:
+ FileSystemSPtr _inner_fs;
+ // Map from small file path to packed file slice location
+ std::unordered_map<std::string, PackedSliceLocation> _index_map;
+ bool _index_map_initialized = false;
+ PackedAppendContext _append_info;
+};
+
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_trailer.cpp b/be/src/io/fs/packed_file_trailer.cpp
new file mode 100644
index 00000000000..29ddf9db603
--- /dev/null
+++ b/be/src/io/fs/packed_file_trailer.cpp
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_trailer.h"
+
+#include <array>
+#include <fstream>
+
+#include "common/status.h"
+#include "util/coding.h"
+
+namespace doris::io {
+
+Status parse_packed_file_trailer(std::string_view data, cloud::PackedFileFooterPB* debug_pb,
+ uint32_t* version) {
+ if (debug_pb == nullptr || version == nullptr) {
+ return Status::InvalidArgument("Output parameters must not be null");
+ }
+ if (data.size() < kPackedFileTrailerSuffixSize) {
+ return Status::InternalError("Packed file too small to contain trailer");
+ }
+
+ const size_t suffix_offset = data.size() - kPackedFileTrailerSuffixSize;
+ const auto* suffix_ptr = reinterpret_cast<const uint8_t*>(data.data() + suffix_offset);
+ const uint32_t trailer_size = decode_fixed32_le(suffix_ptr);
+ const uint32_t trailer_version = decode_fixed32_le(suffix_ptr + sizeof(uint32_t));
+
+ // Preferred format: [PackedFileFooterPB][length][version]
+ if (trailer_size > 0 && trailer_size <= data.size() - kPackedFileTrailerSuffixSize) {
+ const size_t payload_offset = data.size() - kPackedFileTrailerSuffixSize - trailer_size;
+ std::string_view payload(data.data() + payload_offset, trailer_size);
+ if (payload.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
+ return Status::InternalError("Packed file trailer payload too large");
+ }
+ cloud::PackedFileFooterPB parsed_pb;
+ if (parsed_pb.ParseFromArray(payload.data(), static_cast<int>(payload.size()))) {
+ debug_pb->Swap(&parsed_pb);
+ *version = trailer_version;
+ return Status::OK();
+ }
+ }
+
+ // Legacy format fallback: [PackedFileInfoPB][length]
+ if (data.size() < sizeof(uint32_t)) {
+ return Status::InternalError("Packed file trailer corrupted");
+ }
+ const size_t legacy_suffix_offset = data.size() - sizeof(uint32_t);
+ const auto* legacy_ptr = reinterpret_cast<const uint8_t*>(data.data() + legacy_suffix_offset);
+ const uint32_t legacy_size = decode_fixed32_le(legacy_ptr);
+ if (legacy_size == 0 || legacy_size > data.size() - sizeof(uint32_t)) {
+ return Status::InternalError("Packed file trailer corrupted");
+ }
+ const size_t legacy_payload_offset = data.size() - sizeof(uint32_t) - legacy_size;
+ std::string_view legacy_payload(data.data() + legacy_payload_offset, legacy_size);
+ cloud::PackedFileInfoPB packed_info;
+ if (legacy_payload.size() > static_cast<size_t>(std::numeric_limits<int>::max())) {
+ return Status::InternalError("Packed file legacy trailer payload too large");
+ }
+ if (!packed_info.ParseFromArray(legacy_payload.data(),
+ static_cast<int>(legacy_payload.size()))) {
+ return Status::InternalError("Failed to parse packed file trailer");
+ }
+ debug_pb->Clear();
+ debug_pb->mutable_packed_file_info()->Swap(&packed_info);
+ *version = 0;
+ return Status::OK();
+}
+
+Status read_packed_file_trailer(const std::string& file_path, cloud::PackedFileFooterPB* debug_pb,
+ uint32_t* version) {
+ if (debug_pb == nullptr || version == nullptr) {
+ return Status::InvalidArgument("Output parameters must not be null");
+ }
+
+ std::ifstream file(file_path, std::ios::binary);
+ if (!file.is_open()) {
+ return Status::IOError("Failed to open packed file {}", file_path);
+ }
+
+ file.seekg(0, std::ios::end);
+ const std::streamoff file_size = file.tellg();
+ if (file_size < static_cast<std::streamoff>(sizeof(uint32_t))) {
+ return Status::InternalError("Packed file {} is too small", file_path);
+ }
+
+ auto read_tail = [&](std::streamoff count, std::string* out) -> Status {
+ out->assign(static_cast<size_t>(count), '\0');
+ file.seekg(file_size - count);
+ file.read(out->data(), count);
+ if (!file) {
+ return Status::IOError("Failed to read last {} bytes from {}", count, file_path);
+ }
+ return Status::OK();
+ };
+
+ // Try new format first.
+ if (file_size >= static_cast<std::streamoff>(kPackedFileTrailerSuffixSize)) {
+ std::array<char, kPackedFileTrailerSuffixSize> suffix {};
+ file.seekg(file_size - static_cast<std::streamoff>(suffix.size()));
+ file.read(suffix.data(), suffix.size());
+ if (file) {
+ const uint32_t trailer_size =
+ decode_fixed32_le(reinterpret_cast<uint8_t*>(suffix.data()));
+ const uint32_t trailer_version =
+ decode_fixed32_le(reinterpret_cast<uint8_t*>(suffix.data()) + sizeof(uint32_t));
+ const std::streamoff required =
+ static_cast<std::streamoff>(kPackedFileTrailerSuffixSize + trailer_size);
+ if (trailer_size > 0 && file_size >= required) {
+ std::string tail;
+ RETURN_IF_ERROR(read_tail(required, &tail));
+ Status st = parse_packed_file_trailer(tail, debug_pb, version);
+ if (st.ok() && *version == trailer_version) {
+ return st;
+ }
+ }
+ }
+ file.clear();
+ }
+
+ // Legacy fallback: PackedFileInfoPB + length.
+ std::array<char, sizeof(uint32_t)> legacy_suffix {};
+ file.seekg(file_size - static_cast<std::streamoff>(legacy_suffix.size()));
+ file.read(legacy_suffix.data(), legacy_suffix.size());
+ if (!file) {
+ return Status::IOError("Failed to read legacy trailer length from {}", file_path);
+ }
+ const uint32_t legacy_size =
+ decode_fixed32_le(reinterpret_cast<uint8_t*>(legacy_suffix.data()));
+ const std::streamoff required = static_cast<std::streamoff>(sizeof(uint32_t) + legacy_size);
+ if (legacy_size == 0 || file_size < required) {
+ return Status::InternalError("Packed file trailer corrupted for {}", file_path);
+ }
+ std::string tail;
+ RETURN_IF_ERROR(read_tail(required, &tail));
+ return parse_packed_file_trailer(tail, debug_pb, version);
+}
+
+} // namespace doris::io
diff --git a/be/src/cloud/cloud_rowset_writer.h b/be/src/io/fs/packed_file_trailer.h
similarity index 58%
copy from be/src/cloud/cloud_rowset_writer.h
copy to be/src/io/fs/packed_file_trailer.h
index 1bb2b3d38b8..9741ed95c06 100644
--- a/be/src/cloud/cloud_rowset_writer.h
+++ b/be/src/io/fs/packed_file_trailer.h
@@ -17,19 +17,22 @@
#pragma once
-#include "olap/rowset/beta_rowset_writer.h"
+#include <cstdint>
+#include <string>
+#include <string_view>
-namespace doris {
+#include "common/status.h"
+#include "gen_cpp/cloud.pb.h"
-class CloudRowsetWriter : public BaseBetaRowsetWriter {
-public:
- CloudRowsetWriter();
+namespace doris::io {
- ~CloudRowsetWriter() override;
+constexpr uint32_t kPackedFileTrailerVersion = 1;
+constexpr size_t kPackedFileTrailerSuffixSize = sizeof(uint32_t) * 2;
- Status init(const RowsetWriterContext& rowset_writer_context) override;
+Status parse_packed_file_trailer(std::string_view data, cloud::PackedFileFooterPB* debug_pb,
+ uint32_t* version);
- Status build(RowsetSharedPtr& rowset) override;
-};
+Status read_packed_file_trailer(const std::string& file_path, cloud::PackedFileFooterPB* debug_pb,
+ uint32_t* version);
-} // namespace doris
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_writer.cpp b/be/src/io/fs/packed_file_writer.cpp
new file mode 100644
index 00000000000..6a345e0d8a8
--- /dev/null
+++ b/be/src/io/fs/packed_file_writer.cpp
@@ -0,0 +1,233 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_writer.h"
+
+#include <bvar/recorder.h>
+#include <bvar/window.h>
+#include <glog/logging.h>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "common/status.h"
+#include "runtime/exec_env.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+bvar::IntRecorder packed_file_writer_first_append_to_close_ms_recorder;
+bvar::Window<bvar::IntRecorder> packed_file_writer_first_append_to_close_ms_window(
+ "packed_file_writer_first_append_to_close_ms",
+ &packed_file_writer_first_append_to_close_ms_recorder, /*window_size=*/10);
+
+PackedFileWriter::PackedFileWriter(FileWriterPtr inner_writer, Path path,
+ PackedAppendContext append_info)
+ : _inner_writer(std::move(inner_writer)),
+ _file_path(path.native()),
+ _packed_file_manager(PackedFileManager::instance()),
+ _append_info(std::move(append_info)) {
+ DCHECK(_inner_writer != nullptr);
+ DCHECK(!_file_path.empty());
+}
+
+PackedFileWriter::~PackedFileWriter() {
+ if (_state == State::OPENED) {
+ LOG(WARNING) << "PackedFileWriter destroyed without being closed, file: " << _file_path;
+ }
+}
+
+Status PackedFileWriter::appendv(const Slice* data, size_t data_cnt) {
+ if (_state != State::OPENED) {
+ return Status::InternalError("Cannot append to closed or closing writer for file: " +
+ _file_path);
+ }
+
+ if (!_first_append_timestamp.has_value()) {
+ _first_append_timestamp = std::chrono::steady_clock::now();
+ }
+
+ // Calculate total size to append
+ size_t total_size = 0;
+ for (size_t i = 0; i < data_cnt; ++i) {
+ total_size += data[i].size;
+ }
+
+ if (total_size == 0) {
+ return Status::OK();
+ }
+
+ // Check if we should switch to direct write mode
+ if (!_is_direct_write && _bytes_appended + total_size > config::small_file_threshold_bytes) {
+ RETURN_IF_ERROR(_switch_to_direct_write());
+ _is_direct_write = true;
+ }
+
+ // Write data based on current mode
+ if (_is_direct_write) {
+ RETURN_IF_ERROR(_inner_writer->appendv(data, data_cnt));
+ } else {
+ // Buffer small file data
+ for (size_t i = 0; i < data_cnt; ++i) {
+ _buffer.append(data[i].data, data[i].size);
+ }
+ }
+
+ _bytes_appended += total_size;
+ return Status::OK();
+}
+
+Status PackedFileWriter::close(bool non_block) {
+ if (_state == State::CLOSED) {
+ return Status::OK();
+ }
+
+ auto record_close_latency = [this]() {
+ if (_close_latency_recorded || !_first_append_timestamp.has_value()) {
+ return;
+ }
+ auto now = std::chrono::steady_clock::now();
+ auto latency_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *_first_append_timestamp)
+ .count();
+ packed_file_writer_first_append_to_close_ms_recorder << latency_ms;
+ if (auto* sampler = packed_file_writer_first_append_to_close_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ _close_latency_recorded = true;
+ };
+
+ if (_state == State::ASYNC_CLOSING) {
+ if (non_block) {
+ return Status::InternalError("Don't submit async close multi times");
+ }
+ if (!_is_direct_write) {
+ RETURN_IF_ERROR(_wait_packed_upload());
+ } else {
+ RETURN_IF_ERROR(_inner_writer->close(false));
+ }
+ _state = State::CLOSED;
+ if (!non_block) {
+ record_close_latency();
+ }
+ return Status::OK();
+ }
+
+ if (non_block) {
+ return _close_async();
+ } else {
+ return _close_sync();
+ }
+}
+
+Status PackedFileWriter::_close_async() {
+ if (!_is_direct_write) {
+ // Send small file data to packed manager
+ RETURN_IF_ERROR(_send_to_packed_manager());
+ } else {
+ // For large files, just close the inner writer asynchronously
+ RETURN_IF_ERROR(_inner_writer->close(true));
+ }
+ _state = State::ASYNC_CLOSING;
+ return Status::OK();
+}
+
+Status PackedFileWriter::_close_sync() {
+ if (!_is_direct_write) {
+ // Send small file data to pack manager and wait for upload
+ RETURN_IF_ERROR(_send_to_packed_manager());
+ RETURN_IF_ERROR(_wait_packed_upload());
+ } else {
+ // For large files, close the inner writer synchronously
+ RETURN_IF_ERROR(_inner_writer->close(false));
+ }
+ _state = State::CLOSED;
+ if (!_close_latency_recorded) {
+ auto now = std::chrono::steady_clock::now();
+ if (_first_append_timestamp.has_value()) {
+ auto latency_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *_first_append_timestamp)
+ .count();
+ packed_file_writer_first_append_to_close_ms_recorder << latency_ms;
+ if (auto* sampler =
+ packed_file_writer_first_append_to_close_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ _close_latency_recorded = true;
+ }
+ }
+ return Status::OK();
+}
+
+Status PackedFileWriter::_wait_packed_upload() {
+ DCHECK(!_is_direct_write);
+ // Only wait if we have data that was sent to packed manager
+ if (_bytes_appended > 0 && _packed_file_manager != nullptr) {
+ return _packed_file_manager->wait_upload_done(_file_path);
+ }
+ return Status::OK();
+}
+
+Status PackedFileWriter::_switch_to_direct_write() {
+ DCHECK(!_is_direct_write);
+
+ // If we have buffered data, write it to inner writer first
+ if (_buffer.size() > 0) {
+ Slice buffer_slice(_buffer.data(), _buffer.size());
+ RETURN_IF_ERROR(_inner_writer->appendv(&buffer_slice, 1));
+ _buffer.clear();
+ }
+
+ return Status::OK();
+}
+
+Status PackedFileWriter::_send_to_packed_manager() {
+ DCHECK(!_is_direct_write);
+
+ if (_packed_file_manager == nullptr) {
+ return Status::InternalError("PackedFileManager is not available");
+ }
+ LOG(INFO) << "send_to_packed_manager: " << _file_path << " buffer size: " << _buffer.size();
+
+ if (_append_info.resource_id.empty()) {
+ return Status::InternalError("Missing resource id for packed file append");
+ }
+
+ if (_append_info.txn_id <= 0) {
+ return Status::InvalidArgument("Missing valid txn id for packed file append: " +
+ _file_path);
+ }
+
+ Slice data_slice(_buffer.data(), _buffer.size());
+ RETURN_IF_ERROR(_packed_file_manager->append_small_file(_file_path, data_slice, _append_info));
+ _buffer.clear();
+ return Status::OK();
+}
+
+Status PackedFileWriter::get_packed_slice_location(PackedSliceLocation* location) const {
+ DCHECK(_state == State::CLOSED)
+ << " file_path: " << _file_path << " bytes_appended: " << _bytes_appended;
+ if (_is_direct_write) {
+ *location = PackedSliceLocation {};
+ return Status::OK();
+ }
+ RETURN_IF_ERROR(_packed_file_manager->get_packed_slice_location(_file_path, location));
+ LOG(INFO) << "get_packed_slice_location: " << _file_path
+ << " packed_path: " << location->packed_file_path << " " << location->offset << " "
+ << location->size;
+ return Status::OK();
+}
+} // namespace doris::io
diff --git a/be/src/io/fs/packed_file_writer.h b/be/src/io/fs/packed_file_writer.h
new file mode 100644
index 00000000000..9499b0912a4
--- /dev/null
+++ b/be/src/io/fs/packed_file_writer.h
@@ -0,0 +1,91 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <chrono>
+#include <optional>
+#include <string>
+
+#include "common/status.h"
+#include "io/fs/file_reader_writer_fwd.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/packed_file_manager.h"
+#include "io/fs/path.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+// FileWriter wrapper that buffers small files and writes them to packed file
+// If file size exceeds threshold, it directly writes to inner writer
+// Otherwise, it buffers data and writes to PackedFileManager on close
+class PackedFileWriter final : public FileWriter {
+public:
+ PackedFileWriter(FileWriterPtr inner_writer, Path path,
+ PackedAppendContext append_info = PackedAppendContext {});
+ ~PackedFileWriter() override;
+
+ PackedFileWriter(const PackedFileWriter&) = delete;
+ const PackedFileWriter& operator=(const PackedFileWriter&) = delete;
+
+ Status appendv(const Slice* data, size_t data_cnt) override;
+
+ const Path& path() const override { return _inner_writer->path(); }
+
+ size_t bytes_appended() const override { return _bytes_appended; }
+
+ State state() const override { return _state; }
+
+ Status close(bool non_block = false) override;
+
+ // Get merge file segment index information
+ // This method should be called after close(true) to get the index information
+ // Returns empty index if file is not in merge file
+ Status get_packed_slice_location(PackedSliceLocation* location) const;
+
+private:
+ // Async close: submit data without waiting
+ Status _close_async();
+
+ // Sync close: submit data and wait for completion
+ Status _close_sync();
+
+ // Switch from merge file mode to direct write mode
+ Status _switch_to_direct_write();
+
+ // Send buffered data to merge file manager
+ Status _send_to_packed_manager();
+
+ // Wait for packed file to be uploaded to S3
+ Status _wait_packed_upload();
+
+ FileWriterPtr _inner_writer;
+ std::string _file_path; // Store file path as string for packed file manager
+
+ // Buffer for small files
+ std::string _buffer;
+ size_t _bytes_appended = 0;
+ State _state = State::OPENED;
+ bool _is_direct_write = false; // Whether to use direct write mode
+ PackedFileManager* _packed_file_manager = nullptr; // Packed file manager instance
+ mutable PackedSliceLocation _packed_slice_location; // Packed slice info (mutable for lazy init)
+ PackedAppendContext _append_info;
+ std::optional<std::chrono::steady_clock::time_point> _first_append_timestamp;
+ bool _close_latency_recorded = false;
+};
+
+} // namespace doris::io
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 79e19967e77..a1b72b8dc98 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -18,7 +18,9 @@
#include "io/fs/s3_file_writer.h"
#include <aws/s3/model/CompletedPart.h>
+#include <bvar/recorder.h>
#include <bvar/reducer.h>
+#include <bvar/window.h>
#include <fmt/core.h>
#include <glog/logging.h>
@@ -52,6 +54,10 @@ bvar::Adder<uint64_t> s3_file_being_written("s3_file_writer_file_being_written")
bvar::Adder<uint64_t> s3_file_writer_async_close_queuing("s3_file_writer_async_close_queuing");
bvar::Adder<uint64_t> s3_file_writer_async_close_processing(
"s3_file_writer_async_close_processing");
+bvar::IntRecorder s3_file_writer_first_append_to_close_ms_recorder;
+bvar::Window<bvar::IntRecorder> s3_file_writer_first_append_to_close_ms_window(
+ "s3_file_writer_first_append_to_close_ms",
+ &s3_file_writer_first_append_to_close_ms_recorder, /*window_size=*/10);
S3FileWriter::S3FileWriter(std::shared_ptr<ObjClientHolder> client, std::string bucket,
std::string key, const FileWriterOptions* opts)
@@ -120,10 +126,48 @@ void S3FileWriter::_wait_until_finish(std::string_view task_name) {
}
Status S3FileWriter::close(bool non_block) {
+ auto record_close_latency = [this]() {
+ if (_close_latency_recorded || !_first_append_timestamp.has_value()) {
+ return;
+ }
+ auto now = std::chrono::steady_clock::now();
+ auto latency_ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+ now - *_first_append_timestamp)
+ .count();
+ s3_file_writer_first_append_to_close_ms_recorder << latency_ms;
+ if (auto* sampler = s3_file_writer_first_append_to_close_ms_recorder.get_sampler()) {
+ sampler->take_sample();
+ }
+ _close_latency_recorded = true;
+ };
+
if (state() == State::CLOSED) {
- return Status::InternalError("S3FileWriter already closed, file path {}, file key {}",
- _obj_storage_path_opts.path.native(),
- _obj_storage_path_opts.key);
+ if (_async_close_pack != nullptr) {
+ _st = _async_close_pack->future.get();
+ _async_close_pack = nullptr;
+ // Return the final close status so that a blocking close issued after
+ // an async close observes the real result just like the legacy behavior.
+ if (!non_block && _st.ok()) {
+ record_close_latency();
+ }
+ return _st;
+ }
+ if (non_block) {
+ if (_st.ok()) {
+ record_close_latency();
+ return Status::Error<ErrorCode::ALREADY_CLOSED>(
+ "S3FileWriter already closed, file path {}, file key {}",
+ _obj_storage_path_opts.path.native(), _obj_storage_path_opts.key);
+ }
+ return _st;
+ }
+ if (_st.ok()) {
+ record_close_latency();
+ return Status::Error<ErrorCode::ALREADY_CLOSED>(
+ "S3FileWriter already closed, file path {}, file key {}",
+ _obj_storage_path_opts.path.native(), _obj_storage_path_opts.key);
+ }
+ return _st;
}
if (state() == State::ASYNC_CLOSING) {
if (non_block) {
@@ -136,6 +180,9 @@ Status S3FileWriter::close(bool non_block) {
_state = State::CLOSED;
// The next time we call close() with no matter non_block true or false, it would always return the
// '_st' value because this writer is already closed.
+ if (!non_block && _st.ok()) {
+ record_close_latency();
+ }
return _st;
}
if (non_block) {
@@ -146,12 +193,17 @@ Status S3FileWriter::close(bool non_block) {
return ExecEnv::GetInstance()->non_block_close_thread_pool()->submit_func([&]() {
s3_file_writer_async_close_queuing << -1;
s3_file_writer_async_close_processing << 1;
- _async_close_pack->promise.set_value(_close_impl());
+ _st = _close_impl();
+ _state = State::CLOSED;
+ _async_close_pack->promise.set_value(_st);
s3_file_writer_async_close_processing << -1;
});
}
_st = _close_impl();
_state = State::CLOSED;
+ if (!non_block && _st.ok()) {
+ record_close_latency();
+ }
return _st;
}
@@ -236,6 +288,10 @@ Status S3FileWriter::appendv(const Slice* data, size_t data_cnt) {
_obj_storage_path_opts.path.native());
}
+ if (!_first_append_timestamp.has_value()) {
+ _first_append_timestamp = std::chrono::steady_clock::now();
+ }
+
size_t buffer_size = config::s3_write_buffer_size;
TEST_SYNC_POINT_RETURN_WITH_VALUE("s3_file_writer::appenv", Status());
for (size_t i = 0; i < data_cnt; i++) {
diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h
index a3d114e668d..38e68b14c4d 100644
--- a/be/src/io/fs/s3_file_writer.h
+++ b/be/src/io/fs/s3_file_writer.h
@@ -19,8 +19,10 @@
#include <bthread/countdown_event.h>
+#include <chrono>
#include <cstddef>
#include <memory>
+#include <optional>
#include <string>
#include "common/status.h"
@@ -111,6 +113,8 @@ private:
std::unique_ptr<AsyncCloseStatusPack> _async_close_pack;
State _state {State::OPENED};
std::shared_ptr<ObjClientHolder> _obj_client;
+ std::optional<std::chrono::steady_clock::time_point> _first_append_timestamp;
+ bool _close_latency_recorded = false;
};
} // namespace io
diff --git a/be/src/olap/calc_delete_bitmap_executor.h b/be/src/olap/calc_delete_bitmap_executor.h
index a9faf019a97..babfd987e5d 100644
--- a/be/src/olap/calc_delete_bitmap_executor.h
+++ b/be/src/olap/calc_delete_bitmap_executor.h
@@ -22,6 +22,7 @@
#include <atomic>
#include <iosfwd>
#include <memory>
+#include <shared_mutex>
#include <utility>
#include <vector>
@@ -61,6 +62,26 @@ public:
const std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap);
+ // submit a generic function to the thread pool
+ template <typename Func>
+ Status submit_func(Func&& func) {
+ {
+ std::shared_lock rlock(_lock);
+ RETURN_IF_ERROR(_status);
+ _resource_ctx = thread_context()->resource_ctx();
+ }
+ return _thread_token->submit_func([this, func = std::forward<Func>(func)]() {
+ SCOPED_ATTACH_TASK(_resource_ctx);
+ auto st = func();
+ if (!st.ok()) {
+ std::lock_guard wlock(_lock);
+ if (_status.ok()) {
+ _status = st;
+ }
+ }
+ });
+ }
+
// wait all tasks in token to be completed.
Status wait();
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 543fad1f3dd..b1f98d9d3db 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -1160,6 +1160,7 @@ Status CompactionMixin::construct_output_rowset_writer(RowsetWriterContext& ctx)
ctx.newest_write_timestamp = _newest_write_timestamp;
ctx.write_type = DataWriteType::TYPE_COMPACTION;
ctx.compaction_type = compaction_type();
+ ctx.allow_packed_file = false;
_output_rs_writer = DORIS_TRY(_tablet->create_rowset_writer(ctx, _is_vertical));
_pending_rs_guard = _engine.add_pending_rowset(ctx);
return Status::OK();
@@ -1616,6 +1617,7 @@ Status CloudCompactionMixin::construct_output_rowset_writer(RowsetWriterContext&
ctx.newest_write_timestamp = _newest_write_timestamp;
ctx.write_type = DataWriteType::TYPE_COMPACTION;
ctx.compaction_type = compaction_type();
+ ctx.allow_packed_file = false;
// We presume that the data involved in cumulative compaction is sufficiently 'hot'
// and should always be retained in the cache.
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
index 46d00bb55fa..fbaaf8b5740 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -206,16 +206,24 @@ Status InvertedIndexFileCollection::add(int seg_id, IndexFileWriterPtr&& index_w
return Status::OK();
}
-Status InvertedIndexFileCollection::close() {
+Status InvertedIndexFileCollection::begin_close() {
std::lock_guard lock(_lock);
for (auto&& [id, writer] : _inverted_index_file_writers) {
- RETURN_IF_ERROR(writer->close());
+ RETURN_IF_ERROR(writer->begin_close());
_total_size += writer->get_index_file_total_size();
}
return Status::OK();
}
+Status InvertedIndexFileCollection::finish_close() {
+ std::lock_guard lock(_lock);
+ for (auto&& [id, writer] : _inverted_index_file_writers) {
+ RETURN_IF_ERROR(writer->finish_close());
+ }
+ return Status::OK();
+}
+
Result<std::vector<const InvertedIndexFileInfo*>>
InvertedIndexFileCollection::inverted_index_file_info(int seg_id_offset) {
std::lock_guard lock(_lock);
@@ -324,34 +332,82 @@ Status BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
(_context.partial_update_info && _context.partial_update_info->is_partial_update())) {
return Status::OK();
}
- RowsetSharedPtr rowset_ptr;
- RETURN_IF_ERROR(_build_tmp(rowset_ptr));
- auto beta_rowset = reinterpret_cast<BetaRowset*>(rowset_ptr.get());
- std::vector<segment_v2::SegmentSharedPtr> segments;
- RETURN_IF_ERROR(beta_rowset->load_segments(segment_id, segment_id + 1, &segments));
std::vector<RowsetSharedPtr> specified_rowsets;
{
std::shared_lock meta_rlock(_context.tablet->get_header_lock());
specified_rowsets =
_context.tablet->get_rowset_by_ids(_context.mow_context->rowset_ids.get());
}
- OlapStopWatch watch;
- RETURN_IF_ERROR(BaseTablet::calc_delete_bitmap(
- _context.tablet, rowset_ptr, segments, specified_rowsets,
- _context.mow_context->delete_bitmap, _context.mow_context->max_version,
- _calc_delete_bitmap_token.get()));
- size_t total_rows = std::accumulate(
- segments.begin(), segments.end(), 0,
- [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
- LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: " << _context.tablet->tablet_id()
- << ", rowset_ids: " << _context.mow_context->rowset_ids->size()
- << ", cur max_version: " << _context.mow_context->max_version
- << ", transaction_id: " << _context.mow_context->txn_id << ", delete_bitmap_count: "
- << _context.mow_context->delete_bitmap->get_delete_bitmap_count()
- << ", delete_bitmap_cardinality: "
- << _context.mow_context->delete_bitmap->cardinality()
- << ", cost: " << watch.get_elapse_time_us() << "(us), total rows: " << total_rows;
- return Status::OK();
+
+ // Submit the entire delete bitmap calculation process to thread pool for async execution
+ // This avoids blocking memtable flush thread while waiting for file upload to complete
+ // The process includes: file_writer->close(), _build_tmp, load_segments, and calc_delete_bitmap
+ return _calc_delete_bitmap_token->submit_func(
+ [this, segment_id, specified_rowsets = std::move(specified_rowsets)]() -> Status {
+ Status st = Status::OK();
+ // Step 1: Close file_writer (must be done before load_segments)
+ auto* file_writer = _seg_files.get(segment_id);
+ if (file_writer && file_writer->state() != io::FileWriter::State::CLOSED) {
+ MonotonicStopWatch close_timer;
+ close_timer.start();
+ st = file_writer->close();
+ close_timer.stop();
+
+ auto close_time_ms = close_timer.elapsed_time_milliseconds();
+ if (close_time_ms > 1000) {
+ LOG(INFO) << "file_writer->close() took " << close_time_ms
+ << "ms for segment_id=" << segment_id
+ << ", tablet_id=" << _context.tablet_id
+ << ", rowset_id=" << _context.rowset_id;
+ }
+ if (!st.ok()) {
+ return st;
+ }
+ }
+
+ OlapStopWatch watch;
+ // Step 2: Build tmp rowset (needs file_writer to be closed)
+ RowsetSharedPtr rowset_ptr;
+ st = _build_tmp(rowset_ptr);
+ if (!st.ok()) {
+ return st;
+ }
+
+ // Step 3: Load segments (needs file_writer to be closed and rowset to be built)
+ auto* beta_rowset = reinterpret_cast<BetaRowset*>(rowset_ptr.get());
+ std::vector<segment_v2::SegmentSharedPtr> segments;
+ st = beta_rowset->load_segments(segment_id, segment_id + 1, &segments);
+ if (!st.ok()) {
+ return st;
+ }
+
+ // Step 4: Calculate delete bitmap
+ st = BaseTablet::calc_delete_bitmap(
+ _context.tablet, rowset_ptr, segments, specified_rowsets,
+ _context.mow_context->delete_bitmap, _context.mow_context->max_version,
+ nullptr, nullptr, nullptr);
+ if (!st.ok()) {
+ return st;
+ }
+
+ size_t total_rows =
+ std::accumulate(segments.begin(), segments.end(), 0,
+ [](size_t sum, const segment_v2::SegmentSharedPtr& s) {
+ return sum += s->num_rows();
+ });
+ LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: "
+ << _context.tablet->tablet_id()
+ << ", rowset_ids: " << _context.mow_context->rowset_ids->size()
+ << ", cur max_version: " << _context.mow_context->max_version
+ << ", transaction_id: " << _context.mow_context->txn_id
+ << ", delete_bitmap_count: "
+ << _context.mow_context->delete_bitmap->get_delete_bitmap_count()
+ << ", delete_bitmap_cardinality: "
+ << _context.mow_context->delete_bitmap->cardinality()
+ << ", cost: " << watch.get_elapse_time_us()
+ << "(us), total rows: " << total_rows;
+ return Status::OK();
+ });
}
Status BetaRowsetWriter::init(const RowsetWriterContext& rowset_writer_context) {
@@ -947,7 +1003,6 @@ Status BaseBetaRowsetWriter::_build_rowset_meta(RowsetMeta* rowset_meta, bool ch
// TODO write zonemap to meta
rowset_meta->set_empty((num_rows_written + _num_rows_written) == 0);
rowset_meta->set_creation_time(time(nullptr));
-
return Status::OK();
}
@@ -1050,7 +1105,8 @@ Status BetaRowsetWriter::create_segment_writer_for_segcompaction(
_segcompaction_worker->get_file_writer().reset(file_writer.release());
if (auto& idx_file_writer = _segcompaction_worker->get_inverted_index_file_writer();
idx_file_writer != nullptr) {
- RETURN_IF_ERROR(idx_file_writer->close());
+ RETURN_IF_ERROR(idx_file_writer->begin_close());
+ RETURN_IF_ERROR(idx_file_writer->finish_close());
}
_segcompaction_worker->get_inverted_index_file_writer().reset(index_file_writer.release());
return Status::OK();
@@ -1109,22 +1165,6 @@ Status BaseBetaRowsetWriter::add_segment(uint32_t segment_id, const SegmentStati
}
if (_context.mow_context != nullptr) {
- // ensure that the segment file writing is complete
- auto* file_writer = _seg_files.get(segment_id);
- if (file_writer && file_writer->state() != io::FileWriter::State::CLOSED) {
- MonotonicStopWatch close_timer;
- close_timer.start();
- RETURN_IF_ERROR(file_writer->close());
- close_timer.stop();
-
- auto close_time_ms = close_timer.elapsed_time_milliseconds();
- if (close_time_ms > 1000) {
- LOG(INFO) << "file_writer->close() took " << close_time_ms
- << "ms for segment_id=" << segment_id
- << ", tablet_id=" << _context.tablet_id
- << ", rowset_id=" << _context.rowset_id;
- }
- }
RETURN_IF_ERROR(_generate_delete_bitmap(segment_id));
}
return Status::OK();
diff --git a/be/src/olap/rowset/beta_rowset_writer.h b/be/src/olap/rowset/beta_rowset_writer.h
index 26547f7ece2..49802bb95eb 100644
--- a/be/src/olap/rowset/beta_rowset_writer.h
+++ b/be/src/olap/rowset/beta_rowset_writer.h
@@ -91,7 +91,10 @@ public:
// Close all file writers
// If the inverted index file writer is not closed, an error will be thrown during destruction
- Status close();
+ Status begin_close();
+
+ // Wait for all inverted index file writers to be closed
+ Status finish_close();
// Get inverted index file info in segment id order.
// `seg_id_offset` is the offset of the segment id relative to the subscript of `_inverted_index_file_writers`,
@@ -198,7 +201,7 @@ private:
// for this segment
protected:
Status _generate_delete_bitmap(int32_t segment_id);
- Status _build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num = false);
+ virtual Status _build_rowset_meta(RowsetMeta* rowset_meta, bool check_segment_num = false);
Status _create_file_writer(const std::string& path, io::FileWriterPtr& file_writer);
virtual Status _close_file_writers();
virtual Status _check_segment_number_limit(size_t segnum);
@@ -214,9 +217,11 @@ protected:
// Some index files are written during normal compaction and some files are written during index compaction.
// After all index writes are completed, call this method to write the final compound index file.
Status _close_inverted_index_file_writers() {
- RETURN_NOT_OK_STATUS_WITH_WARN(_idx_files.close(),
+ RETURN_NOT_OK_STATUS_WITH_WARN(_idx_files.begin_close(),
"failed to close index file when build new rowset");
this->_total_index_size += _idx_files.get_total_index_size();
+ RETURN_NOT_OK_STATUS_WITH_WARN(_idx_files.finish_close(),
+ "failed to wait close index file when build new rowset");
return Status::OK();
}
diff --git a/be/src/olap/rowset/rowset_factory.cpp b/be/src/olap/rowset/rowset_factory.cpp
index 1524b9682f0..8e01d77aaea 100644
--- a/be/src/olap/rowset/rowset_factory.cpp
+++ b/be/src/olap/rowset/rowset_factory.cpp
@@ -74,9 +74,9 @@ Result<std::unique_ptr<RowsetWriter>> RowsetFactory::create_rowset_writer(
// TODO(plat1ko): cloud vertical rowset writer
std::unique_ptr<RowsetWriter> writer;
if (is_vertical) {
- writer = std::make_unique<VerticalBetaRowsetWriter<CloudRowsetWriter>>();
+ writer = std::make_unique<VerticalBetaRowsetWriter<CloudRowsetWriter>>(engine);
} else {
- writer = std::make_unique<CloudRowsetWriter>();
+ writer = std::make_unique<CloudRowsetWriter>(engine);
}
RETURN_IF_ERROR_RESULT(writer->init(context));
diff --git a/be/src/olap/rowset/rowset_meta.cpp b/be/src/olap/rowset/rowset_meta.cpp
index 27ede602200..63408a8452e 100644
--- a/be/src/olap/rowset/rowset_meta.cpp
+++ b/be/src/olap/rowset/rowset_meta.cpp
@@ -31,6 +31,8 @@
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
+#include "io/fs/packed_file_manager.h"
+#include "io/fs/packed_file_system.h"
#include "json2pb/json_to_pb.h"
#include "json2pb/pb_to_json.h"
#include "olap/base_tablet.h"
@@ -130,7 +132,33 @@ io::FileSystemSPtr RowsetMeta::fs() {
// TODO: return a Result<FileSystemSPtr> in this method?
return nullptr;
}
- return io::make_file_system(fs, algorithm.value());
+
+ auto wrapped = io::make_file_system(fs, algorithm.value());
+
+ // Apply packed file system if enabled and index_map is not empty
+ if (_rowset_meta_pb.packed_slice_locations_size() > 0) {
+ std::unordered_map<std::string, io::PackedSliceLocation> index_map;
+ for (const auto& [path, index_pb] : _rowset_meta_pb.packed_slice_locations()) {
+ io::PackedSliceLocation index;
+ index.packed_file_path = index_pb.packed_file_path();
+ index.offset = index_pb.offset();
+ index.size = index_pb.size();
+ index.packed_file_size =
+ index_pb.has_packed_file_size() ? index_pb.packed_file_size() : -1;
+ index.tablet_id = tablet_id();
+ index.rowset_id = _rowset_id.to_string();
+ index.resource_id = wrapped->id();
+ index_map[path] = index;
+ }
+ if (!index_map.empty()) {
+ io::PackedAppendContext append_info;
+ append_info.tablet_id = tablet_id();
+ append_info.rowset_id = _rowset_id.to_string();
+ append_info.txn_id = txn_id();
+ wrapped = std::make_shared<io::PackedFileSystem>(wrapped, index_map, append_info);
+ }
+ }
+ return wrapped;
#else
return fs;
#endif
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 4cd346ce414..48f55075ef1 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -424,6 +424,17 @@ public:
RowsetMeta(const RowsetMeta&) = delete;
RowsetMeta operator=(const RowsetMeta&) = delete;
+ void add_packed_slice_location(const std::string& segment_path,
+ const std::string& packed_file_path, int64_t offset,
+ int64_t size, int64_t packed_file_size) {
+ auto* index_map = _rowset_meta_pb.mutable_packed_slice_locations();
+ auto& index_pb = (*index_map)[segment_path];
+ index_pb.set_packed_file_path(packed_file_path);
+ index_pb.set_offset(offset);
+ index_pb.set_size(size);
+ index_pb.set_packed_file_size(packed_file_size);
+ }
+
private:
bool _deserialize_from_pb(std::string_view value);
diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h
index b61f52d9632..fde047c2795 100644
--- a/be/src/olap/rowset/rowset_writer_context.h
+++ b/be/src/olap/rowset/rowset_writer_context.h
@@ -22,11 +22,15 @@
#include <functional>
#include <optional>
+#include <string_view>
+#include <unordered_map>
+#include "cloud/config.h"
#include "common/status.h"
#include "io/fs/encrypted_fs_factory.h"
#include "io/fs/file_system.h"
#include "io/fs/file_writer.h"
+#include "io/fs/packed_file_system.h"
#include "olap/olap_define.h"
#include "olap/partial_update_info.h"
#include "olap/storage_policy.h"
@@ -114,6 +118,21 @@ struct RowsetWriterContext {
bool is_transient_rowset_writer = false;
+ // Intent flag: caller can actively turn merge-file feature on/off for this rowset.
+ // This describes whether we *want* to try small-file merging.
+ bool allow_packed_file = true;
+
+ // Effective flag: whether this context actually ends up using MergeFileSystem for writes.
+ // This is decided inside fs() based on enable_merge_file plus other conditions
+ // (cloud mode, S3 filesystem, V1 inverted index, global config, etc.), and once
+ // set to true it remains stable even if config::enable_merge_file changes later.
+ mutable bool packed_file_active = false;
+
+ // Cached FileSystem instance to ensure consistency across multiple fs() calls.
+ // This prevents creating multiple MergeFileSystem instances and ensures
+ // packed_file_active flag remains consistent.
+ mutable io::FileSystemSPtr _cached_fs = nullptr;
+
// For collect segment statistics for compaction
std::vector<RowsetReaderSharedPtr> input_rs_readers;
@@ -142,7 +161,12 @@ struct RowsetWriterContext {
}
}
- io::FileSystemSPtr fs() {
+ io::FileSystemSPtr fs() const {
+ // Return cached instance if available to ensure consistency across multiple calls
+ if (_cached_fs != nullptr) {
+ return _cached_fs;
+ }
+
auto fs = [this]() -> io::FileSystemSPtr {
if (is_local_rowset()) {
return io::global_local_filesystem();
@@ -151,7 +175,11 @@ struct RowsetWriterContext {
}
}();
- if (!encrypt_algorithm.has_value()) {
+ bool is_s3_fs = fs->type() == io::FileSystemType::S3;
+
+ auto algorithm = encrypt_algorithm;
+
+ if (!algorithm.has_value()) {
#ifndef BE_TEST
constexpr std::string_view msg =
"RowsetWriterContext::determine_encryption is not called when creating this "
@@ -161,14 +189,48 @@ struct RowsetWriterContext {
LOG(WARNING) << st;
DCHECK(false) << st;
#else
- encrypt_algorithm = EncryptionAlgorithmPB::PLAINTEXT;
+ algorithm = EncryptionAlgorithmPB::PLAINTEXT;
#endif
- return fs;
}
- return io::make_file_system(fs, encrypt_algorithm.value());
+
+ // Apply encryption if needed
+ if (algorithm.has_value()) {
+ fs = io::make_file_system(fs, algorithm.value());
+ }
+
+ // Apply packed file system for write path if enabled
+ // Create empty index_map for write path
+ // Index information will be populated after write completes
+ bool has_v1_inverted_index = tablet_schema != nullptr &&
+ tablet_schema->has_inverted_index() &&
+ tablet_schema->get_inverted_index_storage_format() ==
+ InvertedIndexStorageFormatPB::V1;
+
+ if (has_v1_inverted_index && allow_packed_file && config::enable_packed_file) {
+ static constexpr std::string_view kMsg =
+ "Disable packed file for V1 inverted index tablet to avoid missing index "
+ "metadata (temporary workaround)";
+ LOG(INFO) << kMsg << ", tablet_id=" << tablet_id << ", rowset_id=" << rowset_id;
+ }
+
+ // Only enable merge file for S3 file system, not for HDFS or other remote file systems
+ packed_file_active = allow_packed_file && config::is_cloud_mode() &&
+ config::enable_packed_file && !has_v1_inverted_index && is_s3_fs;
+
+ if (packed_file_active) {
+ io::PackedAppendContext append_info;
+ append_info.tablet_id = tablet_id;
+ append_info.rowset_id = rowset_id.to_string();
+ append_info.txn_id = txn_id;
+ fs = std::make_shared<io::PackedFileSystem>(fs, append_info);
+ }
+
+ // Cache the result to ensure consistency across multiple calls
+ _cached_fs = fs;
+ return fs;
}
- io::FileSystem& fs_ref() { return *fs(); }
+ io::FileSystem& fs_ref() const { return *fs(); }
io::FileWriterOptions get_file_writer_options() {
io::FileWriterOptions opts {
diff --git a/be/src/olap/rowset/segment_creator.cpp b/be/src/olap/rowset/segment_creator.cpp
index 489f24728cc..b8a434d5aad 100644
--- a/be/src/olap/rowset/segment_creator.cpp
+++ b/be/src/olap/rowset/segment_creator.cpp
@@ -111,7 +111,9 @@ Status SegmentFlusher::_internal_parse_variant_columns(vectorized::Block& block)
}
Status SegmentFlusher::close() {
- return _seg_files.close();
+ RETURN_IF_ERROR(_seg_files.close());
+ RETURN_IF_ERROR(_idx_files.finish_close());
+ return Status::OK();
}
Status SegmentFlusher::_add_rows(std::unique_ptr<segment_v2::SegmentWriter>& segment_writer,
diff --git a/be/src/olap/rowset/segment_v2/index_file_writer.cpp b/be/src/olap/rowset/segment_v2/index_file_writer.cpp
index 19fdbc5412f..9d30f5b5126 100644
--- a/be/src/olap/rowset/segment_v2/index_file_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/index_file_writer.cpp
@@ -23,6 +23,7 @@
#include <filesystem>
#include "common/status.h"
+#include "io/fs/packed_file_writer.h"
#include "io/fs/s3_file_writer.h"
#include "io/fs/stream_sink_file_writer.h"
#include "olap/rowset/segment_v2/ann_index/ann_index_files.h"
@@ -128,6 +129,11 @@ Status IndexFileWriter::add_into_searcher_cache() {
// StreamSinkFileWriter not found file is normal.
return Status::OK();
}
+ if (dynamic_cast<io::PackedFileWriter*>(_idx_v2_writer.get()) != nullptr) {
+ // PackedFileWriter: file may be merged, skip cache for now.
+ // The cache will be populated on first read.
+ return Status::OK();
+ }
LOG(WARNING) << "IndexFileWriter::add_into_searcher_cache for " << _index_path_prefix
<< ", error " << st.msg();
return st;
@@ -179,14 +185,15 @@ Result<std::unique_ptr<IndexSearcherBuilder>> IndexFileWriter::_construct_index_
return IndexSearcherBuilder::create_index_searcher_builder(reader_type);
}
-Status IndexFileWriter::close() {
+Status IndexFileWriter::begin_close() {
DCHECK(!_closed) << debug_string();
_closed = true;
if (_indices_dirs.empty()) {
// An empty file must still be created even if there are no indexes to write
if (dynamic_cast<io::StreamSinkFileWriter*>(_idx_v2_writer.get()) != nullptr ||
- dynamic_cast<io::S3FileWriter*>(_idx_v2_writer.get()) != nullptr) {
- return _idx_v2_writer->close();
+ dynamic_cast<io::S3FileWriter*>(_idx_v2_writer.get()) != nullptr ||
+ dynamic_cast<io::PackedFileWriter*>(_idx_v2_writer.get()) != nullptr) {
+ return _idx_v2_writer->close(true);
}
return Status::OK();
}
@@ -218,7 +225,24 @@ Status IndexFileWriter::close() {
err.what());
}
}
- LOG_INFO("IndexFileWriter closing, enable_write_index_searcher_cache: {}",
+ return Status::OK();
+}
+
+Status IndexFileWriter::finish_close() {
+ DCHECK(_closed) << debug_string();
+ if (_indices_dirs.empty()) {
+ // An empty file must still be created even if there are no indexes to write
+ if (dynamic_cast<io::StreamSinkFileWriter*>(_idx_v2_writer.get()) != nullptr ||
+ dynamic_cast<io::S3FileWriter*>(_idx_v2_writer.get()) != nullptr ||
+ dynamic_cast<io::PackedFileWriter*>(_idx_v2_writer.get()) != nullptr) {
+ return _idx_v2_writer->close(false);
+ }
+ return Status::OK();
+ }
+ if (_idx_v2_writer != nullptr && _idx_v2_writer->state() != io::FileWriter::State::CLOSED) {
+ RETURN_IF_ERROR(_idx_v2_writer->close(false));
+ }
+ LOG_INFO("IndexFileWriter finish_close, enable_write_index_searcher_cache: {}",
config::enable_write_index_searcher_cache);
Status st = Status::OK();
if (config::enable_write_index_searcher_cache) {
diff --git a/be/src/olap/rowset/segment_v2/index_file_writer.h b/be/src/olap/rowset/segment_v2/index_file_writer.h
index 84dd5a41158..5adb8f76ac1 100644
--- a/be/src/olap/rowset/segment_v2/index_file_writer.h
+++ b/be/src/olap/rowset/segment_v2/index_file_writer.h
@@ -57,7 +57,13 @@ public:
Status delete_index(const TabletIndex* index_meta);
Status initialize(InvertedIndexDirectoryMap& indices_dirs);
Status add_into_searcher_cache();
- Status close();
+ // Begin the close process. This mainly triggers the asynchronous close operation of
+ // _idx_v2_writer by calling close(true), which starts the close process but returns
+ // immediately without waiting for completion.
+ Status begin_close();
+ // Finish the close process. This waits for the close operation to complete by calling
+ // _idx_v2_writer->close(false), which blocks until the close is fully done.
+ Status finish_close();
const InvertedIndexFileInfo* get_index_file_info() const {
DCHECK(_closed) << debug_string();
return &_file_info;
@@ -72,6 +78,9 @@ public:
std::vector<std::string> get_index_file_names() const;
std::string debug_string() const;
+ // Get internal file writer (for merge file index collection)
+ io::FileWriter* get_file_writer() const { return _idx_v2_writer.get(); }
+
private:
Status _insert_directory_into_map(int64_t index_id, const std::string& index_suffix,
std::shared_ptr<DorisFSDirectory> dir);
diff --git a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
index 33a0ad801a8..b67d1211806 100644
--- a/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
+++ b/be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
@@ -19,6 +19,7 @@
#include "CLucene/SharedHeader.h"
#include "CLucene/_SharedHeader.h"
+#include "cloud/config.h"
#include "common/status.h"
#include "inverted_index_common.h"
#include "inverted_index_desc.h"
@@ -432,7 +433,7 @@ void DorisFSDirectory::FSIndexOutputV2::close() {
_index_v2_file_writer = nullptr;
})
if (_index_v2_file_writer) {
- auto ret = _index_v2_file_writer->close();
+ auto ret = _index_v2_file_writer->close(true);
DBUG_EXECUTE_IF("DorisFSDirectory::FSIndexOutput._set_writer_close_status_error",
{ ret = Status::Error<INTERNAL_ERROR>("writer close status error"); })
if (!ret.ok()) {
@@ -924,23 +925,28 @@ DorisFSDirectory* DorisFSDirectoryFactory::getDirectory(const io::FileSystemSPtr
if (config::inverted_index_ram_dir_enable && can_use_ram_dir) {
dir = _CLNEW DorisRAMFSDirectory();
} else {
- bool exists = false;
- auto st = _fs->exists(file, &exists);
- DBUG_EXECUTE_IF("DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok", {
- st = Status::Error<ErrorCode::INTERNAL_ERROR>(
- "debug point: DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok");
- })
- LOG_AND_THROW_IF_ERROR(st, "Get directory exists IO error");
- if (!exists) {
- st = _fs->create_directory(file);
- DBUG_EXECUTE_IF(
- "DorisFSDirectoryFactory::getDirectory_create_directory_status_is_not_ok", {
- st = Status::Error<ErrorCode::INTERNAL_ERROR>(
- "debug point: "
- "DorisFSDirectoryFactory::getDirectory_create_directory_status_is_"
- "not_ok");
- })
- LOG_AND_THROW_IF_ERROR(st, "Get directory create directory IO error");
+ // cloud mode does not need to create directory
+ if (!config::is_cloud_mode()) {
+ bool exists = false;
+ auto st = _fs->exists(file, &exists);
+ DBUG_EXECUTE_IF("DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok", {
+ st = Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "debug point: "
+ "DorisFSDirectoryFactory::getDirectory_exists_status_is_not_ok");
+ })
+ LOG_AND_THROW_IF_ERROR(st, "Get directory exists IO error");
+ if (!exists) {
+ st = _fs->create_directory(file);
+ DBUG_EXECUTE_IF(
+ "DorisFSDirectoryFactory::getDirectory_create_directory_status_is_not_ok", {
+ st = Status::Error<ErrorCode::INTERNAL_ERROR>(
+ "debug point: "
+ "DorisFSDirectoryFactory::getDirectory_create_directory_status_"
+ "is_"
+ "not_ok");
+ })
+ LOG_AND_THROW_IF_ERROR(st, "Get directory create directory IO error");
+ }
}
dir = _CLNEW DorisFSDirectory();
}
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h
index 4a28048cd3c..f87fc910068 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/segment_writer.h
@@ -148,7 +148,7 @@ public:
*inverted_index_file_size = 0;
return Status::OK();
}
- RETURN_IF_ERROR(_index_file_writer->close());
+ RETURN_IF_ERROR(_index_file_writer->begin_close());
*inverted_index_file_size = _index_file_writer->get_index_file_total_size();
return Status::OK();
}
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
index 9528f12f5c7..51dfd217bfc 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h
@@ -125,7 +125,7 @@ public:
*inverted_index_file_size = 0;
return Status::OK();
}
- RETURN_IF_ERROR(_index_file_writer->close());
+ RETURN_IF_ERROR(_index_file_writer->begin_close());
*inverted_index_file_size = _index_file_writer->get_index_file_total_size();
return Status::OK();
}
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index 96552711266..32174107274 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -699,6 +699,7 @@ Result<RowsetSharedPtr> VBaseSchemaChangeWithSorting::_internal_sorting(
context.tablet_schema = new_tablet_schema;
context.newest_write_timestamp = newest_write_timestamp;
context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
+ context.allow_packed_file = false;
std::unique_ptr<RowsetWriter> rowset_writer;
// TODO(plat1ko): Use monad op
if (auto result = new_tablet->create_rowset_writer(context, false); !result.has_value())
@@ -727,6 +728,7 @@ Result<RowsetSharedPtr> VLocalSchemaChangeWithSorting::_internal_sorting(
context.tablet_schema = new_tablet_schema;
context.newest_write_timestamp = newest_write_timestamp;
context.write_type = DataWriteType::TYPE_SCHEMA_CHANGE;
+ context.allow_packed_file = false;
std::unique_ptr<RowsetWriter> rowset_writer;
// TODO(plat1ko): Use monad op
if (auto result = new_tablet->create_rowset_writer(context, false); !result.has_value())
@@ -1242,6 +1244,7 @@ Status SchemaChangeJob::_convert_historical_rowsets(const SchemaChangeParams& sc
context.segments_overlap = rs_reader->rowset()->rowset_meta()->segments_overlap();
context.tablet_schema = _new_tablet_schema;
context.newest_write_timestamp = rs_reader->newest_write_timestamp();
+ context.allow_packed_file = false;
if (!rs_reader->rowset()->is_local()) {
auto maybe_resource = rs_reader->rowset()->rowset_meta()->remote_storage_resource();
diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp
index 10a041fb8b0..a3ee9171fcf 100644
--- a/be/src/olap/task/index_builder.cpp
+++ b/be/src/olap/task/index_builder.cpp
@@ -377,13 +377,20 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
_index_file_writers.emplace(seg_ptr->id(), std::move(index_file_writer));
}
for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- auto st = index_file_writer->close();
+ auto st = index_file_writer->begin_close();
if (!st.ok()) {
LOG(ERROR) << "close index_file_writer error:" << st;
return st;
}
inverted_index_size += index_file_writer->get_index_file_total_size();
}
+ for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+ auto st = index_file_writer->finish_close();
+ if (!st.ok()) {
+ LOG(ERROR) << "wait close index_file_writer error:" << st;
+ return st;
+ }
+ }
_index_file_writers.clear();
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
output_rowset_meta->set_total_disk_size(output_rowset_meta->total_disk_size() +
@@ -619,7 +626,7 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
_olap_data_convertor->reset();
}
for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
- auto st = index_file_writer->close();
+ auto st = index_file_writer->begin_close();
DBUG_EXECUTE_IF("IndexBuilder::handle_single_rowset_file_writer_close_error", {
st = Status::Error<ErrorCode::INVERTED_INDEX_CLUCENE_ERROR>(
"debug point: handle_single_rowset_file_writer_close_error");
@@ -630,6 +637,13 @@ Status IndexBuilder::handle_single_rowset(RowsetMetaSharedPtr output_rowset_meta
}
inverted_index_size += index_file_writer->get_index_file_total_size();
}
+ for (auto&& [seg_id, index_file_writer] : _index_file_writers) {
+ auto st = index_file_writer->finish_close();
+ if (!st.ok()) {
+ LOG(ERROR) << "wait close index_file_writer error:" << st;
+ return st;
+ }
+ }
_index_column_writers.clear();
_index_file_writers.clear();
output_rowset_meta->set_data_disk_size(output_rowset_meta->data_disk_size());
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 48435bc7182..035f5f67d05 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -64,6 +64,7 @@ struct WriteCooldownMetaExecutors;
namespace io {
class FileCacheFactory;
class HdfsMgr;
+class PackedFileManager;
} // namespace io
namespace segment_v2 {
class InvertedIndexSearcherCache;
@@ -293,6 +294,7 @@ public:
kerberos::KerberosTicketMgr* kerberos_ticket_mgr() { return _kerberos_ticket_mgr; }
io::HdfsMgr* hdfs_mgr() { return _hdfs_mgr; }
+ io::PackedFileManager* packed_file_manager() { return _packed_file_manager; }
IndexPolicyMgr* index_policy_mgr() { return _index_policy_mgr; }
#ifdef BE_TEST
@@ -549,6 +551,7 @@ private:
kerberos::KerberosTicketMgr* _kerberos_ticket_mgr = nullptr;
io::HdfsMgr* _hdfs_mgr = nullptr;
+ io::PackedFileManager* _packed_file_manager = nullptr;
};
template <>
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 10cbcf92661..ec0c5609d11 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -132,6 +132,7 @@
// /doris/thirdparty/installed/include/hadoop_hdfs/hdfs.h:61:19: note: expanded from macro 'EINTERNAL'
// #define EINTERNAL 255
#include "io/fs/hdfs/hdfs_mgr.h"
+#include "io/fs/packed_file_manager.h"
// clang-format on
namespace doris {
@@ -408,6 +409,13 @@ Status ExecEnv::_init(const std::vector<StorePath>& store_paths,
_workload_sched_mgr = new WorkloadSchedPolicyMgr();
_workload_sched_mgr->start(this);
+ // Initialize packed file manager
+ _packed_file_manager = io::PackedFileManager::instance();
+ if (config::is_cloud_mode()) {
+ RETURN_IF_ERROR(_packed_file_manager->init());
+ _packed_file_manager->start_background_manager();
+ }
+
_index_policy_mgr = new IndexPolicyMgr();
RETURN_IF_ERROR(_spill_stream_mgr->init());
@@ -897,6 +905,11 @@ void ExecEnv::destroy() {
SAFE_DELETE(_dns_cache);
SAFE_DELETE(_kerberos_ticket_mgr);
SAFE_DELETE(_hdfs_mgr);
+ // PackedFileManager is a singleton, just stop its background thread
+ if (_packed_file_manager) {
+ _packed_file_manager->stop_background_manager();
+ _packed_file_manager = nullptr;
+ }
SAFE_DELETE(_process_profile);
SAFE_DELETE(_heap_profiler);
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index ae0614b3213..9f4b2e0f5ab 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -487,6 +487,7 @@ void HttpService::register_cloud_handler(CloudStorageEngine& engine) {
_ev_http_server->register_handler(HttpMethod::GET, "/api/delete_bitmap/count_agg_cache",
count_agg_cache_delete_bitmap_action);
#ifdef ENABLE_INJECTION_POINT
+
InjectionPointAction* injection_point_action = _pool.add(new InjectionPointAction);
_ev_http_server->register_handler(HttpMethod::GET, "/api/injection_point/{op}",
injection_point_action);
diff --git a/be/src/tools/CMakeLists.txt b/be/src/tools/CMakeLists.txt
index 93e18980bb5..ab1fc5fba19 100644
--- a/be/src/tools/CMakeLists.txt
+++ b/be/src/tools/CMakeLists.txt
@@ -53,3 +53,32 @@ add_custom_command(TARGET meta_tool POST_BUILD
COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:meta_tool>.dbg $<TARGET_FILE:meta_tool>
)
endif()
+
+add_executable(packed_file_tool
+ packed_file_tool.cpp
+)
+
+pch_reuse(packed_file_tool)
+
+set_target_properties(packed_file_tool PROPERTIES ENABLE_EXPORTS 1)
+
+if (COMPILER_CLANG)
+ target_compile_options(packed_file_tool PRIVATE
+ -Wno-implicit-int-conversion
+ -Wno-shorten-64-to-32
+ )
+endif()
+
+target_link_libraries(packed_file_tool
+ ${DORIS_LINK_LIBS}
+)
+
+install(TARGETS packed_file_tool DESTINATION ${OUTPUT_DIR}/lib/)
+
+if (NOT OS_MACOSX)
+add_custom_command(TARGET packed_file_tool POST_BUILD
+ COMMAND ${CMAKE_OBJCOPY} --only-keep-debug $<TARGET_FILE:packed_file_tool> $<TARGET_FILE:packed_file_tool>.dbg
+ COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded $<TARGET_FILE:packed_file_tool>
+ COMMAND ${CMAKE_OBJCOPY} --add-gnu-debuglink=$<TARGET_FILE:packed_file_tool>.dbg $<TARGET_FILE:packed_file_tool>
+ )
+endif()
diff --git a/be/src/tools/packed_file_tool.cpp b/be/src/tools/packed_file_tool.cpp
new file mode 100644
index 00000000000..0e5b8b06403
--- /dev/null
+++ b/be/src/tools/packed_file_tool.cpp
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gen_cpp/cloud.pb.h>
+#include <gflags/gflags.h>
+
+#include <iostream>
+#include <string>
+
+#include "common/status.h"
+#include "io/fs/packed_file_trailer.h"
+#include "json2pb/pb_to_json.h"
+
+DEFINE_string(file, "", "Path to a local merge/packed file");
+
+int main(int argc, char** argv) {
+ google::SetUsageMessage(
+ "Dump packed file trailer for debugging.\n"
+ "Usage: packed_file_tool --file=/path/to/merge_file");
+ google::ParseCommandLineFlags(&argc, &argv, true);
+
+ if (FLAGS_file.empty()) {
+ std::cerr << "Flag --file is required\n";
+ return -1;
+ }
+
+ doris::cloud::PackedFileFooterPB debug_info;
+ uint32_t version = 0;
+ doris::Status st = doris::io::read_packed_file_trailer(FLAGS_file, &debug_info, &version);
+ if (!st.ok()) {
+ std::cerr << "Failed to read packed file trailer: " << st.to_string() << std::endl;
+ return -1;
+ }
+
+ json2pb::Pb2JsonOptions options;
+ options.pretty_json = true;
+ std::string json;
+ json2pb::ProtoMessageToJson(debug_info, &json, options);
+
+ std::cout << "trailer_version: " << version << "\n" << json << std::endl;
+ return 0;
+}
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 1c4ca710926..e1468ed1bb5 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -204,6 +204,15 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::create(const S3ClientConf
return nullptr;
}
+#ifdef BE_TEST
+ {
+ std::lock_guard l(_lock);
+ if (_test_client_creator) {
+ return _test_client_creator(s3_conf);
+ }
+ }
+#endif
+
uint64_t hash = s3_conf.get_hash();
{
std::lock_guard l(_lock);
@@ -225,6 +234,19 @@ std::shared_ptr<io::ObjStorageClient> S3ClientFactory::create(const S3ClientConf
return obj_client;
}
+#ifdef BE_TEST
+void S3ClientFactory::set_client_creator_for_test(
+ std::function<std::shared_ptr<io::ObjStorageClient>(const S3ClientConf&)> creator) {
+ std::lock_guard l(_lock);
+ _test_client_creator = std::move(creator);
+}
+
+void S3ClientFactory::clear_client_creator_for_test() {
+ std::lock_guard l(_lock);
+ _test_client_creator = nullptr;
+}
+#endif
+
std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_azure_client(
const S3ClientConf& s3_conf) {
#ifdef USE_AZURE
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 75eeefa4d93..46282e525df 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -25,6 +25,7 @@
#include <gen_cpp/AgentService_types.h>
#include <gen_cpp/cloud.pb.h>
+#include <functional>
#include <map>
#include <memory>
#include <mutex>
@@ -154,6 +155,13 @@ public:
S3RateLimiterHolder* rate_limiter(S3RateLimitType type);
+#ifdef BE_TEST
+ void set_client_creator_for_test(
+ std::function<std::shared_ptr<io::ObjStorageClient>(const S3ClientConf&)> creator);
+
+ void clear_client_creator_for_test();
+#endif
+
private:
std::shared_ptr<io::ObjStorageClient> _create_s3_client(const S3ClientConf& s3_conf);
std::shared_ptr<io::ObjStorageClient> _create_azure_client(const S3ClientConf& s3_conf);
@@ -173,6 +181,9 @@ private:
std::unordered_map<uint64_t, std::shared_ptr<io::ObjStorageClient>> _cache;
std::string _ca_cert_file_path;
std::array<std::unique_ptr<S3RateLimiterHolder>, 2> _rate_limiters;
+#ifdef BE_TEST
+ std::function<std::shared_ptr<io::ObjStorageClient>(const S3ClientConf&)> _test_client_creator;
+#endif
};
} // end namespace doris
diff --git a/be/test/io/fs/packed_file_concurrency_test.cpp b/be/test/io/fs/packed_file_concurrency_test.cpp
new file mode 100644
index 00000000000..06db472b95e
--- /dev/null
+++ b/be/test/io/fs/packed_file_concurrency_test.cpp
@@ -0,0 +1,753 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <aws/core/client/AWSError.h>
+#include <aws/core/http/HttpResponse.h>
+#include <aws/core/utils/Outcome.h>
+#include <aws/s3/S3Client.h>
+#include <aws/s3/S3Errors.h>
+#include <aws/s3/model/CompleteMultipartUploadRequest.h>
+#include <aws/s3/model/CompleteMultipartUploadResult.h>
+#include <aws/s3/model/CompletedPart.h>
+#include <aws/s3/model/CreateMultipartUploadRequest.h>
+#include <aws/s3/model/CreateMultipartUploadResult.h>
+#include <aws/s3/model/HeadObjectRequest.h>
+#include <aws/s3/model/HeadObjectResult.h>
+#include <aws/s3/model/PutObjectRequest.h>
+#include <aws/s3/model/PutObjectResult.h>
+#include <aws/s3/model/UploadPartRequest.h>
+#include <aws/s3/model/UploadPartResult.h>
+#include <gtest/gtest.h>
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <filesystem>
+#include <future>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <random>
+#include <string>
+#include <string_view>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "common/status.h"
+#include "cpp/sync_point.h"
+#include "fmt/format.h"
+#include "glog/logging.h"
+#include "io/cache/block_file_cache_factory.h"
+#include "io/cache/cached_remote_file_reader.h"
+#include "io/cache/file_cache_common.h"
+#include "io/cache/fs_file_cache_storage.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/packed_file_manager.h"
+#include "io/fs/packed_file_reader.h"
+#include "io/fs/packed_file_system.h"
+#include "io/fs/packed_file_writer.h"
+#include "io/fs/path.h"
+#include "io/fs/s3_file_system.h"
+#include "io/io_common.h"
+#include "runtime/exec_env.h"
+#include "runtime/thread_context.h"
+#include "util/s3_util.h"
+#include "util/slice.h"
+#include "util/threadpool.h"
+
+namespace doris::io {
+
+using PackedFileManager = PackedFileManager;
+using PackedFileSystem = PackedFileSystem;
+using PackedFileWriter = PackedFileWriter;
+using PackedFileReader = PackedFileReader;
+using PackedSliceLocation = PackedSliceLocation;
+using PackedAppendContext = PackedAppendContext;
+
+namespace {
+
+constexpr int64_t kMinSegmentBytes = 10 * 1024;
+constexpr int64_t kMaxSegmentBytes = 100 * 1024;
+constexpr int kThreadCount = 10;
+constexpr int kIterationPerThread = 100;
+constexpr int64_t kLargeThreshold = 512 * 1024;
+
+std::unique_ptr<FileCacheFactory> g_owned_cache_factory;
+bool g_owned_factory_in_use = false;
+bool g_cache_initialized = false;
+std::string g_cache_base_path;
+bool g_thread_pool_initialized = false;
+bool g_s3_upload_pool_initialized = false;
+std::shared_ptr<FileSystem> g_remote_file_system;
+bool g_owned_fd_cache = false;
+
+FileCacheFactory* ensure_file_cache_factory_on_exec_env() {
+ auto* exec_env = ExecEnv::GetInstance();
+ auto* factory = exec_env->file_cache_factory();
+ if (factory == nullptr) {
+ if (!g_owned_cache_factory) {
+ g_owned_cache_factory = std::make_unique<FileCacheFactory>();
+ }
+ factory = g_owned_cache_factory.get();
+ exec_env->set_file_cache_factory(factory);
+ g_owned_factory_in_use = true;
+ }
+ if (exec_env->file_cache_open_fd_cache() == nullptr) {
+ exec_env->set_file_cache_open_fd_cache(std::make_unique<FDCache>());
+ g_owned_fd_cache = true;
+ }
+ return factory;
+}
+
+void release_owned_file_cache_factory() {
+ if (g_owned_factory_in_use) {
+ ExecEnv::GetInstance()->set_file_cache_factory(nullptr);
+ g_owned_cache_factory.reset();
+ g_owned_factory_in_use = false;
+ }
+}
+
+void ensure_non_block_close_pool() {
+ auto* exec_env = ExecEnv::GetInstance();
+ if (exec_env->non_block_close_thread_pool() == nullptr) {
+ std::unique_ptr<ThreadPool> pool;
+ auto st = ThreadPoolBuilder("NonBlockCloseThreadPool")
+ .set_min_threads(12)
+ .set_max_threads(48)
+ .build(&pool);
+ ASSERT_TRUE(st.ok()) << st;
+ exec_env->set_non_block_close_thread_pool(std::move(pool));
+ g_thread_pool_initialized = true;
+ }
+}
+
+void release_non_block_close_pool() {
+ if (g_thread_pool_initialized) {
+ ExecEnv::GetInstance()->set_non_block_close_thread_pool(nullptr);
+ g_thread_pool_initialized = false;
+ }
+}
+
+void ensure_s3_upload_pool() {
+ auto* exec_env = ExecEnv::GetInstance();
+ if (exec_env->s3_file_upload_thread_pool() == nullptr) {
+ std::unique_ptr<ThreadPool> pool;
+ auto st = ThreadPoolBuilder("s3_upload_file_thread_pool")
+ .set_min_threads(5)
+ .set_max_threads(10)
+ .build(&pool);
+ ASSERT_TRUE(st.ok()) << st;
+ exec_env->set_s3_file_upload_thread_pool(std::move(pool));
+ g_s3_upload_pool_initialized = true;
+ }
+}
+
+void release_s3_upload_pool() {
+ if (g_s3_upload_pool_initialized) {
+ ExecEnv::GetInstance()->set_s3_file_upload_thread_pool(nullptr);
+ g_s3_upload_pool_initialized = false;
+ }
+}
+
+struct MockS3Store {
+ struct UploadCtx {
+ std::string bucket;
+ std::string key;
+ std::map<int, std::string> parts;
+ };
+
+ std::mutex mutex;
+ uint64_t next_upload_id {1};
+ std::unordered_map<std::string, UploadCtx> uploads;
+ std::unordered_map<std::string, std::string> objects;
+
+ std::string make_key(std::string_view bucket, std::string_view key) const {
+ return fmt::format("{}/{}", bucket, key);
+ }
+};
+
+MockS3Store& mock_s3_store() {
+ static MockS3Store store;
+ return store;
+}
+
+void reset_mock_s3_store() {
+ auto& store = mock_s3_store();
+ std::lock_guard lock(store.mutex);
+ store.next_upload_id = 1;
+ store.uploads.clear();
+ store.objects.clear();
+}
+
+class MockObjStorageClient : public ObjStorageClient {
+public:
+ explicit MockObjStorageClient(MockS3Store* store) : _store(store) {}
+
+ ObjectStorageUploadResponse create_multipart_upload(
+ const ObjectStoragePathOptions& opts) override {
+ ObjectStorageUploadResponse resp;
+ resp.resp = ObjectStorageResponse::OK();
+ std::lock_guard lock(_store->mutex);
+ auto upload_id = fmt::format("upload-{}", _store->next_upload_id++);
+ resp.upload_id = upload_id;
+ _store->uploads[upload_id] = MockS3Store::UploadCtx {opts.bucket, opts.key, {}};
+ return resp;
+ }
+
+ ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts,
+ std::string_view stream) override {
+ std::lock_guard lock(_store->mutex);
+ _store->objects[_store->make_key(opts.bucket, opts.key)] =
+ std::string(stream.data(), stream.size());
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts,
+ std::string_view stream, int part_num) override {
+ ObjectStorageUploadResponse resp;
+ if (!opts.upload_id) {
+ resp.resp = make_error("missing upload id");
+ return resp;
+ }
+ std::lock_guard lock(_store->mutex);
+ auto ctx_it = _store->uploads.find(*opts.upload_id);
+ if (ctx_it == _store->uploads.end()) {
+ resp.resp = make_error("upload context not found");
+ return resp;
+ }
+ ctx_it->second.parts[part_num] = std::string(stream.data(), stream.size());
+ resp.resp = ObjectStorageResponse::OK();
+ resp.etag = fmt::format("\"mock-etag-{}\"", part_num);
+ return resp;
+ }
+
+ ObjectStorageResponse complete_multipart_upload(
+ const ObjectStoragePathOptions& opts,
+ const std::vector<ObjectCompleteMultiPart>& completed_parts) override {
+ if (!opts.upload_id) {
+ return make_error("missing upload id");
+ }
+ std::lock_guard lock(_store->mutex);
+ auto ctx_it = _store->uploads.find(*opts.upload_id);
+ if (ctx_it == _store->uploads.end()) {
+ return make_error("upload context not found");
+ }
+ std::string data;
+ for (const auto& part : completed_parts) {
+ auto part_it = ctx_it->second.parts.find(part.part_num);
+ if (part_it == ctx_it->second.parts.end()) {
+ return make_error("missing upload part");
+ }
+ data.append(part_it->second);
+ }
+ _store->objects[_store->make_key(ctx_it->second.bucket, ctx_it->second.key)] =
+ std::move(data);
+ _store->uploads.erase(ctx_it);
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override {
+ ObjectStorageHeadResponse resp;
+ std::lock_guard lock(_store->mutex);
+ auto key = _store->make_key(opts.bucket, opts.key);
+ auto it = _store->objects.find(key);
+ if (it == _store->objects.end()) {
+ resp.resp = make_error("object not found",
+ static_cast<int>(Aws::Http::HttpResponseCode::NOT_FOUND));
+ return resp;
+ }
+ resp.resp = ObjectStorageResponse::OK();
+ resp.file_size = it->second.size();
+ return resp;
+ }
+
+ ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer,
+ size_t offset, size_t bytes_read,
+ size_t* size_return) override {
+ std::lock_guard lock(_store->mutex);
+ auto key = _store->make_key(opts.bucket, opts.key);
+ auto it = _store->objects.find(key);
+ if (it == _store->objects.end()) {
+ return make_error("object not found");
+ }
+ if (offset > it->second.size()) {
+ return make_error("offset exceeds object size");
+ }
+ size_t to_copy = std::min(bytes_read, it->second.size() - offset);
+ memcpy(buffer, it->second.data() + offset, to_copy);
+ *size_return = to_copy;
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts,
+ std::vector<FileInfo>* files) override {
+ std::lock_guard lock(_store->mutex);
+ std::string prefix = _store->make_key(opts.bucket, opts.prefix);
+ for (const auto& [key, data] : _store->objects) {
+ if (key.rfind(prefix, 0) == 0) {
+ FileInfo info;
+ info.file_name = key.substr(prefix.size());
+ info.file_size = data.size();
+ info.is_file = true;
+ files->push_back(std::move(info));
+ }
+ }
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts,
+ std::vector<std::string> objs) override {
+ std::lock_guard lock(_store->mutex);
+ for (const auto& obj : objs) {
+ _store->objects.erase(_store->make_key(opts.bucket, obj));
+ }
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override {
+ std::lock_guard lock(_store->mutex);
+ _store->objects.erase(_store->make_key(opts.bucket, opts.key));
+ return ObjectStorageResponse::OK();
+ }
+
+ ObjectStorageResponse delete_objects_recursively(
+ const ObjectStoragePathOptions& opts) override {
+ std::lock_guard lock(_store->mutex);
+ std::string prefix = _store->make_key(opts.bucket, opts.prefix);
+ for (auto it = _store->objects.begin(); it != _store->objects.end();) {
+ if (it->first.rfind(prefix, 0) == 0) {
+ it = _store->objects.erase(it);
+ } else {
+ ++it;
+ }
+ }
+ return ObjectStorageResponse::OK();
+ }
+
+ std::string generate_presigned_url(const ObjectStoragePathOptions& opts,
+ int64_t /*expiration_secs*/,
+ const S3ClientConf& /*conf*/) override {
+ return fmt::format("mock://{}/{}", opts.bucket, opts.key);
+ }
+
+private:
+ static ObjectStorageResponse make_error(std::string msg, int http_code = 500) {
+ ObjectStorageResponse resp;
+ resp.status.code = static_cast<int>(ErrorCode::INTERNAL_ERROR);
+ resp.status.msg = std::move(msg);
+ resp.http_code = http_code;
+ return resp;
+ }
+
+ MockS3Store* _store;
+};
+
+std::shared_ptr<MockObjStorageClient> g_mock_obj_client;
+
+void install_mock_environment() {
+ g_mock_obj_client = std::make_shared<MockObjStorageClient>(&mock_s3_store());
+ auto client = g_mock_obj_client;
+ S3ClientFactory::instance().set_client_creator_for_test(
+ [client](const S3ClientConf&) { return client; });
+
+ auto* sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ sp->set_call_back("PackedFileManager::update_meta_service", [](std::vector<std::any>&& args) {
+ auto pair = try_any_cast_ret<Status>(args);
+ pair->first = Status::OK();
+ pair->second = true;
+ });
+}
+
+void remove_mock_environment() {
+ S3ClientFactory::instance().clear_client_creator_for_test();
+ g_mock_obj_client.reset();
+
+ auto* sp = SyncPoint::get_instance();
+ sp->clear_call_back("PackedFileManager::update_meta_service");
+ sp->disable_processing();
+}
+
+class MockRemoteReader : public FileReader {
+public:
+ MockRemoteReader(Path path, std::string bucket, MockS3Store* store)
+ : _path(std::move(path)), _bucket(std::move(bucket)), _store(store) {}
+
+ Status close() override {
+ _closed = true;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ size_t size() const override {
+ std::lock_guard lock(_store->mutex);
+ auto it = _store->objects.find(_store->make_key(_bucket, _path.native()));
+ return it == _store->objects.end() ? 0 : it->second.size();
+ }
+
+ bool closed() const override { return _closed; }
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) override {
+ std::lock_guard lock(_store->mutex);
+ auto key = _store->make_key(_bucket, _path.native());
+ auto it = _store->objects.find(key);
+ if (it == _store->objects.end()) {
+ return Status::InternalError("mock object {} not found", key);
+ }
+ const auto& data = it->second;
+ if (offset > data.size()) {
+ return Status::InternalError("offset {} exceeds size {} for {}", offset, data.size(),
+ key);
+ }
+ size_t to_copy = std::min(result.size, data.size() - offset);
+ memcpy(result.data, data.data() + offset, to_copy);
+ *bytes_read = to_copy;
+ return Status::OK();
+ }
+
+private:
+ Path _path;
+ std::string _bucket;
+ MockS3Store* _store;
+ bool _closed = false;
+};
+
+class MockRemoteFileSystem : public FileSystem {
+public:
+ MockRemoteFileSystem(MockS3Store* store, std::string bucket, std::string prefix = {})
+ : FileSystem("mock_remote_fs", FileSystemType::S3),
+ _store(store),
+ _bucket(std::move(bucket)),
+ _prefix(std::move(prefix)) {}
+
+protected:
+ Status open_file_impl(const Path& file, FileReaderSPtr* reader,
+ const FileReaderOptions* opts) override {
+ auto normalized_path = _normalize_remote_path(file);
+ auto raw = std::make_shared<MockRemoteReader>(Path(normalized_path), _bucket, _store);
+ FileReaderOptions local_opts = opts ? *opts : FileReaderOptions();
+ local_opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE;
+ local_opts.is_doris_table = true;
+ *reader = std::make_shared<CachedRemoteFileReader>(raw, local_opts);
+ return Status::OK();
+ }
+
+ Status create_file_impl(const Path& /*file*/, FileWriterPtr* /*writer*/,
+ const FileWriterOptions* /*opts*/) override {
+ return Status::NotSupported("mock remote fs does not support create");
+ }
+
+ Status create_directory_impl(const Path& /*dir*/, bool /*failed_if_exists*/) override {
+ return Status::NotSupported("mock remote fs does not support directories");
+ }
+
+ Status delete_file_impl(const Path& /*file*/) override {
+ return Status::NotSupported("mock remote fs does not support delete");
+ }
+
+ Status delete_directory_impl(const Path& /*dir*/) override {
+ return Status::NotSupported("mock remote fs does not support delete dir");
+ }
+
+ Status batch_delete_impl(const std::vector<Path>& /*files*/) override {
+ return Status::NotSupported("mock remote fs does not support batch delete");
+ }
+
+ Status exists_impl(const Path& file, bool* res) const override {
+ std::lock_guard lock(_store->mutex);
+ *res = _store->objects.count(_store->make_key(_bucket, _normalize_remote_path(file))) > 0;
+ return Status::OK();
+ }
+
+ Status file_size_impl(const Path& file, int64_t* file_size) const override {
+ std::lock_guard lock(_store->mutex);
+ auto it = _store->objects.find(_store->make_key(_bucket, _normalize_remote_path(file)));
+ if (it == _store->objects.end()) {
+ return Status::InternalError("mock object {} not found", file.native());
+ }
+ *file_size = it->second.size();
+ return Status::OK();
+ }
+
+ Status rename_impl(const Path& /*orig_name*/, const Path& /*new_name*/) override {
+ return Status::NotSupported("mock remote fs does not support rename");
+ }
+
+ Status list_impl(const Path& /*dir*/, bool /*only_file*/, std::vector<FileInfo>* /*files*/,
+ bool* /*exists*/) override {
+ return Status::NotSupported("mock remote fs does not support list");
+ }
+
+ Status absolute_path(const Path& file, Path& abs_path) const override {
+ abs_path = file;
+ return Status::OK();
+ }
+
+private:
+ std::string _normalize_remote_path(const Path& file) const {
+ constexpr std::string_view kS3Scheme = "s3://";
+ std::string_view view(file.native());
+ if (view.rfind(kS3Scheme, 0) == 0) {
+ view.remove_prefix(kS3Scheme.size());
+ auto slash_pos = view.find('/');
+ if (slash_pos == std::string_view::npos) {
+ return {};
+ }
+ auto uri_bucket = view.substr(0, slash_pos);
+ view.remove_prefix(std::min(view.size(), slash_pos + 1));
+ DCHECK(uri_bucket.empty() || uri_bucket == _bucket)
+ << "unexpected bucket " << uri_bucket << " for path " << file.native();
+ }
+ std::string normalized(view);
+ if (!_prefix.empty() && normalized.rfind(_prefix, 0) != 0) {
+ normalized = fmt::format("{}/{}", _prefix, normalized);
+ }
+ return normalized;
+ }
+
+ MockS3Store* _store;
+ std::string _bucket;
+ std::string _prefix;
+};
+
+void reset_manager_state(PackedFileManager* manager) {
+ manager->stop_background_manager();
+ manager->clear_state_for_test();
+ manager->file_systems_for_test().clear();
+ manager->default_file_system_for_test().reset();
+ manager->reset_packed_file_bvars_for_test();
+}
+
+void create_file_cache_once(const std::string& base_path) {
+ static std::once_flag once;
+ std::call_once(once, [&] {
+ ensure_file_cache_factory_on_exec_env();
+ std::filesystem::create_directories(base_path);
+ auto settings = get_file_cache_settings(64 * 1024 * 1024, 8 * 1024 * 1024);
+ auto status = FileCacheFactory::instance()->create_file_cache(base_path, settings);
+ ASSERT_TRUE(status.ok()) << status;
+ g_cache_initialized = true;
+ g_cache_base_path = base_path;
+ });
+}
+
+void cleanup_file_cache_resources() {
+ if (g_cache_initialized) {
+ auto* factory = ExecEnv::GetInstance()->file_cache_factory();
+ if (factory != nullptr) {
+ factory->_caches.clear();
+ factory->_path_to_cache.clear();
+ factory->_capacity = 0;
+ }
+ g_cache_initialized = false;
+ g_cache_base_path.clear();
+ }
+ g_remote_file_system.reset();
+ if (g_owned_fd_cache) {
+ ExecEnv::GetInstance()->set_file_cache_open_fd_cache(nullptr);
+ g_owned_fd_cache = false;
+ }
+ release_owned_file_cache_factory();
+}
+
+} // namespace
+
+class MergeFileConcurrencyTest : public ::testing::Test {
+protected:
+ static void SetUpTestSuite() {
+ install_mock_environment();
+ reset_mock_s3_store();
+ ensure_non_block_close_pool();
+ ensure_s3_upload_pool();
+ S3Conf s3_conf;
+ s3_conf.bucket = "mock-bucket";
+ s3_conf.prefix = "merge-file-concurrency";
+ s3_conf.client_conf.ak = "ak";
+ s3_conf.client_conf.sk = "sk";
+ s3_conf.client_conf.endpoint = "http://endpoint";
+ s3_conf.client_conf.region = "region";
+ auto fs_or = S3FileSystem::create(s3_conf, FileSystem::TMP_FS_ID);
+ ASSERT_TRUE(fs_or.has_value()) << fs_or.error();
+ _s3_fs = fs_or.value();
+ g_remote_file_system = std::make_shared<MockRemoteFileSystem>(
+ &mock_s3_store(), _s3_fs->bucket(), _s3_fs->prefix());
+
+ std::string cache_path =
+ (std::filesystem::current_path() / "ut_dir/packed_file_cache").string();
+ create_file_cache_once(cache_path);
+ }
+
+ static void TearDownTestSuite() {
+ remove_mock_environment();
+ cleanup_file_cache_resources();
+ release_non_block_close_pool();
+ release_s3_upload_pool();
+ _s3_fs.reset();
+ }
+
+ void SetUp() override {
+ reset_mock_s3_store();
+ _old_small_threshold = config::small_file_threshold_bytes;
+ _old_merge_threshold = config::packed_file_size_threshold_bytes;
+ _old_cleanup_seconds = config::uploaded_file_retention_seconds;
+ _old_time_threshold = config::packed_file_time_threshold_ms;
+ _old_enable_file_cache = config::enable_file_cache;
+
+ config::small_file_threshold_bytes = kLargeThreshold;
+ config::packed_file_size_threshold_bytes = kLargeThreshold * 2;
+ config::uploaded_file_retention_seconds = 10;
+ config::packed_file_time_threshold_ms = 1;
+ config::enable_file_cache = true;
+
+ reset_manager_state(PackedFileManager::instance());
+ PackedFileManager::instance()->start_background_manager();
+ }
+
+ void TearDown() override {
+ config::small_file_threshold_bytes = _old_small_threshold;
+ config::packed_file_size_threshold_bytes = _old_merge_threshold;
+ config::uploaded_file_retention_seconds = _old_cleanup_seconds;
+ config::packed_file_time_threshold_ms = _old_time_threshold;
+ config::enable_file_cache = _old_enable_file_cache;
+ reset_manager_state(PackedFileManager::instance());
+ }
+
+ static std::shared_ptr<S3FileSystem> _s3_fs;
+
+private:
+ int64_t _old_small_threshold = 0;
+ int64_t _old_merge_threshold = 0;
+ int64_t _old_cleanup_seconds = 0;
+ int64_t _old_time_threshold = 0;
+ bool _old_enable_file_cache = false;
+};
+
+std::shared_ptr<S3FileSystem> MergeFileConcurrencyTest::_s3_fs;
+
+TEST_F(MergeFileConcurrencyTest, ConcurrentWriteReadCorrectness) {
+ auto* manager = PackedFileManager::instance();
+ ASSERT_NE(_s3_fs, nullptr);
+ ASSERT_NE(g_remote_file_system, nullptr);
+
+ std::vector<std::string> resource_ids;
+ resource_ids.reserve(kThreadCount);
+ for (int i = 0; i < kThreadCount; ++i) {
+ auto id = fmt::format("resource-{}", i);
+ resource_ids.emplace_back(id);
+ manager->file_systems_for_test()[id] = _s3_fs;
+ }
+
+ std::atomic<int> success_count {0};
+ std::vector<std::thread> workers;
+ workers.reserve(kThreadCount);
+
+ for (int tid = 0; tid < kThreadCount; ++tid) {
+ workers.emplace_back([&, tid] {
+ SCOPED_INIT_THREAD_CONTEXT();
+ std::mt19937_64 rng(0x9e3779b97f4a7c15ULL + tid);
+ std::uniform_int_distribution<int> seg_cnt_dist(1, 4);
+ std::uniform_int_distribution<int> seg_size_dist(static_cast<int>(kMinSegmentBytes),
+ static_cast<int>(kMaxSegmentBytes));
+ std::uniform_int_distribution<int> read_size_dist(4 * 1024, 32 * 1024);
+
+ for (int iter = 0; iter < kIterationPerThread; ++iter) {
+ std::string path = fmt::format("/tablet_{}/rowset_{}/file_{}", tid, iter, iter);
+
+ PackedAppendContext append_info;
+ append_info.resource_id = resource_ids[tid];
+ append_info.tablet_id = 1000 + tid;
+ append_info.rowset_id = fmt::format("rowset-{}-{}", tid, iter);
+ append_info.txn_id = (tid + 1) * 100000 + iter + 1;
+
+ PackedFileSystem writer_fs(_s3_fs, append_info);
+ FileWriterPtr writer;
+ ASSERT_TRUE(writer_fs.create_file(Path(path), &writer, nullptr).ok());
+ auto* merge_writer = dynamic_cast<PackedFileWriter*>(writer.get());
+ ASSERT_NE(merge_writer, nullptr);
+
+ const int seg_count = seg_cnt_dist(rng);
+ std::vector<std::string> payloads;
+ payloads.reserve(seg_count);
+ std::string expected;
+ for (int seg = 0; seg < seg_count; ++seg) {
+ int len = seg_size_dist(rng);
+ std::string chunk(len, static_cast<char>('a' + (seg + tid) % 26));
+ payloads.emplace_back(std::move(chunk));
+ expected.append(payloads.back());
+ Slice slice(payloads.back());
+ ASSERT_TRUE(writer->append(slice).ok());
+ }
+
+ ASSERT_TRUE(writer->close(true).ok());
+ ASSERT_TRUE(writer->close(false).ok());
+
+ PackedSliceLocation index;
+ ASSERT_TRUE(merge_writer->get_packed_slice_location(&index).ok());
+
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ index_map[path] = index;
+ PackedFileSystem reader_fs(g_remote_file_system, index_map, append_info);
+ FileReaderSPtr reader;
+ FileReaderOptions opts;
+ opts.cache_type = FileCachePolicy::FILE_BLOCK_CACHE;
+ opts.is_doris_table = true;
+ ASSERT_TRUE(reader_fs.open_file(Path(path), &reader, &opts).ok());
+ auto* merge_reader = dynamic_cast<PackedFileReader*>(reader.get());
+ ASSERT_NE(merge_reader, nullptr);
+ auto* cached_reader =
+ dynamic_cast<CachedRemoteFileReader*>(merge_reader->_inner_reader.get());
+ ASSERT_NE(cached_reader, nullptr);
+
+ IOContext io_ctx;
+ size_t verified = 0;
+ while (verified < expected.size()) {
+ size_t remain = expected.size() - verified;
+ size_t chunk = std::min<size_t>(remain, read_size_dist(rng));
+ std::string buf(chunk, '\0');
+ Slice read_slice(buf.data(), chunk);
+ size_t bytes_read = 0;
+ ASSERT_TRUE(reader->read_at(verified, read_slice, &bytes_read, &io_ctx).ok());
+ ASSERT_EQ(bytes_read, chunk);
+ ASSERT_EQ(0, std::memcmp(buf.data(), expected.data() + verified, chunk));
+ verified += chunk;
+ }
+
+ success_count.fetch_add(1, std::memory_order_relaxed);
+ }
+ });
+ }
+
+ for (auto& th : workers) {
+ th.join();
+ }
+
+ EXPECT_EQ(success_count.load(), kThreadCount * kIterationPerThread);
+}
+
+} // namespace doris::io
diff --git a/be/test/io/fs/packed_file_manager_test.cpp b/be/test/io/fs/packed_file_manager_test.cpp
new file mode 100644
index 00000000000..645f0896f1a
--- /dev/null
+++ b/be/test/io/fs/packed_file_manager_test.cpp
@@ -0,0 +1,1129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_manager.h"
+
+#include <gtest/gtest.h>
+
+#include <any>
+#include <chrono>
+#include <cstring>
+#include <ctime>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <string>
+#include <thread>
+#include <utility>
+#include <vector>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "common/status.h"
+#include "cpp/sync_point.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/packed_file_trailer.h"
+#include "io/fs/path.h"
+#include "util/coding.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+using doris::Status;
+class MockFileWriter : public FileWriter {
+public:
+ explicit MockFileWriter(std::string path) : _path(std::move(path)) {}
+
+ void set_append_status(Status st) { _append_status = std::move(st); }
+ void set_close_status(Status st) { _close_status = std::move(st); }
+ void set_start_close_status(Status st) { _start_close_status = std::move(st); }
+ void complete_async_close() {
+ if (_state == State::ASYNC_CLOSING) {
+ _state = State::CLOSED;
+ }
+ }
+
+ size_t append_calls() const { return _append_calls; }
+ bool closed() const { return _state == State::CLOSED; }
+ size_t bytes_appended() const override { return _bytes_appended; }
+ const std::string& written_data() const { return _written; }
+
+ Status close(bool non_block = false) override {
+ if (_state == State::CLOSED) {
+ if (non_block) {
+ if (_close_status.ok()) {
+ return Status::Error<ErrorCode::ALREADY_CLOSED>(
+ "MockFileWriter already closed: {}", _path.native());
+ }
+ return _close_status;
+ }
+ return Status::Error<ErrorCode::ALREADY_CLOSED>("MockFileWriter already closed: {}",
+ _path.native());
+ }
+
+ if (!_start_close_status.ok()) {
+ return _start_close_status;
+ }
+
+ if (_state == State::ASYNC_CLOSING) {
+ return Status::InternalError("Don't submit async close multi times");
+ }
+
+ if (non_block) {
+ _state = State::ASYNC_CLOSING;
+ return Status::OK();
+ }
+
+ _state = State::CLOSED;
+ return _close_status;
+ }
+
+ Status appendv(const Slice* data, size_t data_cnt) override {
+ if (!_append_status.ok()) {
+ return _append_status;
+ }
+ for (size_t i = 0; i < data_cnt; ++i) {
+ _written.append(reinterpret_cast<const char*>(data[i].data), data[i].size);
+ _bytes_appended += data[i].size;
+ }
+ ++_append_calls;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ State state() const override { return _state; }
+
+private:
+ Path _path;
+ size_t _bytes_appended = 0;
+ size_t _append_calls = 0;
+ std::string _written;
+ Status _start_close_status = Status::OK();
+ Status _append_status = Status::OK();
+ Status _close_status = Status::OK();
+ State _state = State::OPENED;
+};
+
+class MockFileSystem : public FileSystem {
+public:
+ MockFileSystem() : FileSystem("mock_fs", FileSystemType::LOCAL) {}
+
+ void set_create_status(Status st) { _create_status = std::move(st); }
+ void set_writer_append_status(Status st) { _writer_append_status = std::move(st); }
+ void set_writer_close_status(Status st) { _writer_close_status = std::move(st); }
+
+ MockFileWriter* last_writer() const { return _last_writer; }
+ const std::vector<std::string>& created_paths() const { return _created_paths; }
+
+protected:
+ Status create_file_impl(const Path& file, FileWriterPtr* writer,
+ const FileWriterOptions* /*opts*/) override {
+ if (!_create_status.ok()) {
+ return _create_status;
+ }
+ auto mock_writer = std::make_unique<MockFileWriter>(file.native());
+ mock_writer->set_append_status(_writer_append_status);
+ mock_writer->set_close_status(_writer_close_status);
+ _last_writer = mock_writer.get();
+ *writer = std::move(mock_writer);
+ _created_paths.push_back(file.native());
+ return Status::OK();
+ }
+
+ Status open_file_impl(const Path& /*file*/, FileReaderSPtr* /*reader*/,
+ const FileReaderOptions* /*opts*/) override {
+ return Status::NotSupported("not implemented");
+ }
+
+ Status create_directory_impl(const Path& /*dir*/, bool /*failed_if_exists*/) override {
+ return Status::OK();
+ }
+
+ Status delete_file_impl(const Path& /*file*/) override { return Status::OK(); }
+
+ Status batch_delete_impl(const std::vector<Path>& /*files*/) override { return Status::OK(); }
+
+ Status delete_directory_impl(const Path& /*dir*/) override { return Status::OK(); }
+
+ Status exists_impl(const Path& /*path*/, bool* res) const override {
+ *res = false;
+ return Status::OK();
+ }
+
+ Status file_size_impl(const Path& /*file*/, int64_t* size) const override {
+ *size = 0;
+ return Status::OK();
+ }
+
+ Status list_impl(const Path& /*dir*/, bool /*only_file*/, std::vector<FileInfo>* /*files*/,
+ bool* exists) override {
+ *exists = true;
+ return Status::OK();
+ }
+
+ Status rename_impl(const Path& /*orig_name*/, const Path& /*new_name*/) override {
+ return Status::OK();
+ }
+
+ Status absolute_path(const Path& path, Path& abs_path) const override {
+ abs_path = path;
+ return Status::OK();
+ }
+
+private:
+ Status _create_status = Status::OK();
+ Status _writer_append_status = Status::OK();
+ Status _writer_close_status = Status::OK();
+ MockFileWriter* _last_writer = nullptr;
+ std::vector<std::string> _created_paths;
+};
+
+class PackedFileManagerTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _old_merge_threshold = config::packed_file_size_threshold_bytes;
+ _old_small_threshold = config::small_file_threshold_bytes;
+ _old_retention = config::uploaded_file_retention_seconds;
+ _old_time_threshold = config::packed_file_time_threshold_ms;
+ _old_small_file_count_threshold = config::packed_file_small_file_count_threshold;
+ _old_deploy_mode = config::deploy_mode;
+ _old_cloud_id = config::cloud_unique_id;
+
+ config::packed_file_size_threshold_bytes = 1024;
+ config::small_file_threshold_bytes = 1024;
+ config::uploaded_file_retention_seconds = 60;
+ config::packed_file_time_threshold_ms = 100; // Default 100ms
+ config::packed_file_small_file_count_threshold = 100;
+ config::deploy_mode.clear();
+ config::cloud_unique_id.clear();
+
+ file_system = std::make_shared<MockFileSystem>();
+ manager = std::make_unique<PackedFileManager>();
+ manager->_file_systems[_resource_id] = file_system;
+ ASSERT_TRUE(manager->init().ok());
+ auto& state = manager->current_packed_files_for_test()[_resource_id];
+ ASSERT_TRUE(manager->create_new_packed_file_state_for_test(_resource_id, state).ok());
+ ASSERT_NE(file_system->last_writer(), nullptr);
+ manager->reset_packed_file_bvars_for_test();
+ }
+
+ void TearDown() override {
+ manager.reset();
+ file_system.reset();
+
+ config::packed_file_size_threshold_bytes = _old_merge_threshold;
+ config::small_file_threshold_bytes = _old_small_threshold;
+ config::uploaded_file_retention_seconds = _old_retention;
+ config::packed_file_time_threshold_ms = _old_time_threshold;
+ config::packed_file_small_file_count_threshold = _old_small_file_count_threshold;
+ config::deploy_mode = _old_deploy_mode;
+ config::cloud_unique_id = _old_cloud_id;
+ }
+
+ PackedAppendContext default_append_info() const {
+ PackedAppendContext info;
+ info.resource_id = _resource_id;
+ info.tablet_id = _tablet_id;
+ info.rowset_id = _rowset_id;
+ info.txn_id = _txn_id;
+ return info;
+ }
+
+ std::shared_ptr<MockFileSystem> file_system;
+ std::unique_ptr<PackedFileManager> manager;
+
+private:
+ int64_t _old_merge_threshold = 0;
+ int64_t _old_small_threshold = 0;
+ int64_t _old_retention = 0;
+ int64_t _old_time_threshold = 0;
+ int64_t _old_small_file_count_threshold = 0;
+ std::string _old_deploy_mode;
+ std::string _old_cloud_id;
+ std::string _resource_id = "test_resource";
+ int64_t _tablet_id = 12345;
+ std::string _rowset_id = "rowset_1";
+ int64_t _txn_id = 6789;
+};
+
+TEST_F(PackedFileManagerTest, CreateNewMergeFileStateFailure) {
+ auto failing_fs = std::make_shared<MockFileSystem>();
+ failing_fs->set_create_status(Status::IOError("create failed"));
+ PackedFileManager local_manager;
+ local_manager._file_systems[_resource_id] = failing_fs;
+ auto& state = local_manager.current_packed_files_for_test()[_resource_id];
+ EXPECT_FALSE(local_manager.create_new_packed_file_state_for_test(_resource_id, state).ok());
+}
+
+TEST_F(PackedFileManagerTest, AppendSmallFileSuccess) {
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+ std::string payload = "abc";
+ Slice slice(payload);
+
+ auto info = default_append_info();
+ Status st = manager->append_small_file("s/path", slice, info);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(writer->append_calls(), 1);
+ EXPECT_EQ(writer->bytes_appended(), payload.size());
+
+ auto it = manager->global_slice_locations_for_test().find("s/path");
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, payload.size());
+ EXPECT_EQ(it->second.offset, 0);
+ EXPECT_EQ(it->second.tablet_id, info.tablet_id);
+ EXPECT_EQ(it->second.rowset_id, info.rowset_id);
+ EXPECT_EQ(it->second.resource_id, info.resource_id);
+ EXPECT_EQ(it->second.txn_id, info.txn_id);
+}
+
+TEST_F(PackedFileManagerTest, AppendFailsWithoutTxnId) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ info.txn_id = 0;
+
+ Status st = manager->append_small_file("missing_txn", slice, info);
+ EXPECT_EQ(st.code(), doris::ErrorCode::INVALID_ARGUMENT);
+ EXPECT_EQ(manager->global_slice_locations_for_test().count("missing_txn"), 0);
+}
+
+TEST_F(PackedFileManagerTest, AppendLargeFileSkipped) {
+ config::small_file_threshold_bytes = 4;
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+
+ std::string payload = "0123456789";
+ Slice slice(payload);
+ auto info = default_append_info();
+ Status st = manager->append_small_file("large", slice, info);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(writer->append_calls(), 0);
+ EXPECT_EQ(manager->global_slice_locations_for_test().count("large"), 0);
+}
+
+TEST_F(PackedFileManagerTest, AppendWriterFailure) {
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+ writer->set_append_status(Status::IOError("append fail"));
+
+ std::string payload = "data";
+ Slice slice(payload);
+ auto info = default_append_info();
+ Status st = manager->append_small_file("broken", slice, info);
+ EXPECT_FALSE(st.ok());
+ EXPECT_EQ(manager->global_slice_locations_for_test().count("broken"), 0);
+}
+
+TEST_F(PackedFileManagerTest, AppendTriggersRotationWhenThresholdReached) {
+ config::packed_file_size_threshold_bytes = 8;
+ std::string payload1 = "aaaa";
+ std::string payload2 = "bbbb";
+ Slice slice1(payload1);
+ Slice slice2(payload2);
+ auto info = default_append_info();
+
+ EXPECT_TRUE(manager->append_small_file("file1", slice1, info).ok());
+
+ size_t create_before = file_system->created_paths().size();
+ EXPECT_TRUE(manager->append_small_file("file2", slice2, info).ok());
+ EXPECT_EQ(file_system->created_paths().size(), create_before + 1);
+ EXPECT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+}
+
+TEST_F(PackedFileManagerTest, AppendTriggersRotationWhenFileCountThresholdReached) {
+ config::packed_file_small_file_count_threshold = 2;
+ config::packed_file_size_threshold_bytes = 1024; // keep size threshold out of the way
+
+ std::string payload = "data";
+ Slice slice(payload);
+ auto info = default_append_info();
+
+ EXPECT_TRUE(manager->append_small_file("file1", slice, info).ok());
+ EXPECT_TRUE(manager->uploading_packed_files_for_test().empty());
+
+ size_t created_before = file_system->created_paths().size();
+ EXPECT_TRUE(manager->append_small_file("file2", slice, info).ok());
+ EXPECT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+ EXPECT_EQ(file_system->created_paths().size(), created_before + 1);
+
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ ASSERT_NE(uploading, nullptr);
+ EXPECT_EQ(uploading->slice_locations.size(), 2);
+
+ auto* new_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(new_state, nullptr);
+ EXPECT_NE(uploading->packed_file_path, new_state->packed_file_path);
+}
+
+TEST_F(PackedFileManagerTest, MarkCurrentMergeFileForUploadMovesState) {
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ auto current_path = current_state->packed_file_path;
+ EXPECT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ EXPECT_EQ(uploading->packed_file_path, current_path);
+ EXPECT_EQ(uploading->state.load(), PackedFileManager::PackedFileState::READY_TO_UPLOAD);
+ auto* new_state = manager->current_packed_files_for_test()[_resource_id].get();
+ EXPECT_NE(new_state, nullptr);
+ EXPECT_NE(new_state->packed_file_path, current_path);
+}
+
+TEST_F(PackedFileManagerTest, GetMergeFileIndexForUnknownPathReturnsNotFound) {
+ PackedSliceLocation index;
+ auto status = manager->get_packed_slice_location("missing", &index);
+ EXPECT_EQ(status.code(), doris::ErrorCode::NOT_FOUND);
+}
+
+TEST_F(PackedFileManagerTest, GetMergeFileIndexReturnsStoredValue) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ EXPECT_TRUE(manager->append_small_file("stored", slice, info).ok());
+ PackedSliceLocation index;
+ EXPECT_TRUE(manager->get_packed_slice_location("stored", &index).ok());
+ EXPECT_EQ(index.size, payload.size());
+ EXPECT_EQ(index.offset, 0);
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ EXPECT_EQ(index.packed_file_path, current_state->packed_file_path);
+}
+
+TEST_F(PackedFileManagerTest, WaitWriteDoneReturnsOkWhenAlreadyUploaded) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("path", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ {
+ std::lock_guard lock(uploading->upload_mutex);
+ uploading->state = PackedFileManager::PackedFileState::UPLOADED;
+ uploading->upload_time = std::time(nullptr);
+ }
+ manager->uploaded_packed_files_for_test()[uploading->packed_file_path] = uploading;
+ manager->uploading_packed_files_for_test().clear();
+ uploading->upload_cv.notify_all();
+
+ EXPECT_TRUE(manager->wait_upload_done("path").ok());
+}
+
+TEST_F(PackedFileManagerTest, WaitWriteDoneReturnsErrorWhenFailed) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("bad", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ {
+ std::lock_guard lock(uploading->upload_mutex);
+ uploading->state = PackedFileManager::PackedFileState::FAILED;
+ uploading->last_error = "meta failed";
+ uploading->upload_time = std::time(nullptr);
+ }
+ manager->uploaded_packed_files_for_test()[uploading->packed_file_path] = uploading;
+ manager->uploading_packed_files_for_test().clear();
+ uploading->upload_cv.notify_all();
+
+ auto status = manager->wait_upload_done("bad");
+ EXPECT_FALSE(status.ok());
+ EXPECT_NE(status.to_string().find("meta failed"), std::string::npos);
+}
+
+TEST_F(PackedFileManagerTest, WaitWriteDoneReturnsErrorWhenPathMissing) {
+ auto indices = manager->global_slice_locations_for_test().find("ghost");
+ ASSERT_EQ(indices, manager->global_slice_locations_for_test().end());
+ auto status = manager->wait_upload_done("ghost");
+ EXPECT_FALSE(status.ok());
+}
+
+TEST_F(PackedFileManagerTest, WaitWriteDoneBlocksUntilUploadCompletes) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("waiting", slice, info).ok());
+
+ auto current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ {
+ std::lock_guard lock(current_state->upload_mutex);
+ current_state->state = PackedFileManager::PackedFileState::UPLOADING;
+ }
+
+ std::promise<void> ready;
+ std::future<void> ready_future = ready.get_future();
+ std::thread waiter([&] {
+ ready.set_value();
+ auto status = manager->wait_upload_done("waiting");
+ EXPECT_TRUE(status.ok());
+ });
+
+ ready_future.wait();
+ std::this_thread::sleep_for(std::chrono::milliseconds(20));
+ {
+ std::lock_guard lock(current_state->upload_mutex);
+ current_state->state = PackedFileManager::PackedFileState::UPLOADED;
+ }
+ current_state->upload_cv.notify_all();
+
+ waiter.join();
+}
+
+TEST_F(PackedFileManagerTest, UploadMergeFileReturnsErrorWhenWriterNull) {
+ auto status = manager->finalize_packed_file_upload("path", nullptr);
+ EXPECT_FALSE(status.ok());
+}
+
+TEST_F(PackedFileManagerTest, UploadMergeFilePropagatesWriterCloseFailure) {
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+ writer->set_start_close_status(Status::IOError("close fail"));
+ auto status = manager->finalize_packed_file_upload(writer->path().native(), writer);
+ EXPECT_FALSE(status.ok());
+}
+
+TEST_F(PackedFileManagerTest, UploadMergeFileSucceedsWhenWriterCloses) {
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+ auto status = manager->finalize_packed_file_upload(writer->path().native(), writer);
+ EXPECT_TRUE(status.ok());
+ EXPECT_EQ(writer->state(), FileWriter::State::ASYNC_CLOSING);
+}
+
+TEST_F(PackedFileManagerTest, ProcessUploadingFilesSetsFailedWhenMetaUpdateFails) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("meta_fail", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+
+ manager->process_uploading_packed_files();
+
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 0);
+ ASSERT_EQ(manager->uploaded_packed_files_for_test().size(), 1);
+ auto uploaded = manager->uploaded_packed_files_for_test().begin()->second;
+ EXPECT_EQ(uploaded->state.load(), PackedFileManager::PackedFileState::FAILED);
+ EXPECT_FALSE(uploaded->last_error.empty());
+}
+
+TEST_F(PackedFileManagerTest, ProcessUploadingFilesCompletesAsyncUpload) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("async_success", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ auto* writer = dynamic_cast<MockFileWriter*>(uploading->writer.get());
+ ASSERT_NE(writer, nullptr);
+ uploading->state = PackedFileManager::PackedFileState::UPLOADING;
+ ASSERT_TRUE(writer->close(true).ok());
+
+ manager->process_uploading_packed_files();
+ EXPECT_EQ(uploading->state.load(), PackedFileManager::PackedFileState::UPLOADING);
+ EXPECT_EQ(manager->uploaded_packed_files_for_test().size(), 0);
+
+ writer->complete_async_close();
+ manager->process_uploading_packed_files();
+ EXPECT_EQ(manager->uploading_packed_files_for_test().size(), 0);
+ ASSERT_EQ(manager->uploaded_packed_files_for_test().size(), 1);
+ auto uploaded = manager->uploaded_packed_files_for_test().begin()->second;
+ EXPECT_EQ(uploaded->state.load(), PackedFileManager::PackedFileState::UPLOADED);
+}
+
+TEST_F(PackedFileManagerTest, ProcessUploadingFilesSetsFailedWhenAsyncCloseFails) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("async_fail", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ auto* writer = dynamic_cast<MockFileWriter*>(uploading->writer.get());
+ ASSERT_NE(writer, nullptr);
+ uploading->state = PackedFileManager::PackedFileState::UPLOADING;
+ writer->set_close_status(Status::IOError("async close fail"));
+ ASSERT_TRUE(writer->close(true).ok());
+
+ manager->process_uploading_packed_files();
+ EXPECT_EQ(uploading->state.load(), PackedFileManager::PackedFileState::UPLOADING);
+
+ writer->complete_async_close();
+ manager->process_uploading_packed_files();
+ EXPECT_EQ(manager->uploading_packed_files_for_test().size(), 0);
+ ASSERT_EQ(manager->uploaded_packed_files_for_test().size(), 1);
+ auto failed = manager->uploaded_packed_files_for_test().begin()->second;
+ EXPECT_EQ(failed->state.load(), PackedFileManager::PackedFileState::FAILED);
+ EXPECT_NE(failed->last_error.find("async close fail"), std::string::npos);
+}
+
+TEST_F(PackedFileManagerTest, AppendPackedFileInfoToFileTail) {
+ std::string payload = "abc";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("trailer_path", slice, info).ok());
+ ASSERT_TRUE(manager->mark_current_packed_file_for_upload(_resource_id).ok());
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+
+ auto* sp = SyncPoint::get_instance();
+ sp->enable_processing();
+ sp->set_call_back("PackedFileManager::update_meta_service", [](std::vector<std::any>&& args) {
+ auto pair = try_any_cast_ret<Status>(args);
+ pair->first = Status::OK();
+ pair->second = true;
+ });
+
+ manager->process_uploading_packed_files();
+
+ sp->clear_call_back("PackedFileManager::update_meta_service");
+ sp->disable_processing();
+
+ ASSERT_EQ(manager->uploading_packed_files_for_test().size(), 1);
+ auto uploading = manager->uploading_packed_files_for_test().begin()->second;
+ auto* writer = dynamic_cast<MockFileWriter*>(uploading->writer.get());
+ ASSERT_NE(writer, nullptr);
+
+ const auto& data = writer->written_data();
+ cloud::PackedFileFooterPB parsed_footer;
+ uint32_t version = 0;
+ auto st = parse_packed_file_trailer(data, &parsed_footer, &version);
+ ASSERT_TRUE(st.ok()) << st;
+ ASSERT_EQ(version, kPackedFileTrailerVersion);
+ ASSERT_TRUE(parsed_footer.has_packed_file_info());
+
+ const auto& parsed_info = parsed_footer.packed_file_info();
+ ASSERT_EQ(parsed_info.slices_size(), 1);
+ EXPECT_EQ(parsed_info.slices(0).path(), "trailer_path");
+ EXPECT_EQ(parsed_info.slices(0).offset(), 0);
+ EXPECT_EQ(parsed_info.slices(0).size(), payload.size());
+ EXPECT_EQ(parsed_info.resource_id(), info.resource_id);
+}
+
+TEST_F(PackedFileManagerTest, CleanupExpiredDataRemovesOldEntries) {
+ config::uploaded_file_retention_seconds = 0;
+ auto state = std::make_shared<PackedFileManager::PackedFileContext>();
+ state->packed_file_path = "old";
+ state->resource_id = _resource_id;
+ state->state = PackedFileManager::PackedFileState::UPLOADED;
+ state->upload_time = 0;
+ manager->uploaded_packed_files_for_test()[state->packed_file_path] = state;
+
+ manager->cleanup_expired_data();
+
+ EXPECT_TRUE(manager->uploaded_packed_files_for_test().empty());
+}
+
+TEST_F(PackedFileManagerTest, MergeFileBvarMetricsUpdated) {
+ manager->reset_packed_file_bvars_for_test();
+ auto state = std::make_shared<PackedFileManager::PackedFileContext>();
+ state->total_size = 300;
+ PackedSliceLocation idx1;
+ idx1.size = 100;
+ PackedSliceLocation idx2;
+ idx2.size = 200;
+ state->slice_locations["a"] = idx1;
+ state->slice_locations["b"] = idx2;
+
+ manager->record_packed_file_metrics_for_test(state.get());
+
+ EXPECT_EQ(manager->packed_file_total_count_for_test(), 1);
+ EXPECT_EQ(manager->packed_file_total_small_file_num_for_test(), 2);
+ EXPECT_EQ(manager->packed_file_total_size_bytes_for_test(), 300);
+ EXPECT_DOUBLE_EQ(manager->packed_file_avg_small_file_num_for_test(), 2.0);
+ EXPECT_DOUBLE_EQ(manager->packed_file_avg_file_size_for_test(), 300.0);
+
+ manager->reset_packed_file_bvars_for_test();
+ EXPECT_EQ(manager->packed_file_total_count_for_test(), 0);
+ EXPECT_EQ(manager->packed_file_total_small_file_num_for_test(), 0);
+ EXPECT_EQ(manager->packed_file_total_size_bytes_for_test(), 0);
+ EXPECT_DOUBLE_EQ(manager->packed_file_avg_small_file_num_for_test(), 0.0);
+ EXPECT_DOUBLE_EQ(manager->packed_file_avg_file_size_for_test(), 0.0);
+}
+
+// Multiple small file imports, segment size < threshold, triggers merge file
+TEST_F(PackedFileManagerTest, MultipleSmallFilesTriggerMergeFile) {
+ // Set small file threshold to 100 bytes
+ config::small_file_threshold_bytes = 100;
+ // Set merge file size threshold to 500 bytes (will trigger rotation when reached)
+ config::packed_file_size_threshold_bytes = 500;
+
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+
+ auto info = default_append_info();
+ constexpr int file_count = 10;
+ constexpr int file_size = 50; // Each file is 50 bytes, smaller than threshold 100
+
+ // Append multiple small files
+ for (int i = 0; i < file_count; ++i) {
+ std::string path = "small_file_" + std::to_string(i);
+ std::string payload(file_size, 'a' + (i % 26)); // 50 bytes per file
+ Slice slice(payload);
+
+ Status st = manager->append_small_file(path, slice, info);
+ EXPECT_TRUE(st.ok()) << "Failed to append file " << i << ": " << st.msg();
+
+ // Verify file is in global index map
+ auto it = manager->global_slice_locations_for_test().find(path);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, file_size);
+ EXPECT_EQ(it->second.tablet_id, info.tablet_id);
+ EXPECT_EQ(it->second.rowset_id, info.rowset_id);
+ }
+
+ // Verify all files are in global index map
+ // Note: When total_size + new_file_size >= threshold, a new merge file is created
+ // So files may be distributed across multiple merge files
+ int total_files_in_index = 0;
+ int64_t total_size_in_index = 0;
+ for (int i = 0; i < file_count; ++i) {
+ std::string path = "small_file_" + std::to_string(i);
+ auto it = manager->global_slice_locations_for_test().find(path);
+ if (it != manager->global_slice_locations_for_test().end()) {
+ total_files_in_index++;
+ total_size_in_index += it->second.size;
+ EXPECT_LT(it->second.size, config::small_file_threshold_bytes)
+ << "File " << path << " size " << it->second.size
+ << " should be less than threshold " << config::small_file_threshold_bytes;
+ }
+ }
+ EXPECT_EQ(total_files_in_index, file_count);
+ EXPECT_EQ(total_size_in_index, file_count * file_size);
+
+ // Count files across all merge files (current and uploading)
+ int total_files_in_packed_files = 0;
+ int64_t total_size_in_packed_files = 0;
+
+ // Check current merge file
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ total_files_in_packed_files += current_state->slice_locations.size();
+ total_size_in_packed_files += current_state->total_size;
+
+ // Check uploading merge files
+ for (const auto& [path, state] : manager->uploading_packed_files_for_test()) {
+ total_files_in_packed_files += state->slice_locations.size();
+ total_size_in_packed_files += state->total_size;
+ }
+
+ EXPECT_EQ(total_files_in_packed_files, file_count);
+ EXPECT_EQ(total_size_in_packed_files, file_count * file_size);
+
+ // When total size reaches threshold, merge file should be rotated
+ // Add one more file to trigger rotation (if not already triggered)
+ std::string path_trigger = "trigger_file";
+ std::string payload_trigger(file_size, 'z');
+ Slice slice_trigger(payload_trigger);
+
+ size_t create_before = file_system->created_paths().size();
+ Status st = manager->append_small_file(path_trigger, slice_trigger, info);
+ EXPECT_TRUE(st.ok());
+
+ // Should create new merge file when threshold is reached
+ EXPECT_GE(file_system->created_paths().size(), create_before);
+ // May have uploading merge files if threshold was reached
+ EXPECT_GE(manager->uploading_packed_files_for_test().size(), 0);
+
+ // New merge file should have the trigger file
+ auto* new_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(new_state, nullptr);
+ // Trigger file should be in current state or uploading state
+ auto trigger_it = manager->global_slice_locations_for_test().find(path_trigger);
+ EXPECT_NE(trigger_it, manager->global_slice_locations_for_test().end());
+}
+
+// Multiple files close to threshold, boundary value verification for merge file
+TEST_F(PackedFileManagerTest, FilesNearThresholdBoundaryMergeFile) {
+ // Set small file threshold to 100 bytes
+ config::small_file_threshold_bytes = 100;
+ // Set merge file size threshold to 500 bytes
+ config::packed_file_size_threshold_bytes = 500;
+
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+
+ auto info = default_append_info();
+
+ // Files exactly at threshold - 1 (should be merged)
+ {
+ std::string path1 = "boundary_file_99";
+ std::string payload1(99, 'x'); // 99 bytes, just below threshold
+ Slice slice1(payload1);
+
+ Status st = manager->append_small_file(path1, slice1, info);
+ EXPECT_TRUE(st.ok());
+
+ auto it = manager->global_slice_locations_for_test().find(path1);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, 99);
+ EXPECT_LT(it->second.size, config::small_file_threshold_bytes);
+ }
+
+ // Files exactly at threshold (should be merged, uses > not >=, so 100 <= 100 means merged)
+ {
+ std::string path2 = "boundary_file_100";
+ std::string payload2(100, 'y'); // 100 bytes, exactly at threshold
+ Slice slice2(payload2);
+
+ Status st = manager->append_small_file(path2, slice2, info);
+ EXPECT_TRUE(st.ok());
+
+ // File at threshold should be merged (check uses >, so 100 is not > 100, so it's merged)
+ auto it = manager->global_slice_locations_for_test().find(path2);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, 100);
+ EXPECT_LE(it->second.size, config::small_file_threshold_bytes);
+ }
+
+ // Files at threshold + 1 (should NOT be merged)
+ {
+ std::string path3 = "boundary_file_101";
+ std::string payload3(101, 'z'); // 101 bytes, above threshold
+ Slice slice3(payload3);
+
+ size_t append_calls_before = writer->append_calls();
+ Status st = manager->append_small_file(path3, slice3, info);
+ EXPECT_TRUE(st.ok());
+
+ // File above threshold should not be merged
+ auto it = manager->global_slice_locations_for_test().find(path3);
+ EXPECT_EQ(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(writer->append_calls(), append_calls_before);
+ }
+
+ // Multiple files close to threshold, verify merge file behavior
+ // Add files that are just below threshold to fill up merge file
+ constexpr int file_count = 5;
+ constexpr int file_size = 99; // Just below threshold
+
+ for (int i = 0; i < file_count; ++i) {
+ std::string path = "near_threshold_file_" + std::to_string(i);
+ std::string payload(file_size, 'a' + (i % 26));
+ Slice slice(payload);
+
+ Status st = manager->append_small_file(path, slice, info);
+ EXPECT_TRUE(st.ok()) << "Failed to append file " << i << ": " << st.msg();
+
+ // Verify file is merged
+ auto it = manager->global_slice_locations_for_test().find(path);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, file_size);
+ EXPECT_LT(it->second.size, config::small_file_threshold_bytes);
+ }
+
+ // Verify merge file state
+ // Note: When total_size + new_file_size >= threshold, a new merge file is created
+ // So files may be distributed across multiple merge files
+ int total_packed_files = 0;
+ int64_t total_merged_size = 0;
+
+ // Count files in current merge file
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ total_packed_files += current_state->slice_locations.size();
+ total_merged_size += current_state->total_size;
+
+ // Count files in uploading merge files
+ for (const auto& [path, state] : manager->uploading_packed_files_for_test()) {
+ total_packed_files += state->slice_locations.size();
+ total_merged_size += state->total_size;
+ }
+
+ // Should have 1 file from first test (99 bytes) + 1 file at threshold (100 bytes) + 5 files from loop (99 bytes each) = 7 files
+ EXPECT_EQ(total_packed_files, 7);
+ EXPECT_EQ(total_merged_size, 99 + 100 + (file_count * file_size));
+
+ // Verify all merged files are at or below threshold (uses >, so == threshold is merged)
+ for (const auto& [path, index] : current_state->slice_locations) {
+ EXPECT_LE(index.size, config::small_file_threshold_bytes)
+ << "File " << path << " size " << index.size
+ << " should be less than or equal to threshold "
+ << config::small_file_threshold_bytes;
+ }
+ for (const auto& [path, state] : manager->uploading_packed_files_for_test()) {
+ for (const auto& [file_path, index] : state->slice_locations) {
+ EXPECT_LE(index.size, config::small_file_threshold_bytes)
+ << "File " << file_path << " size " << index.size
+ << " should be less than or equal to threshold "
+ << config::small_file_threshold_bytes;
+ }
+ }
+
+ // Note: writer->bytes_appended() may not reflect all merged data because
+ // each merge file has its own writer, and we're only checking the last writer
+ // So we don't verify writer->bytes_appended() here
+}
+
+// Merge file result check, generated path and filename meet expectations
+TEST_F(PackedFileManagerTest, MergeFilePathAndFilenameFormat) {
+ // Verify initial merge file path format
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ const std::string& packed_file_path = current_state->packed_file_path;
+
+ // Path should start with "data/packed_file/"
+ EXPECT_TRUE(packed_file_path.find("data/packed_file/") == 0)
+ << "Merge file path should start with 'data/packed_file/', got: " << packed_file_path;
+}
+
+// Test timeout triggers upload, causing small files to be uploaded directly
+TEST_F(PackedFileManagerTest, TimeoutTriggersDirectUpload) {
+ // Set a very large merge file size threshold so size won't trigger upload
+ config::packed_file_size_threshold_bytes = 10 * 1024 * 1024; // 10MB
+ // Set a short timeout threshold - 50ms to trigger timeout quickly
+ config::packed_file_time_threshold_ms = 50;
+
+ // Reset manager to use new config
+ manager.reset();
+ file_system.reset();
+ file_system = std::make_shared<MockFileSystem>();
+ manager = std::make_unique<PackedFileManager>();
+ manager->_file_systems[_resource_id] = file_system;
+ ASSERT_TRUE(manager->init().ok());
+ auto& state = manager->current_packed_files_for_test()[_resource_id];
+ ASSERT_TRUE(manager->create_new_packed_file_state_for_test(_resource_id, state).ok());
+ ASSERT_NE(file_system->last_writer(), nullptr);
+
+ // Add a small file that won't trigger size-based upload
+ std::string payload = "small_file_data";
+ Slice slice(payload);
+ auto info = default_append_info();
+ ASSERT_TRUE(manager->append_small_file("small_file_1", slice, info).ok());
+
+ // Get the merge file path before timeout
+ PackedSliceLocation index_before;
+ ASSERT_TRUE(manager->get_packed_slice_location("small_file_1", &index_before).ok());
+ std::string packed_file_path_before = index_before.packed_file_path;
+
+ // Verify file is in current merge file (not uploaded yet)
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ EXPECT_EQ(current_state->packed_file_path, packed_file_path_before);
+ EXPECT_EQ(current_state->state.load(), PackedFileManager::PackedFileState::ACTIVE);
+
+ // Start background manager thread
+ manager->start_background_manager();
+
+ // Wait for timeout to be triggered and processed
+ // Background thread checks every check_interval_ms (half of timeout threshold = 25ms)
+ // We need to wait at least timeout threshold (50ms) + some margin for processing
+ std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+ // Verify that the merge file was marked for upload due to timeout
+ // The original merge file should be in uploading or uploaded state, or new file created
+ bool found_in_uploading = false;
+ bool found_in_uploaded = false;
+ std::string new_packed_file_path;
+ {
+ auto* new_current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ if (new_current_state && new_current_state->packed_file_path != packed_file_path_before) {
+ new_packed_file_path = new_current_state->packed_file_path;
+ }
+ }
+
+ {
+ std::lock_guard<std::mutex> lock(manager->_packed_files_mutex);
+ if (manager->uploading_packed_files_for_test().find(packed_file_path_before) !=
+ manager->uploading_packed_files_for_test().end()) {
+ found_in_uploading = true;
+ }
+ if (manager->uploaded_packed_files_for_test().find(packed_file_path_before) !=
+ manager->uploaded_packed_files_for_test().end()) {
+ found_in_uploaded = true;
+ }
+ }
+
+ // The merge file should have been moved to uploading/uploaded state or new file created
+ EXPECT_TRUE(found_in_uploading || found_in_uploaded || !new_packed_file_path.empty())
+ << "Merge file should be marked for upload due to timeout";
+
+ // Verify the small file index still points to the original merge file
+ PackedSliceLocation index_after;
+ ASSERT_TRUE(manager->get_packed_slice_location("small_file_1", &index_after).ok());
+ EXPECT_EQ(index_after.packed_file_path, packed_file_path_before);
+ EXPECT_EQ(index_after.size, payload.size());
+
+ // Process uploading files to complete the upload
+ manager->process_uploading_packed_files();
+
+ // Stop background manager
+ manager->stop_background_manager();
+
+ // Verify the file is eventually in uploaded or uploading state
+ {
+ std::lock_guard<std::mutex> lock(manager->_packed_files_mutex);
+ bool is_uploaded =
+ (manager->uploaded_packed_files_for_test().find(packed_file_path_before) !=
+ manager->uploaded_packed_files_for_test().end());
+ bool is_uploading =
+ (manager->uploading_packed_files_for_test().find(packed_file_path_before) !=
+ manager->uploading_packed_files_for_test().end());
+ EXPECT_TRUE(is_uploaded || is_uploading)
+ << "Merge file should be in uploading or uploaded state after timeout";
+ }
+}
+
+// Continuous import of small files while modifying threshold, merge file should still trigger
+TEST_F(PackedFileManagerTest, ModifyThresholdDuringContinuousImport) {
+ // Set initial threshold to 100 bytes
+ config::small_file_threshold_bytes = 100;
+ config::packed_file_size_threshold_bytes = 500;
+
+ auto writer = file_system->last_writer();
+ ASSERT_NE(writer, nullptr);
+
+ auto info = default_append_info();
+ constexpr int file_size = 50; // 50 bytes per file, should be merged
+
+ // Phase 1: Import some files with initial threshold (100 bytes)
+ constexpr int phase1_file_count = 5;
+ for (int i = 0; i < phase1_file_count; ++i) {
+ std::string path = "file_phase1_" + std::to_string(i);
+ std::string payload(file_size, 'a' + (i % 26));
+ Slice slice(payload);
+
+ Status st = manager->append_small_file(path, slice, info);
+ EXPECT_TRUE(st.ok()) << "Failed to append file " << i << ": " << st.msg();
+
+ // Verify file is in global index map
+ auto it = manager->global_slice_locations_for_test().find(path);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, file_size);
+ EXPECT_LT(it->second.size, config::small_file_threshold_bytes);
+ }
+
+ // Phase 2: Modify threshold to 200 bytes (larger than before)
+ config::small_file_threshold_bytes = 200;
+
+ // Phase 3: Continue importing files with new threshold
+ // Files should still be merged because 50 < 200
+ constexpr int phase2_file_count = 5;
+ for (int i = 0; i < phase2_file_count; ++i) {
+ std::string path = "file_phase2_" + std::to_string(i);
+ std::string payload(file_size, 'x' + (i % 26));
+ Slice slice(payload);
+
+ Status st = manager->append_small_file(path, slice, info);
+ EXPECT_TRUE(st.ok()) << "Failed to append file phase2_" << i << ": " << st.msg();
+
+ // Verify file is in global index map
+ auto it = manager->global_slice_locations_for_test().find(path);
+ ASSERT_NE(it, manager->global_slice_locations_for_test().end());
+ EXPECT_EQ(it->second.size, file_size);
+ EXPECT_LT(it->second.size, config::small_file_threshold_bytes);
+ }
+
+ // Phase 4: Modify threshold to 30 bytes (smaller than file size)
+ config::small_file_threshold_bytes = 30;
+
+ // Phase 5: Import a file larger than new threshold
+ // This file should NOT be merged because 50 > 30
+ {
+ std::string path = "file_large_after_threshold_change";
+ std::string payload(file_size, 'z');
+ Slice slice(payload);
+
+ Status st = manager->append_small_file(path, slice, info);
+ EXPECT_TRUE(st.ok());
+
+ // Verify file is NOT in global index map (should be skipped)
+ auto it = manager->global_slice_locations_for_test().find(path);
+ EXPECT_EQ(it, manager->global_slice_locations_for_test().end())
+ << "File larger than threshold should not be merged";
+ }
+
+ // Verify all files from phase 1 and phase 2 are in merge files
+ int total_packed_files = 0;
+ int64_t total_merged_size = 0;
+
+ // Count files in current merge file
+ auto* current_state = manager->current_packed_files_for_test()[_resource_id].get();
+ ASSERT_NE(current_state, nullptr);
+ total_packed_files += current_state->slice_locations.size();
+ total_merged_size += current_state->total_size;
+
+ // Count files in uploading merge files
+ for (const auto& [path, state] : manager->uploading_packed_files_for_test()) {
+ total_packed_files += state->slice_locations.size();
+ total_merged_size += state->total_size;
+ }
+
+ // Should have phase1_file_count + phase2_file_count = 10 files merged
+ EXPECT_EQ(total_packed_files, phase1_file_count + phase2_file_count);
+ EXPECT_EQ(total_merged_size, (phase1_file_count + phase2_file_count) * file_size);
+
+ // Verify all phase 1 and phase 2 files are in global index map
+ for (int i = 0; i < phase1_file_count; ++i) {
+ std::string path = "file_phase1_" + std::to_string(i);
+ auto it = manager->global_slice_locations_for_test().find(path);
+ EXPECT_NE(it, manager->global_slice_locations_for_test().end())
+ << "Phase 1 file " << i << " should be in global index map";
+ if (it != manager->global_slice_locations_for_test().end()) {
+ EXPECT_EQ(it->second.size, file_size);
+ }
+ }
+
+ for (int i = 0; i < phase2_file_count; ++i) {
+ std::string path = "file_phase2_" + std::to_string(i);
+ auto it = manager->global_slice_locations_for_test().find(path);
+ EXPECT_NE(it, manager->global_slice_locations_for_test().end())
+ << "Phase 2 file " << i << " should be in global index map";
+ if (it != manager->global_slice_locations_for_test().end()) {
+ EXPECT_EQ(it->second.size, file_size);
+ }
+ }
+
+ // Phase 6: Modify threshold back to a larger value (e.g., 100 bytes)
+ // This verifies that threshold changes are applied dynamically
+ config::small_file_threshold_bytes = 100;
+
+ // Phase 7: Verify merge file rotation can still be triggered after threshold change
+ // Add a small file (25 bytes) that should be merged with new threshold
+ {
+ std::string path_trigger = "trigger_file_after_threshold";
+ constexpr int trigger_file_size = 25; // 25 bytes, should be merged (25 < 100)
+ std::string payload_trigger(trigger_file_size, 't');
+ Slice slice_trigger(payload_trigger);
+
+ size_t create_before = file_system->created_paths().size();
+ Status st = manager->append_small_file(path_trigger, slice_trigger, info);
+ EXPECT_TRUE(st.ok());
+
+ // Should create new merge file when threshold is reached (if not already)
+ EXPECT_GE(file_system->created_paths().size(), create_before);
+
+ // Verify trigger file is in global index map (should be merged because 25 < 100)
+ auto trigger_it = manager->global_slice_locations_for_test().find(path_trigger);
+ EXPECT_NE(trigger_it, manager->global_slice_locations_for_test().end())
+ << "Trigger file should be in global index map after threshold change";
+ if (trigger_it != manager->global_slice_locations_for_test().end()) {
+ EXPECT_EQ(trigger_it->second.size, trigger_file_size);
+ }
+ }
+}
+
+} // namespace doris::io
diff --git a/be/test/io/fs/packed_file_reader_test.cpp b/be/test/io/fs/packed_file_reader_test.cpp
new file mode 100644
index 00000000000..feaf5c37f3d
--- /dev/null
+++ b/be/test/io/fs/packed_file_reader_test.cpp
@@ -0,0 +1,271 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_reader.h"
+
+#include <gtest/gtest.h>
+
+#include <cstring>
+#include <memory>
+#include <string>
+
+#include "common/status.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/path.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+using doris::Status;
+
+// Mock FileReader for testing PackedFileReader
+class MockFileReader : public FileReader {
+public:
+ explicit MockFileReader(std::string content) : _content(std::move(content)) {}
+
+ Status close() override {
+ _closed = true;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ size_t size() const override { return _content.size(); }
+
+ bool closed() const override { return _closed; }
+
+ void set_read_status(Status st) { _read_status = std::move(st); }
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) override {
+ if (!_read_status.ok()) {
+ return _read_status;
+ }
+
+ if (offset >= _content.size()) {
+ *bytes_read = 0;
+ return Status::OK();
+ }
+
+ size_t available = _content.size() - offset;
+ size_t to_read = std::min(result.get_size(), available);
+ std::memcpy(result.mutable_data(), _content.data() + offset, to_read);
+ *bytes_read = to_read;
+ return Status::OK();
+ }
+
+private:
+ Path _path = Path("mock_file");
+ std::string _content;
+ bool _closed = false;
+ Status _read_status = Status::OK();
+};
+
+// Test fixture for PackedFileReader
+class PackedFileReaderTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _packed_file_content = "0123456789abcdefghijklmnopqrstuvwxyz";
+ _inner_reader = std::make_shared<MockFileReader>(_packed_file_content);
+ _packed_file_offset = 10;
+ _file_size = 20;
+ }
+
+ std::string _packed_file_content;
+ FileReaderSPtr _inner_reader;
+ int64_t _packed_file_offset;
+ int64_t _file_size;
+};
+
+TEST_F(PackedFileReaderTest, ReadFromMergeFile) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[100];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Read from offset 0 in the small file (which is at offset 10 in merge file)
+ Status st = reader.read_at(0, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(bytes_read, _file_size);
+
+ // Verify content matches expected portion of merge file
+ std::string expected = _packed_file_content.substr(_packed_file_offset, _file_size);
+ EXPECT_EQ(std::string(buffer, bytes_read), expected);
+}
+
+TEST_F(PackedFileReaderTest, ReadPartialData) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[10];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ Status st = reader.read_at(0, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(bytes_read, 10);
+
+ std::string expected = _packed_file_content.substr(_packed_file_offset, 10);
+ EXPECT_EQ(std::string(buffer, bytes_read), expected);
+}
+
+TEST_F(PackedFileReaderTest, ReadAtOffset) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[10];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Read from offset 5 in the small file (which is at offset 15 in merge file)
+ Status st = reader.read_at(5, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(bytes_read, 10);
+
+ std::string expected = _packed_file_content.substr(_packed_file_offset + 5, 10);
+ EXPECT_EQ(std::string(buffer, bytes_read), expected);
+}
+
+TEST_F(PackedFileReaderTest, ReadBeyondFileSize) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[100];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Try to read beyond file size
+ Status st = reader.read_at(0, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ // Should only read up to file size
+ EXPECT_EQ(bytes_read, _file_size);
+}
+
+TEST_F(PackedFileReaderTest, ReadAtEndOfFile) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[10];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Read from near the end of the file
+ Status st = reader.read_at(15, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ // Should only read remaining bytes (5 bytes)
+ EXPECT_EQ(bytes_read, 5);
+
+ std::string expected = _packed_file_content.substr(_packed_file_offset + 15, 5);
+ EXPECT_EQ(std::string(buffer, bytes_read), expected);
+}
+
+TEST_F(PackedFileReaderTest, ReadBeyondFileBoundary) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[10];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Try to read from beyond file size (offset 25, file_size is 20)
+ // When offset >= file_size, _file_size - offset becomes negative
+ // When cast to size_t, it becomes a very large number
+ // So max_read = min(result.get_size(), large_number) = result.get_size()
+ // actual_offset = merge_offset + offset = 10 + 25 = 35
+ // Content has 36 chars (indices 0-35), so inner reader can read 1 byte at offset 35
+ // The behavior is that it reads 1 byte from the merge file, even though it's beyond the small file size
+ Status st = reader.read_at(25, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ // Should read at most the buffer size (implementation may read from merge file)
+ EXPECT_LE(bytes_read, result.get_size());
+}
+
+TEST_F(PackedFileReaderTest, Close) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ EXPECT_FALSE(reader.closed());
+ Status st = reader.close();
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(reader.closed());
+}
+
+TEST_F(PackedFileReaderTest, CloseMultipleTimes) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ Status st1 = reader.close();
+ EXPECT_TRUE(st1.ok());
+ EXPECT_TRUE(reader.closed());
+
+ // Closing again should be OK
+ Status st2 = reader.close();
+ EXPECT_TRUE(st2.ok());
+ EXPECT_TRUE(reader.closed());
+}
+
+TEST_F(PackedFileReaderTest, PathAndSize) {
+ Path file_path("test_path");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ EXPECT_EQ(reader.path().native(), "test_path");
+ EXPECT_EQ(reader.size(), _file_size);
+}
+
+TEST_F(PackedFileReaderTest, ReadWithZeroOffset) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[5];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ Status st = reader.read_at(0, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(bytes_read, 5);
+
+ std::string expected = _packed_file_content.substr(_packed_file_offset, 5);
+ EXPECT_EQ(std::string(buffer, bytes_read), expected);
+}
+
+TEST_F(PackedFileReaderTest, ReadWithSmallBuffer) {
+ Path file_path("test_file");
+ PackedFileReader reader(_inner_reader, file_path, _packed_file_offset, _file_size);
+
+ char buffer[3];
+ Slice result(buffer, sizeof(buffer));
+ size_t bytes_read = 0;
+
+ // Read multiple times with small buffer
+ std::string all_read;
+ for (int i = 0; i < 7; ++i) {
+ Status st = reader.read_at(i * 3, result, &bytes_read, nullptr);
+ EXPECT_TRUE(st.ok());
+ if (bytes_read > 0) {
+ all_read.append(buffer, bytes_read);
+ }
+ }
+
+ // Should have read all file content
+ std::string expected = _packed_file_content.substr(_packed_file_offset, _file_size);
+ EXPECT_EQ(all_read, expected);
+}
+
+} // namespace doris::io
diff --git a/be/test/io/fs/packed_file_system_test.cpp b/be/test/io/fs/packed_file_system_test.cpp
new file mode 100644
index 00000000000..7f8a4af63df
--- /dev/null
+++ b/be/test/io/fs/packed_file_system_test.cpp
@@ -0,0 +1,439 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_system.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "common/status.h"
+#include "io/fs/file_reader.h"
+#include "io/fs/file_system.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/path.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+using doris::Status;
+
+// Mock FileReader for testing PackedFileSystem
+class MockFileReader : public FileReader {
+public:
+ explicit MockFileReader(std::string content) : _content(std::move(content)) {}
+
+ Status close() override {
+ _closed = true;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ size_t size() const override { return _content.size(); }
+
+ bool closed() const override { return _closed; }
+
+protected:
+ Status read_at_impl(size_t offset, Slice result, size_t* bytes_read,
+ const IOContext* /*io_ctx*/) override {
+ if (offset >= _content.size()) {
+ *bytes_read = 0;
+ return Status::OK();
+ }
+ size_t available = _content.size() - offset;
+ size_t to_read = std::min(result.get_size(), available);
+ std::memcpy(result.mutable_data(), _content.data() + offset, to_read);
+ *bytes_read = to_read;
+ return Status::OK();
+ }
+
+private:
+ Path _path = Path("mock_file");
+ std::string _content;
+ bool _closed = false;
+};
+
+// Mock FileWriter for testing PackedFileSystem
+class MockFileWriterForMerge : public FileWriter {
+public:
+ explicit MockFileWriterForMerge(std::string path) : _path(std::move(path)) {}
+
+ Status close(bool non_block = false) override {
+ if (_state == FileWriter::State::CLOSED) {
+ return Status::OK();
+ }
+ if (non_block) {
+ _state = FileWriter::State::ASYNC_CLOSING;
+ return Status::OK();
+ }
+ _state = FileWriter::State::CLOSED;
+ return Status::OK();
+ }
+
+ Status appendv(const Slice* data, size_t data_cnt) override {
+ for (size_t i = 0; i < data_cnt; ++i) {
+ _written.append(reinterpret_cast<const char*>(data[i].data), data[i].size);
+ _bytes_appended += data[i].size;
+ }
+ ++_append_calls;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ size_t bytes_appended() const override { return _bytes_appended; }
+
+ FileWriter::State state() const override { return _state; }
+
+private:
+ Path _path;
+ size_t _bytes_appended = 0;
+ size_t _append_calls = 0; // Add to match packed_file_writer_test.cpp
+ std::string _written;
+ Status _append_status = Status::OK(); // Add to match packed_file_writer_test.cpp
+ Status _close_status = Status::OK(); // Add to match packed_file_writer_test.cpp
+ FileWriter::State _state = FileWriter::State::OPENED;
+};
+
+// Mock FileSystem for testing PackedFileSystem
+class MockFileSystemForMerge : public FileSystem {
+public:
+ MockFileSystemForMerge() : FileSystem("mock_fs", FileSystemType::LOCAL) {}
+
+ void set_create_status(Status st) { _create_status = std::move(st); }
+ void set_open_status(Status st) { _open_status = std::move(st); }
+ void set_exists_result(bool exists) { _exists_result = exists; }
+ void set_file_size(int64_t size) { _file_size = size; }
+
+ MockFileWriterForMerge* last_writer() const { return _last_writer; }
+ MockFileReader* last_reader() const { return _last_reader; }
+
+protected:
+ Status create_file_impl(const Path& file, FileWriterPtr* writer,
+ const FileWriterOptions* /*opts*/) override {
+ if (!_create_status.ok()) {
+ return _create_status;
+ }
+ auto mock_writer = std::make_unique<MockFileWriterForMerge>(file.native());
+ _last_writer = mock_writer.get();
+ *writer = std::move(mock_writer);
+ return Status::OK();
+ }
+
+ Status open_file_impl(const Path& file, FileReaderSPtr* reader,
+ const FileReaderOptions* /*opts*/) override {
+ if (!_open_status.ok()) {
+ return _open_status;
+ }
+ // Create a mock reader with some content
+ auto mock_reader = std::make_shared<MockFileReader>("mock_content");
+ _last_reader = mock_reader.get();
+ *reader = std::move(mock_reader);
+ return Status::OK();
+ }
+
+ Status exists_impl(const Path& /*path*/, bool* res) const override {
+ *res = _exists_result;
+ return Status::OK();
+ }
+
+ Status file_size_impl(const Path& /*file*/, int64_t* size) const override {
+ *size = _file_size;
+ return Status::OK();
+ }
+
+ Status create_directory_impl(const Path& /*dir*/, bool /*failed_if_exists*/) override {
+ return Status::OK();
+ }
+
+ Status delete_file_impl(const Path& /*file*/) override { return Status::OK(); }
+
+ Status batch_delete_impl(const std::vector<Path>& /*files*/) override { return Status::OK(); }
+
+ Status delete_directory_impl(const Path& /*dir*/) override { return Status::OK(); }
+
+ Status list_impl(const Path& /*dir*/, bool /*only_file*/, std::vector<FileInfo>* /*files*/,
+ bool* exists) override {
+ *exists = true;
+ return Status::OK();
+ }
+
+ Status rename_impl(const Path& /*orig_name*/, const Path& /*new_name*/) override {
+ return Status::OK();
+ }
+
+ Status absolute_path(const Path& path, Path& abs_path) const override {
+ abs_path = path;
+ return Status::OK();
+ }
+
+private:
+ Status _create_status = Status::OK();
+ Status _open_status = Status::OK();
+ bool _exists_result = true;
+ int64_t _file_size = 0;
+ MockFileWriterForMerge* _last_writer = nullptr;
+ MockFileReader* _last_reader = nullptr;
+};
+
+// Test fixture for PackedFileSystem
+class PackedFileSystemTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _old_small_threshold = config::small_file_threshold_bytes;
+ config::small_file_threshold_bytes = 100;
+ _inner_fs = std::make_shared<MockFileSystemForMerge>();
+ _append_info.resource_id = "test_resource";
+ _append_info.tablet_id = 12345;
+ _append_info.rowset_id = "rowset_1";
+ _append_info.txn_id = 6789;
+ }
+
+ void TearDown() override { config::small_file_threshold_bytes = _old_small_threshold; }
+
+ int64_t _old_small_threshold = 0;
+ std::shared_ptr<MockFileSystemForMerge> _inner_fs;
+ PackedAppendContext _append_info;
+};
+
+TEST_F(PackedFileSystemTest, CreateFileWrapsWithPackedFileWriter) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path file_path("test_file");
+ FileWriterPtr writer;
+ Status st = merge_fs.create_file(file_path, &writer, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_NE(writer, nullptr);
+
+ // Writer should be a PackedFileWriter (we can't easily check type, but it should work)
+ std::string data = "test";
+ Slice slice(data);
+ st = writer->appendv(&slice, 1);
+ EXPECT_TRUE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, OpenFileNotInMergeFile) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path file_path("regular_file");
+ FileReaderSPtr reader;
+ Status st = merge_fs.open_file(file_path, &reader, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_NE(reader, nullptr);
+}
+
+TEST_F(PackedFileSystemTest, OpenFileInMergeFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedSliceLocation index;
+ index.packed_file_path = "packed_file_path";
+ index.offset = 10;
+ index.size = 20;
+ index_map["test_file"] = index;
+
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+
+ Path file_path("test_file");
+ FileReaderSPtr reader;
+ Status st = merge_fs.open_file(file_path, &reader, nullptr);
+ EXPECT_TRUE(st.ok());
+ EXPECT_NE(reader, nullptr);
+ EXPECT_EQ(reader->size(), index.size);
+}
+
+TEST_F(PackedFileSystemTest, ExistsForMergeFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedSliceLocation index;
+ index.packed_file_path = "packed_file_path";
+ index.offset = 10;
+ index.size = 20;
+ index_map["test_file"] = index;
+
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+ _inner_fs->set_exists_result(true);
+
+ Path file_path("test_file");
+ bool exists = false;
+ Status st = merge_fs.exists(file_path, &exists);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(exists);
+}
+
+TEST_F(PackedFileSystemTest, ExistsForRegularFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+ _inner_fs->set_exists_result(true);
+
+ Path file_path("regular_file");
+ bool exists = false;
+ Status st = merge_fs.exists(file_path, &exists);
+ EXPECT_TRUE(st.ok());
+ EXPECT_TRUE(exists);
+}
+
+TEST_F(PackedFileSystemTest, ExistsForNonExistentFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+ _inner_fs->set_exists_result(false);
+
+ Path file_path("non_existent_file");
+ bool exists = true;
+ Status st = merge_fs.exists(file_path, &exists);
+ EXPECT_TRUE(st.ok());
+ EXPECT_FALSE(exists);
+}
+
+TEST_F(PackedFileSystemTest, FileSizeForMergeFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedSliceLocation index;
+ index.packed_file_path = "packed_file_path";
+ index.offset = 10;
+ index.size = 30;
+ index_map["test_file"] = index;
+
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+
+ Path file_path("test_file");
+ int64_t file_size = 0;
+ Status st = merge_fs.file_size(file_path, &file_size);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(file_size, index.size);
+}
+
+TEST_F(PackedFileSystemTest, FileSizeForRegularFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+ _inner_fs->set_file_size(100);
+
+ Path file_path("regular_file");
+ int64_t file_size = 0;
+ Status st = merge_fs.file_size(file_path, &file_size);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(file_size, 100);
+}
+
+TEST_F(PackedFileSystemTest, UnsupportedCreateDirectory) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path dir_path("test_dir");
+ Status st = merge_fs.create_directory(dir_path);
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, UnsupportedDeleteFile) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path file_path("test_file");
+ Status st = merge_fs.delete_file(file_path);
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, UnsupportedDeleteDirectory) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path dir_path("test_dir");
+ Status st = merge_fs.delete_directory(dir_path);
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, UnsupportedList) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path dir_path("test_dir");
+ std::vector<FileInfo> files;
+ bool exists = false;
+ Status st = merge_fs.list(dir_path, false, &files, &exists);
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, UnsupportedRename) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path orig_path("orig");
+ Path new_path("new");
+ Status st = merge_fs.rename(orig_path, new_path);
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, AbsolutePath) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path path("test_path");
+ Path abs_path;
+ Status st = merge_fs.absolute_path(path, abs_path);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(abs_path.native(), path.native());
+}
+
+TEST_F(PackedFileSystemTest, MultipleFilesInMergeFile) {
+ std::unordered_map<std::string, PackedSliceLocation> index_map;
+
+ PackedSliceLocation index1;
+ index1.packed_file_path = "packed_file_path";
+ index1.offset = 0;
+ index1.size = 10;
+ index_map["file1"] = index1;
+
+ PackedSliceLocation index2;
+ index2.packed_file_path = "packed_file_path";
+ index2.offset = 10;
+ index2.size = 20;
+ index_map["file2"] = index2;
+
+ PackedFileSystem merge_fs(_inner_fs, index_map, _append_info);
+
+ // Test file1
+ Path file1_path("file1");
+ int64_t file1_size = 0;
+ Status st = merge_fs.file_size(file1_path, &file1_size);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(file1_size, index1.size);
+
+ // Test file2
+ Path file2_path("file2");
+ int64_t file2_size = 0;
+ st = merge_fs.file_size(file2_path, &file2_size);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(file2_size, index2.size);
+}
+
+TEST_F(PackedFileSystemTest, ExistsWithoutIndexMapInitialized) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path file_path("test_file");
+ bool exists = false;
+ Status st = merge_fs.exists(file_path, &exists);
+ // Should fail because index map is not initialized
+ EXPECT_FALSE(st.ok());
+}
+
+TEST_F(PackedFileSystemTest, FileSizeWithoutIndexMapInitialized) {
+ PackedFileSystem merge_fs(_inner_fs, _append_info);
+
+ Path file_path("test_file");
+ int64_t file_size = 0;
+ Status st = merge_fs.file_size(file_path, &file_size);
+ // Should fail because index map is not initialized
+ EXPECT_FALSE(st.ok());
+}
+
+} // namespace doris::io
diff --git a/be/test/io/fs/packed_file_writer_test.cpp b/be/test/io/fs/packed_file_writer_test.cpp
new file mode 100644
index 00000000000..dc8b2b5ec4d
--- /dev/null
+++ b/be/test/io/fs/packed_file_writer_test.cpp
@@ -0,0 +1,259 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_writer.h"
+
+#include <gtest/gtest.h>
+
+#include <memory>
+#include <string>
+
+#include "cloud/config.h"
+#include "common/config.h"
+#include "common/status.h"
+#include "io/fs/file_writer.h"
+#include "io/fs/packed_file_manager.h"
+#include "io/fs/path.h"
+#include "util/slice.h"
+
+namespace doris::io {
+
+using doris::Status;
+
+// Mock FileWriter for testing PackedFileWriter
+class MockFileWriterForMerge : public FileWriter {
+public:
+ explicit MockFileWriterForMerge(std::string path) : _path(std::move(path)) {}
+
+ void set_append_status(Status st) { _append_status = std::move(st); }
+ void set_close_status(Status st) { _close_status = std::move(st); }
+
+ size_t append_calls() const { return _append_calls; }
+ bool closed() const { return _state == FileWriter::State::CLOSED; }
+ size_t bytes_appended() const override { return _bytes_appended; }
+ const std::string& written_data() const { return _written; }
+
+ Status close(bool non_block = false) override {
+ if (_state == FileWriter::State::CLOSED) {
+ return Status::OK();
+ }
+ if (non_block) {
+ _state = FileWriter::State::ASYNC_CLOSING;
+ return Status::OK();
+ }
+ _state = FileWriter::State::CLOSED;
+ return _close_status;
+ }
+
+ Status appendv(const Slice* data, size_t data_cnt) override {
+ if (!_append_status.ok()) {
+ return _append_status;
+ }
+ for (size_t i = 0; i < data_cnt; ++i) {
+ _written.append(reinterpret_cast<const char*>(data[i].data), data[i].size);
+ _bytes_appended += data[i].size;
+ }
+ ++_append_calls;
+ return Status::OK();
+ }
+
+ const Path& path() const override { return _path; }
+
+ FileWriter::State state() const override { return _state; }
+
+private:
+ Path _path;
+ size_t _bytes_appended = 0;
+ size_t _append_calls = 0;
+ std::string _written;
+ Status _append_status = Status::OK();
+ Status _close_status = Status::OK();
+ FileWriter::State _state = FileWriter::State::OPENED;
+};
+
+// Test fixture for PackedFileWriter
+class PackedFileWriterTest : public testing::Test {
+protected:
+ void SetUp() override {
+ _old_small_threshold = config::small_file_threshold_bytes;
+ _old_merge_threshold = config::packed_file_size_threshold_bytes;
+ config::small_file_threshold_bytes = 100;
+ config::packed_file_size_threshold_bytes = 1024;
+
+ // Initialize PackedFileManager for testing
+ auto* manager = PackedFileManager::instance();
+ static_cast<void>(manager->init());
+
+ _inner_writer = std::make_unique<MockFileWriterForMerge>("test_file");
+ _append_info.resource_id = "test_resource";
+ _append_info.tablet_id = 12345;
+ _append_info.rowset_id = "rowset_1";
+ _append_info.txn_id = 6789;
+ }
+
+ void TearDown() override {
+ config::small_file_threshold_bytes = _old_small_threshold;
+ config::packed_file_size_threshold_bytes = _old_merge_threshold;
+ }
+
+ int64_t _old_small_threshold = 0;
+ int64_t _old_merge_threshold = 0;
+ std::unique_ptr<MockFileWriterForMerge> _inner_writer;
+ PackedAppendContext _append_info;
+};
+
+TEST_F(PackedFileWriterTest, SmallFileBuffered) {
+ Path file_path("small_file");
+ // Save pointer before move
+ auto* inner_writer_ptr = _inner_writer.get();
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ std::string data = "small data";
+ Slice slice(data);
+ Status st = writer.appendv(&slice, 1);
+ EXPECT_TRUE(st.ok());
+
+ // Small file should be buffered, not written to inner writer
+ EXPECT_EQ(writer.bytes_appended(), data.size());
+ EXPECT_EQ(inner_writer_ptr->append_calls(), 0);
+}
+
+TEST_F(PackedFileWriterTest, LargeFileDirectWrite) {
+ Path file_path("large_file");
+ // Save pointer before move
+ auto* inner_writer_ptr = _inner_writer.get();
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ // Write data larger than threshold
+ std::string data(150, 'x');
+ Slice slice(data);
+ Status st = writer.appendv(&slice, 1);
+ EXPECT_TRUE(st.ok());
+
+ // Large file should be written directly to inner writer
+ EXPECT_EQ(writer.bytes_appended(), data.size());
+ EXPECT_GT(inner_writer_ptr->append_calls(), 0);
+}
+
+TEST_F(PackedFileWriterTest, SwitchToDirectWrite) {
+ Path file_path("switch_file");
+ // Save pointer before move
+ auto* inner_writer_ptr = _inner_writer.get();
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ // First write small data (buffered)
+ std::string small_data(50, 'a');
+ Slice small_slice(small_data);
+ Status st = writer.appendv(&small_slice, 1);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(inner_writer_ptr->append_calls(), 0);
+
+ // Then write more data that exceeds threshold
+ std::string large_data(60, 'b');
+ Slice large_slice(large_data);
+ st = writer.appendv(&large_slice, 1);
+ EXPECT_TRUE(st.ok());
+
+ // Should have switched to direct write and written buffered data
+ EXPECT_GT(inner_writer_ptr->append_calls(), 0);
+}
+
+TEST_F(PackedFileWriterTest, CloseAsync) {
+ Path file_path("async_file");
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ std::string data = "test data";
+ Slice slice(data);
+ EXPECT_TRUE(writer.appendv(&slice, 1).ok());
+
+ // For small files, async close sends data to merge manager
+ // This may fail if merge manager is not properly initialized
+ Status st = writer.close(true);
+ // If merge manager fails, state remains OPENED, which is acceptable for this test
+ // We just verify that close() was called and didn't crash
+ // The actual behavior depends on merge manager initialization
+ if (st.ok()) {
+ EXPECT_EQ(writer.state(), FileWriter::State::ASYNC_CLOSING);
+ }
+ // If it fails, state may remain OPENED, which is acceptable
+}
+
+TEST_F(PackedFileWriterTest, CloseSync) {
+ Path file_path("sync_file");
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ std::string data = "test data";
+ Slice slice(data);
+ EXPECT_TRUE(writer.appendv(&slice, 1).ok());
+
+ Status st = writer.close(false);
+ // Close may fail if merge manager doesn't have the file, but state should be CLOSED for direct write
+ // For small files, it may fail but we test the structure
+ if (st.ok() || writer.state() == FileWriter::State::CLOSED) {
+ EXPECT_EQ(writer.state(), FileWriter::State::CLOSED);
+ }
+}
+
+TEST_F(PackedFileWriterTest, PathAndBytesAppended) {
+ Path file_path("test_path");
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ // PackedFileWriter::path() returns _inner_writer->path(), which is "test_file" from SetUp
+ EXPECT_EQ(writer.path().native(), "test_file");
+ EXPECT_EQ(writer.bytes_appended(), 0);
+
+ std::string data = "test";
+ Slice slice(data);
+ EXPECT_TRUE(writer.appendv(&slice, 1).ok());
+
+ EXPECT_EQ(writer.bytes_appended(), data.size());
+}
+
+TEST_F(PackedFileWriterTest, MultipleAppends) {
+ Path file_path("multi_append_file");
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ std::string data1 = "first";
+ std::string data2 = "second";
+ std::string data3 = "third";
+
+ Slice slice1(data1);
+ Slice slice2(data2);
+ Slice slice3(data3);
+
+ Status st = writer.appendv(&slice1, 1);
+ EXPECT_TRUE(st.ok());
+ st = writer.appendv(&slice2, 1);
+ EXPECT_TRUE(st.ok());
+ st = writer.appendv(&slice3, 1);
+ EXPECT_TRUE(st.ok());
+
+ EXPECT_EQ(writer.bytes_appended(), data1.size() + data2.size() + data3.size());
+}
+
+TEST_F(PackedFileWriterTest, EmptyAppend) {
+ Path file_path("empty_file");
+ PackedFileWriter writer(std::move(_inner_writer), file_path, _append_info);
+
+ std::string empty_data;
+ Slice slice(empty_data);
+ Status st = writer.appendv(&slice, 1);
+ EXPECT_TRUE(st.ok());
+ EXPECT_EQ(writer.bytes_appended(), 0);
+}
+
+} // namespace doris::io
diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp
index d87056320bb..543fcae69ab 100644
--- a/be/test/io/fs/s3_file_writer_test.cpp
+++ b/be/test/io/fs/s3_file_writer_test.cpp
@@ -1492,7 +1492,8 @@ TEST_F(S3FileWriterTest, test_empty_file) {
auto index_file_writer = std::make_unique<segment_v2::IndexFileWriter>(
fs, index_path, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2,
std::move(file_writer), false);
- EXPECT_TRUE(index_file_writer->close().ok());
+ EXPECT_TRUE(index_file_writer->begin_close().ok());
+ EXPECT_TRUE(index_file_writer->finish_close().ok());
}
} // namespace doris
diff --git a/be/test/io/packed_file_trailer_test.cpp b/be/test/io/packed_file_trailer_test.cpp
new file mode 100644
index 00000000000..78e1f17754b
--- /dev/null
+++ b/be/test/io/packed_file_trailer_test.cpp
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "io/fs/packed_file_trailer.h"
+
+#include <gtest/gtest.h>
+#include <unistd.h>
+
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+#include "common/status.h"
+#include "util/coding.h"
+
+namespace doris::io {
+
+using doris::Status;
+
+namespace {
+std::string unique_temp_file() {
+ auto dir = std::filesystem::temp_directory_path();
+ std::string pattern = (dir / "packed_file_XXXXXX").string();
+ int fd = mkstemp(pattern.data());
+ if (fd != -1) {
+ close(fd);
+ }
+ return pattern;
+}
+
+void write_file(const std::string& path, const std::string& data) {
+ std::ofstream ofs(path, std::ios::binary | std::ios::trunc);
+ ofs.write(data.data(), data.size());
+}
+} // namespace
+
+TEST(PackedFileTrailerTest, ReadNewFormatTrailer) {
+ cloud::PackedFileInfoPB info;
+ info.set_resource_id("resource");
+ info.set_total_slice_num(1);
+ auto* slice = info.add_slices();
+ slice->set_path("a");
+ slice->set_offset(10);
+ slice->set_size(20);
+
+ cloud::PackedFileFooterPB footer_pb;
+ footer_pb.mutable_packed_file_info()->CopyFrom(info);
+
+ std::string serialized_footer;
+ ASSERT_TRUE(footer_pb.SerializeToString(&serialized_footer));
+
+ std::string file_content = "data";
+ file_content.append(serialized_footer);
+ put_fixed32_le(&file_content, static_cast<uint32_t>(serialized_footer.size()));
+ put_fixed32_le(&file_content, kPackedFileTrailerVersion);
+
+ auto path = unique_temp_file();
+ write_file(path, file_content);
+
+ cloud::PackedFileFooterPB parsed;
+ uint32_t version = 0;
+ Status st = read_packed_file_trailer(path, &parsed, &version);
+ ASSERT_TRUE(st.ok()) << st;
+ EXPECT_EQ(version, kPackedFileTrailerVersion);
+ ASSERT_TRUE(parsed.has_packed_file_info());
+ EXPECT_EQ(parsed.packed_file_info().resource_id(), info.resource_id());
+ ASSERT_EQ(parsed.packed_file_info().slices_size(), 1);
+ EXPECT_EQ(parsed.packed_file_info().slices(0).path(), "a");
+ EXPECT_EQ(parsed.packed_file_info().slices(0).offset(), 10);
+ EXPECT_EQ(parsed.packed_file_info().slices(0).size(), 20);
+
+ std::filesystem::remove(path);
+}
+
+TEST(PackedFileTrailerTest, ReadLegacyTrailer) {
+ cloud::PackedFileInfoPB info;
+ info.set_total_slice_num(2);
+ info.set_remaining_slice_bytes(100);
+
+ std::string serialized_info;
+ ASSERT_TRUE(info.SerializeToString(&serialized_info));
+
+ std::string file_content = "legacy_payload";
+ file_content.append(serialized_info);
+ put_fixed32_le(&file_content, static_cast<uint32_t>(serialized_info.size()));
+
+ auto path = unique_temp_file();
+ write_file(path, file_content);
+
+ cloud::PackedFileFooterPB parsed;
+ uint32_t version = 0;
+ Status st = read_packed_file_trailer(path, &parsed, &version);
+ ASSERT_TRUE(st.ok()) << st;
+ EXPECT_EQ(version, 0);
+ ASSERT_TRUE(parsed.has_packed_file_info());
+ EXPECT_EQ(parsed.packed_file_info().total_slice_num(), info.total_slice_num());
+ EXPECT_EQ(parsed.packed_file_info().remaining_slice_bytes(), info.remaining_slice_bytes());
+
+ std::filesystem::remove(path);
+}
+
+} // namespace doris::io
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/empty_index_file_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/empty_index_file_test.cpp
index dbfe8a9e028..0386ce37ec6 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/empty_index_file_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/empty_index_file_test.cpp
@@ -86,7 +86,8 @@ TEST_F(EmptyIndexFileTest, test_empty_index_file) {
auto index_file_writer = std::make_unique<segment_v2::IndexFileWriter>(
fs, index_path, rowset_id, seg_id, InvertedIndexStorageFormatPB::V2,
std::move(file_writer), false);
- EXPECT_TRUE(index_file_writer->close().ok());
+ EXPECT_TRUE(index_file_writer->begin_close().ok());
+ EXPECT_TRUE(index_file_writer->finish_close().ok());
}
} // namespace doris
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query_test.cpp
index f44514423e5..43c07aa3517 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_edge_query_test.cpp
@@ -150,7 +150,9 @@ public:
// Finish and close
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query_test.cpp
index d8925f82d74..330929d4ce9 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_prefix_query_test.cpp
@@ -149,7 +149,9 @@ public:
// Finish and close
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
index 197d0a69465..ae13f542341 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index/query/phrase_query_test.cpp
@@ -150,7 +150,9 @@ public:
// Finish and close
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
index 6aa0d5c0b0e..35bbcedb6b6 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_array_test.cpp
@@ -279,7 +279,8 @@ public:
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
ExpectedDocMap expected = {{"amory", {0, 1}}, {"doris", {0}}, {"commiter", {1}}};
check_terms_stats(index_path_prefix, &expected, {}, InvertedIndexStorageFormatPB::V1,
@@ -365,7 +366,8 @@ public:
st = _inverted_index_builder->add_array_nulls(null_map, block.rows());
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
ExpectedDocMap expected = {{"amory", {0, 1}}, {"doris", {0}}, {"commiter", {1}}};
check_terms_stats(index_path_prefix, &expected, {}, InvertedIndexStorageFormatPB::V1,
@@ -477,7 +479,8 @@ public:
st = _inverted_index_builder->add_array_nulls(null_map, block.rows());
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
// Expected inverted index result: only index non-null elements
// Row 1: non-null in a2 is "test"
@@ -591,7 +594,8 @@ public:
st = _inverted_index_builder->add_array_nulls(null_map, block.rows());
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
// Expected inverted index result: only index non-null elements
// Row 1: non-null in a2 is "test"
@@ -793,7 +797,8 @@ public:
}
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
std::vector<int> expected_null_bitmap = {0, 3, 5, 7};
check_terms_stats(index_path_prefix, &merged_expected, expected_null_bitmap,
@@ -896,7 +901,8 @@ public:
st = _inverted_index_builder->add_array_nulls(null_map, block.rows());
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
// expected inverted index: row0 contains "123" and "456" (doc id 0), row1 is null, row2 contains "789" and "101112" (doc id 2)
ExpectedDocMap expected = {{"123", {0}}, {"456", {0}}, {"789", {2}}, {"101112", {2}}};
@@ -1007,7 +1013,8 @@ public:
EXPECT_EQ(st, Status::OK());
EXPECT_EQ(_inverted_index_builder->finish(), Status::OK());
- EXPECT_EQ(index_file_writer->close(), Status::OK());
+ EXPECT_EQ(index_file_writer->begin_close(), Status::OK());
+ EXPECT_EQ(index_file_writer->finish_close(), Status::OK());
std::vector<int> expected_null_bitmap = {0, 1};
ExpectedDocMap expected {};
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
index 0f86dc364c3..7ff697c67e9 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_compound_reader_test.cpp
@@ -375,7 +375,9 @@ TEST_F(DorisCompoundReaderTest, IntegrationWithFileWriter) {
out->close();
_CLLDELETE(out);
- auto st = index_file_writer->close();
+ auto st = index_file_writer->begin_close();
+ ASSERT_TRUE(st.ok()) << st;
+ st = index_file_writer->finish_close();
ASSERT_TRUE(st.ok()) << st;
auto file_reader = std::make_unique<IndexFileReader>(
@@ -422,7 +424,9 @@ TEST_F(DorisCompoundReaderTest, IntegrationWithFileWriter) {
out->close();
_CLLDELETE(out);
- st = index_file_writer->close();
+ st = index_file_writer->begin_close();
+ ASSERT_TRUE(st.ok()) << st;
+ st = index_file_writer->finish_close();
ASSERT_TRUE(st.ok()) << st;
auto file_reader = std::make_unique<IndexFileReader>(
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_reader_test.cpp
index 51c2e76e8ba..206282cce30 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_file_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_file_reader_test.cpp
@@ -202,8 +202,10 @@ public:
dir->close();
- // Write and close the file - only call close(), not write()
- st = writer->close();
+ // Write and close the file - only call begin_close() and finish_close(), not write()
+ st = writer->begin_close();
+ ASSERT_TRUE(st.ok()) << st.msg();
+ st = writer->finish_close();
ASSERT_TRUE(st.ok()) << st.msg();
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
index 5ac8b773aef..568808699a5 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_file_writer_test.cpp
@@ -247,9 +247,14 @@ TEST_F(IndexFileWriterTest, WriteV1Test) {
out_file->close();
dir->close();
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
if (!close_status.ok()) {
- std::cout << "close error:" << close_status.msg() << std::endl;
+ std::cout << "begin_close error:" << close_status.msg() << std::endl;
+ }
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
+ if (!close_status.ok()) {
+ std::cout << "finish_close error:" << close_status.msg() << std::endl;
}
ASSERT_TRUE(close_status.ok());
@@ -302,7 +307,9 @@ TEST_F(IndexFileWriterTest, WriteV2Test) {
out_file_2->writeString("test2");
out_file_2->close();
dir_2->close();
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
const InvertedIndexFileInfo* file_info = writer.get_index_file_info();
@@ -620,7 +627,7 @@ TEST_F(IndexFileWriterTest, WriteV1ExceptionHandlingTest) {
::testing::_))
.WillOnce(::testing::Throw(CLuceneError(CL_ERR_IO, "Simulated exception", false)));
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -673,7 +680,7 @@ TEST_F(IndexFileWriterTest, WriteV2ExceptionHandlingTest) {
write_index_headers_and_metadata(::testing::_, ::testing::_))
.WillOnce(::testing::Throw(CLuceneError(CL_ERR_IO, "Simulated exception", false)));
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -779,7 +786,7 @@ TEST_F(IndexFileWriterTest, WriteV1OutputTest) {
out_file->close();
dir->close();
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -825,7 +832,7 @@ TEST_F(IndexFileWriterTest, WriteV2OutputTest) {
out_file->close();
dir->close();
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -885,7 +892,7 @@ TEST_F(IndexFileWriterTest, WriteV2OutputCloseErrorTest) {
out_file->close();
dir->close();
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -935,7 +942,7 @@ TEST_F(IndexFileWriterTest, WriteV1OutputCloseErrorTest) {
out_file->close();
dir->close();
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -998,7 +1005,9 @@ TEST_F(IndexFileWriterTest, AddIntoSearcherCacheTest) {
EXPECT_CALL(writer, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder))));
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
auto index_file_key = InvertedIndexDescriptor::get_index_file_cache_key(_index_path_prefix,
@@ -1058,7 +1067,9 @@ TEST_F(IndexFileWriterTest, CacheEvictionTest) {
EXPECT_CALL(writer1, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder1))));
- Status close_status1 = writer1.close();
+ Status close_status1 = writer1.begin_close();
+ ASSERT_TRUE(close_status1.ok());
+ close_status1 = writer1.finish_close();
ASSERT_TRUE(close_status1.ok());
MockIndexFileWriter writer2(_fs, _index_path_prefix + "_2", "rowset2", 2,
@@ -1089,7 +1100,9 @@ TEST_F(IndexFileWriterTest, CacheEvictionTest) {
EXPECT_CALL(writer2, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder2))));
- Status close_status2 = writer2.close();
+ Status close_status2 = writer2.begin_close();
+ ASSERT_TRUE(close_status2.ok());
+ close_status2 = writer2.finish_close();
ASSERT_TRUE(close_status2.ok());
MockIndexFileWriter writer3(_fs, _index_path_prefix + "_3", "rowset3", 3,
@@ -1119,7 +1132,9 @@ TEST_F(IndexFileWriterTest, CacheEvictionTest) {
EXPECT_CALL(writer3, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder3))));
- Status close_status3 = writer3.close();
+ Status close_status3 = writer3.begin_close();
+ ASSERT_TRUE(close_status3.ok());
+ close_status3 = writer3.finish_close();
ASSERT_TRUE(close_status3.ok());
InvertedIndexCacheHandle cache_handle1;
@@ -1189,7 +1204,9 @@ TEST_F(IndexFileWriterTest, CacheUpdateTest) {
EXPECT_CALL(writer, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder))));
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
auto index_file_key = InvertedIndexDescriptor::get_index_file_cache_key(
@@ -1227,7 +1244,9 @@ TEST_F(IndexFileWriterTest, CacheUpdateTest) {
EXPECT_CALL(writer_new, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder_new))));
- Status close_status_new = writer_new.close();
+ Status close_status_new = writer_new.begin_close();
+ ASSERT_TRUE(close_status_new.ok());
+ close_status_new = writer_new.finish_close();
ASSERT_TRUE(close_status_new.ok());
InvertedIndexCacheHandle cache_handle_new;
@@ -1273,7 +1292,9 @@ TEST_F(IndexFileWriterTest, AddIntoSearcherCacheV1Test) {
}));
EXPECT_CALL(writer, _construct_index_searcher_builder(testing::_))
.WillOnce(testing::Return(testing::ByMove(std::move(mock_builder))));
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
auto index_file_key = InvertedIndexDescriptor::get_index_file_cache_key(
@@ -1385,7 +1406,9 @@ TEST_F(IndexFileWriterTest, RowsetWriterCreateIndexFileWriterWithoutRamDir) {
// Cleanup
dir->close();
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ ASSERT_TRUE(status.ok());
+ status = index_file_writer->finish_close();
ASSERT_TRUE(status.ok());
}
@@ -1434,7 +1457,9 @@ TEST_F(IndexFileWriterTest, RowsetWriterCreateIndexFileWriterWithRamDir) {
// Cleanup
dir->close();
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ ASSERT_TRUE(status.ok());
+ status = index_file_writer->finish_close();
ASSERT_TRUE(status.ok());
}
@@ -1483,7 +1508,9 @@ TEST_F(IndexFileWriterTest, RowsetWriterCreateIndexFileWriterNonBaseCompaction)
// Cleanup
dir->close();
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ ASSERT_TRUE(status.ok());
+ status = index_file_writer->finish_close();
ASSERT_TRUE(status.ok());
}
@@ -1561,7 +1588,7 @@ TEST_F(IndexFileWriterTest, MultipleIndicesCreateOutputStreamException) {
.WillOnce(::testing::Throw(CLuceneError(CL_ERR_IO, "Simulated exception", false)));
// When we call close(), it should process both indices and fail on the second one
- Status status = writer_mock.close();
+ Status status = writer_mock.begin_close();
ASSERT_FALSE(status.ok());
ASSERT_EQ(status.code(), ErrorCode::INVERTED_INDEX_CLUCENE_ERROR);
}
@@ -1901,7 +1928,9 @@ TEST_F(IndexFileWriterTest, CloseV1DirectoryDeletionTest) {
dir->close();
// Close should handle directory cleanup
- Status status = writer.close();
+ Status status = writer.begin_close();
+ ASSERT_TRUE(status.ok());
+ status = writer.finish_close();
ASSERT_TRUE(status.ok());
}
@@ -1932,9 +1961,12 @@ TEST_F(IndexFileWriterTest, CloseV2CLuceneExceptionTest) {
dir->close();
// The test should verify that CLucene exceptions are properly caught and handled
- Status status = writer.close();
+ Status status = writer.begin_close();
// Even if there are CLucene errors, the status should still indicate completion
// The error handling should log the error but not crash
+ if (status.ok()) {
+ status = writer.finish_close();
+ }
}
// Test for compound directory deletion error handling
@@ -1966,7 +1998,9 @@ TEST_F(IndexFileWriterTest, CompoundDirectoryDeletionTest) {
if (std::strcmp(dir->getObjectName(), "DorisFSDirectory") == 0) {
// This path should trigger the compound directory deletion code
dir->close();
- Status status = writer.close();
+ Status status = writer.begin_close();
+ ASSERT_TRUE(status.ok());
+ status = writer.finish_close();
ASSERT_TRUE(status.ok());
}
}
@@ -2061,7 +2095,9 @@ TEST_F(IndexFileWriterTest, WriteIndexHeadersAndMetadataTest) {
ASSERT_TRUE(insert_st.ok());
// Test close which should trigger write_index_headers_and_metadata
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
// Clean up
@@ -2138,7 +2174,9 @@ TEST_F(IndexFileWriterTest, EmptyIndexV2Test) {
InvertedIndexStorageFormatPB::V2, std::move(file_writer));
// Close without adding any indices - should handle empty case
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
}
@@ -2149,7 +2187,9 @@ TEST_F(IndexFileWriterTest, StreamSinkFileWriterCloseTest) {
InvertedIndexStorageFormatPB::V2);
// Close should handle the case where _idx_v2_writer is not a StreamSinkFileWriter
- Status close_status = writer.close();
+ Status close_status = writer.begin_close();
+ ASSERT_TRUE(close_status.ok());
+ close_status = writer.finish_close();
ASSERT_TRUE(close_status.ok());
}
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
index a622be232d3..cf3c747b261 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_reader_test.cpp
@@ -153,7 +153,9 @@ public:
// Finish and close
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -215,7 +217,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -272,7 +276,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -2555,7 +2561,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -3025,7 +3033,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -3500,7 +3510,9 @@ public:
if (status.ok()) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
// Try to create reader and test unsupported query
diff --git a/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp b/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
index 3bd7fc5f6a8..eb2b2223284 100644
--- a/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
+++ b/be/test/olap/rowset/segment_v2/inverted_index_writer_test.cpp
@@ -357,7 +357,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
// Verify the terms stats
@@ -425,7 +427,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
// Verify the terms stats
@@ -491,7 +495,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
// For BKD index, we need to verify using BkdIndexReader instead of check_terms_stats
@@ -567,7 +573,9 @@ public:
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
// Restore original config value
@@ -756,12 +764,16 @@ TEST_F(InvertedIndexWriterTest, CompareUnicodeStringWriteResults) {
// Finish and close both writers
status = column_writer_enabled->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer_enabled->close();
+ status = index_file_writer_enabled->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer_enabled->finish_close();
EXPECT_TRUE(status.ok()) << status;
status = column_writer_disabled->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer_disabled->close();
+ status = index_file_writer_disabled->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer_disabled->finish_close();
EXPECT_TRUE(status.ok()) << status;
// Restore original config value
@@ -899,7 +911,9 @@ TEST_F(InvertedIndexWriterTest, ErrorHandlingInFileWriter) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -1023,7 +1037,9 @@ TEST_F(InvertedIndexWriterTest, ArrayValuesWithNulls) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -1157,7 +1173,9 @@ TEST_F(InvertedIndexWriterTest, NumericArrayWithErrorConditions) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -1211,7 +1229,9 @@ TEST_F(InvertedIndexWriterTest, CopyFileErrorHandling) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -1280,7 +1300,9 @@ TEST_F(InvertedIndexWriterTest, CollectionValueProcessing) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
@@ -1344,7 +1366,9 @@ TEST_F(InvertedIndexWriterTest, BKDWriterErrorConditions) {
status = column_writer->finish();
EXPECT_TRUE(status.ok()) << status;
- status = index_file_writer->close();
+ status = index_file_writer->begin_close();
+ EXPECT_TRUE(status.ok()) << status;
+ status = index_file_writer->finish_close();
EXPECT_TRUE(status.ok()) << status;
}
diff --git a/be/test/olap/rowset/segment_v2/segment_corruption_test.cpp b/be/test/olap/rowset/segment_v2/segment_corruption_test.cpp
index c663429e160..c1336912332 100644
--- a/be/test/olap/rowset/segment_v2/segment_corruption_test.cpp
+++ b/be/test/olap/rowset/segment_v2/segment_corruption_test.cpp
@@ -249,7 +249,9 @@ public:
EXPECT_TRUE(file_writer->close().ok());
// Close the index file writer (it was already written by SegmentWriter during finalize)
- st = index_file_writer->close();
+ st = index_file_writer->begin_close();
+ EXPECT_TRUE(st.ok()) << st;
+ st = index_file_writer->finish_close();
EXPECT_TRUE(st.ok()) << st;
return seg_path;
diff --git a/be/test/olap/test_data/rowset_meta.json b/be/test/olap/test_data/rowset_meta.json
index d446e2a34e9..00415c39634 100644
--- a/be/test/olap/test_data/rowset_meta.json
+++ b/be/test/olap/test_data/rowset_meta.json
@@ -18,5 +18,6 @@
"lo": 10
},
"num_segments": 1,
- "has_variant_type_in_schema": false
+ "has_variant_type_in_schema": false,
+ "packed_slice_locations": {}
}
diff --git a/cloud/src/common/bvars.cpp b/cloud/src/common/bvars.cpp
index 11f63234ffb..e35e5e677f6 100644
--- a/cloud/src/common/bvars.cpp
+++ b/cloud/src/common/bvars.cpp
@@ -107,6 +107,7 @@ BvarLatencyRecorderWithTag g_bvar_ms_abort_snapshot("ms", "abort_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_drop_snapshot("ms", "drop_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot("ms", "list_snapshot");
BvarLatencyRecorderWithTag g_bvar_ms_clone_instance("ms", "clone_instance");
+BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info("ms", "update_packed_file_info");
bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
bvar::Window<bvar::Adder<int64_t> > g_bvar_update_delete_bitmap_fail_counter_minute("ms", "update_delete_bitmap_fail", &g_bvar_update_delete_bitmap_fail_counter, 60);
bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
@@ -163,6 +164,22 @@ mBvarIntAdder g_bvar_recycler_instance_recycle_round("recycler_instance_recycle_
mBvarStatus<double> g_bvar_recycler_instance_recycle_time_per_resource("recycler_instance_recycle_time_per_resource", {"instance_id", "resource_type"});
// represents the bytes of resources that can be recycled per ms
mBvarStatus<double> g_bvar_recycler_instance_recycle_bytes_per_ms("recycler_instance_recycle_bytes_per_ms", {"instance_id", "resource_type"});
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_num("recycler",
+ "packed_file_recycled_kv_num");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_bytes(
+ "recycler", "packed_file_recycled_kv_bytes");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycle_cost_ms(
+ "recycler", "packed_file_recycle_cost_ms");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_scanned_kv_num(
+ "recycler", "packed_file_scanned_kv_num");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_corrected_kv_num(
+ "recycler", "packed_file_corrected_kv_num");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_object_num(
+ "recycler", "packed_file_recycled_object_num");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_bytes_object_deleted(
+ "recycler", "packed_file_bytes_object_deleted");
+BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_rowset_scanned_num(
+ "recycler", "packed_file_rowset_scanned_num");
BvarStatusWithTag<int64_t> g_bvar_recycler_batch_delete_rowset_plan_count(
"recycler", "batch_delete_rowset_plan_count");
diff --git a/cloud/src/common/bvars.h b/cloud/src/common/bvars.h
index bda68cd1a0b..25e324b3233 100644
--- a/cloud/src/common/bvars.h
+++ b/cloud/src/common/bvars.h
@@ -619,6 +619,7 @@ extern BvarLatencyRecorderWithTag g_bvar_ms_abort_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_drop_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_list_snapshot;
extern BvarLatencyRecorderWithTag g_bvar_ms_clone_instance;
+extern BvarLatencyRecorderWithTag g_bvar_ms_update_packed_file_info;
extern bvar::Adder<int64_t> g_bvar_update_delete_bitmap_fail_counter;
extern bvar::Adder<int64_t> g_bvar_get_delete_bitmap_fail_counter;
extern BvarLatencyRecorderWithStatus<60> g_bvar_ms_txn_commit_with_tablet_count;
@@ -660,6 +661,14 @@ extern mBvarInt64Adder g_bvar_recycler_instance_recycle_total_bytes_since_starte
extern mBvarIntAdder g_bvar_recycler_instance_recycle_round;
extern mBvarStatus<double> g_bvar_recycler_instance_recycle_time_per_resource;
extern mBvarStatus<double> g_bvar_recycler_instance_recycle_bytes_per_ms;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_num;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_kv_bytes;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycle_cost_ms;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_scanned_kv_num;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_corrected_kv_num;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_recycled_object_num;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_bytes_object_deleted;
+extern BvarStatusWithTag<int64_t> g_bvar_recycler_packed_file_rowset_scanned_num;
extern BvarStatusWithTag<int64_t> g_bvar_recycler_batch_delete_rowset_plan_count;
extern BvarStatusWithTag<int64_t> g_bvar_recycler_batch_delete_failures;
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 7171b9e2afe..49db0e967b9 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -83,6 +83,8 @@ CONF_String(custom_conf_path, "");
// recycler config
CONF_mInt64(recycle_interval_seconds, "3600");
CONF_mInt64(retention_seconds, "259200"); // 72h, global retention time
+CONF_mInt64(packed_file_correction_delay_seconds,
+ "259200"); // seconds to wait before correcting packed files
CONF_Int32(recycle_concurrency, "16");
CONF_mInt32(recycle_job_lease_expired_ms, "60000");
CONF_mInt64(compacted_rowset_retention_seconds, "1800"); // 0.5h
@@ -117,6 +119,12 @@ CONF_mInt64(check_recycle_task_interval_seconds, "600"); // 10min
CONF_mInt64(recycler_sleep_before_scheduling_seconds, "60");
// log a warning if a recycle task takes longer than this duration
CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h
+CONF_mInt32(decrement_packed_file_ref_counts_retry_times, "10");
+CONF_mInt32(packed_file_txn_retry_times, "10");
+// randomized interval to reduce conflict storms in FoundationDB, default 5-50ms
+CONF_mInt64(packed_file_txn_retry_sleep_min_ms, "5");
+CONF_mInt64(packed_file_txn_retry_sleep_max_ms, "50");
+CONF_mInt32(recycle_txn_delete_max_retry_times, "10");
// force recycler to recycle all useless object.
// **just for TEST**
@@ -137,6 +145,7 @@ CONF_mBool(enable_version_key_check, "false");
CONF_mBool(enable_meta_rowset_key_check, "false");
CONF_mBool(enable_snapshot_check, "false");
CONF_mBool(enable_mvcc_meta_key_check, "false");
+CONF_mBool(enable_packed_file_check, "false");
CONF_mBool(enable_mvcc_meta_check, "false");
CONF_mInt64(mow_job_key_check_expiration_diff_seconds, "600"); // 10min
diff --git a/cloud/src/meta-service/meta_service.cpp b/cloud/src/meta-service/meta_service.cpp
index 9a4533288f0..129d1cd4e2d 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -5704,6 +5704,80 @@ void MetaServiceImpl::get_schema_dict(::google::protobuf::RpcController* control
response->mutable_schema_dict()->Swap(&schema_dict);
}
+void MetaServiceImpl::update_packed_file_info(::google::protobuf::RpcController* controller,
+ const UpdatePackedFileInfoRequest* request,
+ UpdatePackedFileInfoResponse* response,
+ ::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_packed_file_info);
+
+ // Validate request parameters
+ if (!request->has_cloud_unique_id() || request->cloud_unique_id().empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud_unique_id is required";
+ return;
+ }
+
+ if (!request->has_packed_file_path() || request->packed_file_path().empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "packed_file_path is required";
+ return;
+ }
+
+ if (!request->has_packed_file_info()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "packed_file_info is required";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, request->cloud_unique_id());
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(INFO) << msg << ", cloud_unique_id=" << request->cloud_unique_id();
+ return;
+ }
+
+ RPC_RATE_LIMIT(update_packed_file_info)
+
+ // Create transaction
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = MetaServiceCode::KV_TXN_CREATE_ERR;
+ msg = fmt::format("failed to create txn, err={}", err);
+ return;
+ }
+
+ // Generate packed file key
+ PackedFileKeyInfo key_info {instance_id, request->packed_file_path()};
+ std::string packed_file_key_str = packed_file_key(key_info);
+
+ // Serialize packed file info
+ std::string packed_file_info_val;
+ if (!request->packed_file_info().SerializeToString(&packed_file_info_val)) {
+ code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
+ msg = "failed to serialize packed_file_info";
+ return;
+ }
+
+ // Put packed file info
+ txn->put(packed_file_key_str, packed_file_info_val);
+
+ // Commit transaction
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = MetaServiceCode::KV_TXN_COMMIT_ERR;
+ msg = fmt::format("failed to commit txn, err={}", err);
+ return;
+ }
+
+ LOG(INFO) << "successfully updated packed file info"
+ << ", instance_id=" << instance_id
+ << ", packed_file_path=" << request->packed_file_path()
+ << ", total_slice_num=" << request->packed_file_info().total_slice_num()
+ << ", ref_cnt=" << request->packed_file_info().ref_cnt()
+ << ", key=" << hex(packed_file_key_str);
+}
+
std::string hide_access_key(const std::string& ak) {
std::string key = ak;
size_t key_len = key.length();
diff --git a/cloud/src/meta-service/meta_service.h b/cloud/src/meta-service/meta_service.h
index 36e9113b77d..44027ed6316 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -167,6 +167,11 @@ public:
const CreateRowsetRequest* request, CreateRowsetResponse* response,
::google::protobuf::Closure* done) override;
+ void update_packed_file_info(::google::protobuf::RpcController* controller,
+ const UpdatePackedFileInfoRequest* request,
+ UpdatePackedFileInfoResponse* response,
+ ::google::protobuf::Closure* done) override;
+
void get_rowset(::google::protobuf::RpcController* controller, const GetRowsetRequest* request,
GetRowsetResponse* response, ::google::protobuf::Closure* done) override;
@@ -622,6 +627,14 @@ public:
call_impl(&cloud::MetaService::update_tmp_rowset, controller, request, response, done);
}
+ void update_packed_file_info(::google::protobuf::RpcController* controller,
+ const UpdatePackedFileInfoRequest* request,
+ UpdatePackedFileInfoResponse* response,
+ ::google::protobuf::Closure* done) override {
+ call_impl(&cloud::MetaService::update_packed_file_info, controller, request, response,
+ done);
+ }
+
void get_rowset(::google::protobuf::RpcController* controller, const GetRowsetRequest* request,
GetRowsetResponse* response, ::google::protobuf::Closure* done) override {
call_impl(&cloud::MetaService::get_rowset, controller, request, response, done);
diff --git a/cloud/src/meta-store/keys.cpp b/cloud/src/meta-store/keys.cpp
index 4c0a746f947..88073ae3a00 100644
--- a/cloud/src/meta-store/keys.cpp
+++ b/cloud/src/meta-store/keys.cpp
@@ -56,6 +56,7 @@ static const char* META_KEY_INFIX_DELETE_BITMAP_LOCK = "delete_bitmap_lock";
static const char* META_KEY_INFIX_DELETE_BITMAP_PENDING = "delete_bitmap_pending";
static const char* META_KEY_INFIX_MOW_TABLET_JOB = "mow_tablet_job";
static const char* META_KEY_INFIX_SCHEMA_DICTIONARY = "tablet_schema_pb_dict";
+static const char* META_KEY_INFIX_PACKED_FILE = "packed_file";
static const char* RECYCLE_KEY_INFIX_INDEX = "index";
static const char* RECYCLE_KEY_INFIX_PART = "partition";
@@ -151,7 +152,7 @@ static void encode_prefix(const T& t, std::string* key) {
JobTabletKeyInfo, JobRecycleKeyInfo, JobSnapshotDataMigratorKeyInfo, JobSnapshotChainCompactorKeyInfo,
RLJobProgressKeyInfo, StreamingJobKeyInfo,
CopyJobKeyInfo, CopyFileKeyInfo, StorageVaultKeyInfo, MetaSchemaPBDictionaryInfo,
- MowTabletJobInfo>);
+ MowTabletJobInfo, PackedFileKeyInfo>);
key->push_back(CLOUD_USER_KEY_SPACE01);
// Prefixes for key families
@@ -171,7 +172,8 @@ static void encode_prefix(const T& t, std::string* key) {
|| std::is_same_v<T, MetaDeleteBitmapInfo>
|| std::is_same_v<T, MetaDeleteBitmapUpdateLockInfo>
|| std::is_same_v<T, MetaPendingDeleteBitmapInfo>
- || std::is_same_v<T, MowTabletJobInfo>) {
+ || std::is_same_v<T, MowTabletJobInfo>
+ || std::is_same_v<T, PackedFileKeyInfo>) {
encode_bytes(META_KEY_PREFIX, key);
} else if constexpr (std::is_same_v<T, PartitionVersionKeyInfo>
|| std::is_same_v<T, TableVersionKeyInfo>) {
@@ -349,6 +351,12 @@ void mow_tablet_job_key(const MowTabletJobInfo& in, std::string* out) {
encode_int64(std::get<2>(in), out); // initiator
}
+void packed_file_key(const PackedFileKeyInfo& in, std::string* out) {
+ encode_prefix(in, out); // 0x01 "meta" ${instance_id}
+ encode_bytes(META_KEY_INFIX_PACKED_FILE, out); // "packed_file"
+ encode_bytes(std::get<1>(in), out); // packed_file_path
+}
+
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in, std::string* out) {
encode_prefix(in, out); // 0x01 "meta" ${instance_id}
encode_bytes(META_KEY_INFIX_DELETE_BITMAP_PENDING, out); // "delete_bitmap_pending"
diff --git a/cloud/src/meta-store/keys.h b/cloud/src/meta-store/keys.h
index 2f4e29e310c..53a24939cde 100644
--- a/cloud/src/meta-store/keys.h
+++ b/cloud/src/meta-store/keys.h
@@ -48,6 +48,7 @@
// 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id} ${version} ${segment_id} -> roaringbitmap
// 0x01 "meta" ${instance_id} "tablet_schema_pb_dict" ${index_id} -> SchemaCloudDictionary
// 0x01 "meta" ${instance_id} "mow_tablet_job" ${table_id} ${initiator_id} -> MowTabletJobPB
+// 0x01 "meta" ${instance_id} "packed_file" ${packed_file_path} -> PackedFileInfoPB
//
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} -> TabletStatsPB
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id} ${partition_id} ${tablet_id} "data_size" -> int64
@@ -241,6 +242,9 @@ using MetaSchemaPBDictionaryInfo = BasicKeyInfo<__LINE__ , std::tuple<std::strin
// 0:instance_id 1:table_id 2:initiator
using MowTabletJobInfo = BasicKeyInfo<__LINE__ , std::tuple<std::string, int64_t, int64_t>>;
+// 0:instance_id 1:packed_file_path
+using PackedFileKeyInfo = BasicKeyInfo<30, std::tuple<std::string, std::string>>;
+
namespace versioned {
// ATTN: Key info definitions in this namespace do not include timestamp and subsequent attributes.
@@ -374,6 +378,7 @@ void meta_delete_bitmap_update_lock_key(const MetaDeleteBitmapUpdateLockInfo& in
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in, std::string* out);
void meta_schema_pb_dictionary_key(const MetaSchemaPBDictionaryInfo& in, std::string* out);
void mow_tablet_job_key(const MowTabletJobInfo& in, std::string* out);
+void packed_file_key(const PackedFileKeyInfo& in, std::string* out);
static inline std::string meta_rowset_key(const MetaRowsetKeyInfo& in) { std::string s; meta_rowset_key(in, &s); return s; }
static inline std::string meta_rowset_tmp_key(const MetaRowsetTmpKeyInfo& in) { std::string s; meta_rowset_tmp_key(in, &s); return s; }
static inline std::string meta_tablet_idx_key(const MetaTabletIdxKeyInfo& in) { std::string s; meta_tablet_idx_key(in, &s); return s; }
@@ -384,6 +389,11 @@ static inline std::string meta_delete_bitmap_update_lock_key(const MetaDeleteBit
static inline std::string meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in) { std::string s; meta_pending_delete_bitmap_key(in, &s); return s; }
static inline std::string meta_schema_pb_dictionary_key(const MetaSchemaPBDictionaryInfo& in) { std::string s; meta_schema_pb_dictionary_key(in, &s); return s; }
static inline std::string mow_tablet_job_key(const MowTabletJobInfo& in) { std::string s; mow_tablet_job_key(in, &s); return s; }
+static inline std::string packed_file_key(const PackedFileKeyInfo& in) {
+ std::string s;
+ packed_file_key(in, &s);
+ return s;
+}
std::string recycle_key_prefix(std::string_view instance_id);
void recycle_index_key(const RecycleIndexKeyInfo& in, std::string* out);
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index 80eaed3d989..51e6f38fc1e 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -28,6 +28,7 @@
#include <algorithm>
#include <chrono>
+#include <climits>
#include <cstdint>
#include <functional>
#include <memory>
@@ -35,6 +36,7 @@
#include <numeric>
#include <sstream>
#include <string_view>
+#include <unordered_map>
#include <unordered_set>
#include <vector>
@@ -60,6 +62,7 @@
#ifdef UNIT_TEST
#include "../test/mock_accessor.h"
#endif
+#include "recycler/recycler.h"
#include "recycler/util.h"
namespace doris::cloud {
@@ -156,7 +159,11 @@ int Checker::start() {
{
std::lock_guard lock(mtx_);
// skip instance in recycling
- if (working_instance_map_.count(instance_id)) continue;
+ if (working_instance_map_.count(instance_id)) {
+ LOG(INFO) << "checker skip instance already working, instance_id="
+ << instance_id;
+ continue;
+ }
}
auto checker = std::make_shared<InstanceChecker>(txn_kv_, instance.instance_id());
if (checker->init(instance) != 0) {
@@ -166,10 +173,14 @@ int Checker::start() {
}
std::string check_job_key;
job_check_key({instance.instance_id()}, &check_job_key);
+ LOG(INFO) << "checker picked instance, instance_id=" << instance.instance_id()
+ << " enqueue_time_s=" << enqueue_time_s;
int ret = prepare_instance_recycle_job(txn_kv_.get(), check_job_key,
instance.instance_id(), ip_port_,
config::check_object_interval_seconds * 1000);
if (ret != 0) { // Prepare failed
+ LOG(WARNING) << "checker prepare job failed, instance_id=" << instance.instance_id()
+ << " ret=" << ret;
continue;
} else {
std::lock_guard lock(mtx_);
@@ -182,54 +193,66 @@ int Checker::start() {
g_bvar_checker_enqueue_cost_s.put(instance_id, ctime_ms / 1000 - enqueue_time_s);
bool success {true};
+ auto log_progress = [&](std::string_view stage) {
+ LOG(INFO) << "checker progress, instance_id=" << instance_id << " stage=" << stage;
+ };
+ log_progress("do_check");
if (int ret = checker->do_check(); ret != 0) {
success = false;
}
if (config::enable_inverted_check) {
+ log_progress("do_inverted_check");
if (int ret = checker->do_inverted_check(); ret != 0) {
success = false;
}
}
if (config::enable_delete_bitmap_inverted_check) {
+ log_progress("do_delete_bitmap_inverted_check");
if (int ret = checker->do_delete_bitmap_inverted_check(); ret != 0) {
success = false;
}
}
if (config::enable_mow_job_key_check) {
+ log_progress("do_mow_job_key_check");
if (int ret = checker->do_mow_job_key_check(); ret != 0) {
success = false;
}
}
if (config::enable_tablet_stats_key_check) {
+ log_progress("do_tablet_stats_key_check");
if (int ret = checker->do_tablet_stats_key_check(); ret != 0) {
success = false;
}
}
if (config::enable_restore_job_check) {
+ log_progress("do_restore_job_check");
if (int ret = checker->do_restore_job_check(); ret != 0) {
success = false;
}
}
if (config::enable_txn_key_check) {
+ log_progress("do_txn_key_check");
if (int ret = checker->do_txn_key_check(); ret != 0) {
success = false;
}
}
if (config::enable_meta_rowset_key_check) {
+ log_progress("do_meta_rowset_key_check");
if (int ret = checker->do_meta_rowset_key_check(); ret != 0) {
success = false;
}
}
if (config::enable_delete_bitmap_storage_optimize_v2_check) {
+ log_progress("do_delete_bitmap_storage_optimize_check_v2");
if (int ret = checker->do_delete_bitmap_storage_optimize_check(2 /*version*/);
ret != 0) {
success = false;
@@ -237,28 +260,40 @@ int Checker::start() {
}
if (config::enable_version_key_check) {
+ log_progress("do_version_key_check");
if (int ret = checker->do_version_key_check(); ret != 0) {
success = false;
}
}
if (config::enable_snapshot_check) {
+ log_progress("do_snapshots_check");
if (int ret = checker->do_snapshots_check(); ret != 0) {
success = false;
}
}
if (config::enable_mvcc_meta_key_check) {
+ log_progress("do_mvcc_meta_key_check");
if (int ret = checker->do_mvcc_meta_key_check(); ret != 0) {
success = false;
}
}
+ if (config::enable_packed_file_check) {
+ log_progress("do_packed_file_check");
+ if (int ret = checker->do_packed_file_check(); ret != 0) {
+ success = false;
+ }
+ }
+
// If instance checker has been aborted, don't finish this job
if (!checker->stopped()) {
finish_instance_recycle_job(txn_kv_.get(), check_job_key, instance.instance_id(),
ip_port_, success, ctime_ms);
}
+ LOG(INFO) << "checker finished instance, instance_id=" << instance.instance_id()
+ << " success=" << success;
{
std::lock_guard lock(mtx_);
working_instance_map_.erase(instance.instance_id());
@@ -618,6 +653,12 @@ int InstanceChecker::do_check() {
for (int i = 0; i < rs_meta.num_segments(); ++i) {
auto path = segment_path(rs_meta.tablet_id(), rs_meta.rowset_id_v2(), i);
+ // Skip check if segment is already packed into a larger file
+ const auto& index_map = rs_meta.packed_slice_locations();
+ if (index_map.find(path) != index_map.end()) {
+ continue;
+ }
+
if (tablet_files_cache.files.contains(path)) {
continue;
}
@@ -672,6 +713,7 @@ int InstanceChecker::do_check() {
}
}
if (!index_ids.empty()) {
+ const auto& index_map = rs_meta.packed_slice_locations();
for (int i = 0; i < rs_meta.num_segments(); ++i) {
std::vector<std::string> index_path_v;
if (rs_meta.tablet_schema().inverted_index_storage_format() ==
@@ -691,6 +733,10 @@ int InstanceChecker::do_check() {
}
if (std::ranges::all_of(index_path_v, [&](const auto& idx_file_path) {
+ // Skip check if inverted index file is already packed into a larger file
+ if (index_map.find(idx_file_path) != index_map.end()) {
+ return true;
+ }
if (!tablet_files_cache.files.contains(idx_file_path)) {
LOG(INFO) << "loss index file: " << idx_file_path;
return false;
@@ -946,21 +992,25 @@ int InstanceChecker::do_inverted_check() {
}
for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) {
+ const auto& path = file->path;
+ if (path == "data/packed_file" || path.starts_with("data/packed_file/")) {
+ continue; // packed_file has dedicated check logic
+ }
++num_scanned;
- int ret = check_segment_file(file->path);
+ int ret = check_segment_file(path);
if (ret != 0) {
LOG(WARNING) << "failed to check segment file, uri=" << accessor->uri()
- << " path=" << file->path;
+ << " path=" << path;
if (ret == 1) {
++num_file_leak;
} else {
check_ret = -1;
}
}
- ret = check_inverted_index_file(file->path);
+ ret = check_inverted_index_file(path);
if (ret != 0) {
LOG(WARNING) << "failed to check index file, uri=" << accessor->uri()
- << " path=" << file->path;
+ << " path=" << path;
if (ret == 1) {
++num_file_leak;
} else {
@@ -2671,4 +2721,362 @@ void InstanceChecker::get_all_accessor(std::vector<StorageVaultAccessor*>* acces
accessors->push_back(accessor.get());
}
}
+
+int InstanceChecker::do_packed_file_check() {
+ LOG(INFO) << "begin to check packed files, instance_id=" << instance_id_;
+ int check_ret = 0;
+ long num_scanned_rowsets = 0;
+ long num_scanned_packed_files = 0;
+ long num_packed_file_loss = 0;
+ long num_packed_file_leak = 0;
+ long num_ref_count_mismatch = 0;
+ long num_small_file_ref_mismatch = 0;
+ using namespace std::chrono;
+ auto start_time = steady_clock::now();
+ DORIS_CLOUD_DEFER {
+ auto cost = duration<float>(steady_clock::now() - start_time).count();
+ LOG(INFO) << "check packed files finished, cost=" << cost
+ << "s. instance_id=" << instance_id_
+ << " num_scanned_rowsets=" << num_scanned_rowsets
+ << " num_scanned_packed_files=" << num_scanned_packed_files
+ << " num_packed_file_loss=" << num_packed_file_loss
+ << " num_packed_file_leak=" << num_packed_file_leak
+ << " num_ref_count_mismatch=" << num_ref_count_mismatch
+ << " num_small_file_ref_mismatch=" << num_small_file_ref_mismatch;
+ };
+
+ // Map to track expected reference count for each packed file
+ // packed_file_path -> expected_ref_count (from rowset metas)
+ std::unordered_map<std::string, int64_t> expected_ref_counts;
+ // Map to track small files referenced in packed files
+ // packed_file_path -> set of small_file_paths
+ std::unordered_map<std::string, std::unordered_set<std::string>> packed_file_small_files;
+
+ // Step 1: Scan all rowset metas to collect packed_slice_locations references
+ // Use efficient range scan instead of iterating through each tablet_id
+ auto collect_packed_refs = [&](const doris::RowsetMetaCloudPB& rs_meta) {
+ const auto& index_map = rs_meta.packed_slice_locations();
+ for (const auto& [small_file_path, index_pb] : index_map) {
+ if (!index_pb.has_packed_file_path() || index_pb.packed_file_path().empty()) {
+ continue;
+ }
+ const std::string& packed_file_path = index_pb.packed_file_path();
+ expected_ref_counts[packed_file_path]++;
+ packed_file_small_files[packed_file_path].insert(small_file_path);
+ }
+ };
+
+ {
+ std::string start_key = meta_rowset_key({instance_id_, 0, 0});
+ std::string end_key = meta_rowset_key({instance_id_, INT64_MAX, 0});
+
+ std::unique_ptr<RangeGetIterator> it;
+ do {
+ if (stopped()) {
+ return -1;
+ }
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn for packed file check";
+ return -1;
+ }
+
+ err = txn->get(start_key, end_key, &it);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to scan rowset metas, err=" << err;
+ check_ret = -1;
+ break;
+ }
+
+ while (it->has_next() && !stopped()) {
+ auto [k, v] = it->next();
+ if (!it->has_next()) {
+ start_key = k;
+ }
+
+ doris::RowsetMetaCloudPB rs_meta;
+ if (!rs_meta.ParseFromArray(v.data(), v.size())) {
+ LOG(WARNING) << "malformed rowset meta, key=" << hex(k);
+ check_ret = -1;
+ continue;
+ }
+
+ num_scanned_rowsets++;
+
+ collect_packed_refs(rs_meta);
+ }
+ start_key.push_back('\x00'); // Update to next smallest key for iteration
+ } while (it->more() && !stopped());
+ }
+
+ // Rowsets in recycle keys may still hold packed file references while ref count
+ // updates are pending, so include them when calculating expected references.
+ {
+ std::string start_key = recycle_rowset_key({instance_id_, 0, ""});
+ std::string end_key = recycle_rowset_key({instance_id_, INT64_MAX, "\xff"});
+
+ std::unique_ptr<RangeGetIterator> it;
+ do {
+ if (stopped()) {
+ return -1;
+ }
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn for recycle rowset scan in packed file check";
+ return -1;
+ }
+
+ err = txn->get(start_key, end_key, &it);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to scan recycle rowset metas, err=" << err;
+ check_ret = -1;
+ break;
+ }
+
+ while (it->has_next() && !stopped()) {
+ auto [k, v] = it->next();
+ if (!it->has_next()) {
+ start_key = k;
+ }
+
+ RecycleRowsetPB recycle_rowset;
+ if (!recycle_rowset.ParseFromArray(v.data(), v.size())) {
+ LOG(WARNING) << "malformed recycle rowset, key=" << hex(k);
+ check_ret = -1;
+ continue;
+ }
+
+ if (!recycle_rowset.has_rowset_meta()) {
+ continue;
+ }
+
+ num_scanned_rowsets++;
+ collect_packed_refs(recycle_rowset.rowset_meta());
+ }
+ start_key.push_back('\x00'); // Update to next smallest key for iteration
+ } while (it->more() && !stopped());
+ }
+
+ // Step 2: Scan all packed file metadata and verify
+ // Also collect all packed file paths from metadata for Step 3
+ // Map: resource_id -> set of packed_file_paths
+ std::unordered_map<std::string, std::unordered_set<std::string>> packed_files_in_metadata;
+ std::string begin = packed_file_key({instance_id_, ""});
+ std::string end = packed_file_key({instance_id_, "\xff"});
+ std::string scan_begin = begin;
+
+ while (true) {
+ if (stopped()) {
+ return -1;
+ }
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn for scanning packed files";
+ return -1;
+ }
+
+ std::unique_ptr<RangeGetIterator> it;
+ err = txn->get(scan_begin, end, &it);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to scan packed file keys, err=" << err;
+ return -1;
+ }
+ if (!it->has_next()) {
+ break;
+ }
+
+ std::string last_key;
+ while (it->has_next()) {
+ auto [k, v] = it->next();
+ last_key.assign(k.data(), k.size());
+ num_scanned_packed_files++;
+
+ std::string packed_file_path;
+ if (!InstanceRecycler::decode_packed_file_key(k, &packed_file_path)) {
+ LOG(WARNING) << "failed to decode packed file key, key=" << hex(k);
+ check_ret = -1;
+ continue;
+ }
+
+ cloud::PackedFileInfoPB packed_info;
+ if (!packed_info.ParseFromArray(v.data(), v.size())) {
+ LOG(WARNING) << "failed to parse packed file info, packed_file_path="
+ << packed_file_path;
+ check_ret = -1;
+ continue;
+ }
+
+ // Step 2.1: Verify packed file exists in storage
+ if (!packed_info.resource_id().empty()) {
+ // Collect packed file path for Step 3
+ packed_files_in_metadata[packed_info.resource_id()].insert(packed_file_path);
+
+ auto* accessor = get_accessor(packed_info.resource_id());
+ if (accessor == nullptr) {
+ LOG(WARNING) << "accessor not found for packed file, resource_id="
+ << packed_info.resource_id()
+ << ", packed_file_path=" << packed_file_path;
+ check_ret = -1;
+ continue;
+ }
+
+ int ret = accessor->exists(packed_file_path);
+ if (ret < 0) {
+ LOG(WARNING) << "failed to check packed file existence, packed_file_path="
+ << packed_file_path << ", ret=" << ret;
+ check_ret = -1;
+ continue;
+ }
+
+ if (ret != 0) {
+ // ret == 1 means file not found, ret > 1 means other error
+ // When packed file doesn't exist in storage, ref_cnt must be 0 and state must be RECYCLING
+ bool ref_cnt_valid = (packed_info.ref_cnt() == 0);
+ bool state_valid = (packed_info.state() == cloud::PackedFileInfoPB::RECYCLING);
+ if (!ref_cnt_valid || !state_valid) {
+ LOG(WARNING) << "packed file not found in storage but metadata is invalid, "
+ "packed_file_path="
+ << packed_file_path << ", ref_cnt=" << packed_info.ref_cnt()
+ << " (expected=0), state=" << packed_info.state()
+ << " (expected=RECYCLING), ret=" << ret;
+ num_packed_file_loss++;
+ check_ret = 1; // Data inconsistency identified
+ }
+ // If ref_cnt == 0 and state == RECYCLING, this is expected (file is being recycled)
+ }
+ // ret == 0 means file exists, which is expected
+ }
+
+ // Step 2.2: Verify reference count matches expected count
+ int64_t expected_ref = expected_ref_counts[packed_file_path];
+ if (packed_info.ref_cnt() != expected_ref) {
+ LOG(WARNING) << "packed file ref count mismatch, packed_file_path="
+ << packed_file_path << ", expected=" << expected_ref
+ << ", actual=" << packed_info.ref_cnt();
+ num_ref_count_mismatch++;
+ check_ret = 1; // Data inconsistency identified
+ }
+
+ // Step 2.3: Verify small files in packed_info match rowset references
+ std::unordered_set<std::string> small_files_in_meta;
+ for (const auto& small_file : packed_info.slices()) {
+ if (!small_file.deleted()) {
+ small_files_in_meta.insert(small_file.path());
+ }
+ }
+
+ const auto& expected_small_files = packed_file_small_files[packed_file_path];
+ if (small_files_in_meta != expected_small_files) {
+ // Check for missing small files
+ for (const auto& expected_path : expected_small_files) {
+ if (small_files_in_meta.find(expected_path) == small_files_in_meta.end()) {
+ LOG(WARNING) << "small file missing in packed file info, packed_file_path="
+ << packed_file_path << ", small_file_path=" << expected_path;
+ num_small_file_ref_mismatch++;
+ check_ret = 1;
+ }
+ }
+ // Check for extra small files (may be deleted, so less critical)
+ for (const auto& meta_path : small_files_in_meta) {
+ if (expected_small_files.find(meta_path) == expected_small_files.end()) {
+ LOG(INFO) << "small file in packed file info not found in rowset metas, "
+ "may be deleted, packed_file_path="
+ << packed_file_path << ", small_file_path=" << meta_path;
+ }
+ }
+ }
+ }
+
+ if (!it->more()) {
+ break;
+ }
+ scan_begin = last_key;
+ scan_begin.push_back('\x00');
+ }
+
+ // Step 3: Check for leaked packed files (exist in storage but not in metadata)
+ // Scan all storage vaults to find packed files and verify they are in metadata
+ {
+ std::vector<StorageVaultAccessor*> accessors;
+ get_all_accessor(&accessors);
+
+ for (StorageVaultAccessor* accessor : accessors) {
+ if (stopped()) {
+ return -1;
+ }
+
+ // Find resource_id for this accessor
+ std::string resource_id;
+ for (const auto& [id, acc] : accessor_map_) {
+ if (acc.get() == accessor) {
+ resource_id = id;
+ break;
+ }
+ }
+
+ if (resource_id.empty()) {
+ continue;
+ }
+
+ // List all files under data/packed_file/ directory
+ std::unique_ptr<ListIterator> list_it;
+ int ret = accessor->list_directory("data/packed_file", &list_it);
+ if (ret != 0) {
+ // Directory may not exist, which is fine
+ if (ret < 0) {
+ LOG(WARNING) << "failed to list packed_file directory, resource_id="
+ << resource_id << ", ret=" << ret;
+ check_ret = -1;
+ }
+ continue;
+ }
+
+ const auto& expected_packed_files = packed_files_in_metadata[resource_id];
+ while (list_it->has_next()) {
+ if (stopped()) {
+ return -1;
+ }
+
+ auto file_meta = list_it->next();
+ if (!file_meta.has_value()) {
+ break;
+ }
+
+ const std::string& file_path = file_meta->path;
+ // Only check files (not directories), and ensure it's a packed file
+ // Skip directories (paths ending with '/') and non-packed-file paths
+ if (file_path.empty() || file_path.back() == '/' ||
+ !file_path.starts_with("data/packed_file/")) {
+ continue;
+ }
+
+ // Check if this packed file is in metadata
+ if (expected_packed_files.find(file_path) == expected_packed_files.end()) {
+ LOG(WARNING) << "packed file found in storage but not in metadata, "
+ "resource_id="
+ << resource_id << ", packed_file_path=" << file_path;
+ num_packed_file_leak++;
+ check_ret = 1; // Data leak identified
+ }
+ }
+ }
+ }
+
+ if (num_packed_file_loss > 0 || num_packed_file_leak > 0 || num_ref_count_mismatch > 0 ||
+ num_small_file_ref_mismatch > 0) {
+ return 1; // Data loss or inconsistency identified
+ }
+
+ if (check_ret < 0) {
+ return check_ret; // Temporary error
+ }
+
+ return 0; // Success
+}
} // namespace doris::cloud
diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h
index 015003dc5e9..ba26d07a70e 100644
--- a/cloud/src/recycler/checker.h
+++ b/cloud/src/recycler/checker.h
@@ -135,6 +135,11 @@ public:
// Return negative if a temporary error occurred during the check process.
int do_mvcc_meta_key_check();
+ // Return 0 if success.
+ // Return 1 if packed file metadata leak or loss is identified.
+ // Return negative if a temporary error occurred during the check process.
+ int do_packed_file_check();
+
StorageVaultAccessor* get_accessor(const std::string& id);
ResourceManager* resource_mgr() const { return resource_mgr_.get(); }
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index b44fc9a9439..7959a72efa3 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -32,12 +32,17 @@
#include <cstdint>
#include <cstdlib>
#include <deque>
+#include <functional>
#include <initializer_list>
#include <memory>
#include <numeric>
+#include <random>
#include <string>
#include <string_view>
+#include <thread>
+#include <unordered_map>
#include <utility>
+#include <variant>
#include "common/defer.h"
#include "common/stopwatch.h"
@@ -76,6 +81,22 @@ namespace doris::cloud {
using namespace std::chrono;
+namespace {
+
+int64_t packed_file_retry_sleep_ms() {
+ const int64_t min_ms = std::max<int64_t>(0, config::packed_file_txn_retry_sleep_min_ms);
+ const int64_t max_ms = std::max<int64_t>(min_ms, config::packed_file_txn_retry_sleep_max_ms);
+ thread_local std::mt19937_64 gen(std::random_device {}());
+ std::uniform_int_distribution<int64_t> dist(min_ms, max_ms);
+ return dist(gen);
+}
+
+void sleep_for_packed_file_retry() {
+ std::this_thread::sleep_for(std::chrono::milliseconds(packed_file_retry_sleep_ms()));
+}
+
+} // namespace
+
// return 0 for success get a key, 1 for key not found, negative for error
[[maybe_unused]] static int txn_get(TxnKv* txn_kv, std::string_view key, std::string& val) {
std::unique_ptr<Transaction> txn;
@@ -750,6 +771,8 @@ int InstanceRecycler::do_recycle() {
.add(task_wrapper(
[this]() -> int { return InstanceRecycler::recycle_tmp_rowsets(); }))
.add(task_wrapper([this]() -> int { return InstanceRecycler::recycle_rowsets(); }))
+ .add(task_wrapper(
+ [this]() -> int { return InstanceRecycler::recycle_packed_files(); }))
.add(task_wrapper(
[this]() { return InstanceRecycler::abort_timeout_txn(); },
[this]() { return InstanceRecycler::recycle_expired_txn_label(); }))
@@ -932,6 +955,549 @@ int InstanceRecycler::recycle_deleted_instance() {
return ret;
}
+int InstanceRecycler::check_rowset_exists(int64_t tablet_id, const std::string& rowset_id,
+ bool* exists, PackedFileRecycleStats* stats) {
+ if (exists == nullptr) {
+ return -1;
+ }
+ *exists = false;
+
+ std::string begin = meta_rowset_key({instance_id_, tablet_id, 0});
+ std::string end = meta_rowset_key({instance_id_, tablet_id + 1, 0});
+ std::string scan_begin = begin;
+
+ while (true) {
+ std::unique_ptr<RangeGetIterator> it_range;
+ int get_ret = txn_get(txn_kv_.get(), scan_begin, end, it_range);
+ if (get_ret < 0) {
+ LOG_WARNING("failed to scan rowset metas when recycling packed file")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("ret", get_ret);
+ return -1;
+ }
+ if (get_ret == 1 || it_range == nullptr || !it_range->has_next()) {
+ return 0;
+ }
+
+ std::string last_key;
+ while (it_range->has_next()) {
+ auto [k, v] = it_range->next();
+ last_key.assign(k.data(), k.size());
+ doris::RowsetMetaCloudPB rowset_meta;
+ if (!rowset_meta.ParseFromArray(v.data(), v.size())) {
+ LOG_WARNING("malformed rowset meta when checking packed file rowset existence")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("key", hex(k));
+ continue;
+ }
+ if (stats) {
+ ++stats->rowset_scan_count;
+ }
+ if (rowset_meta.rowset_id_v2() == rowset_id) {
+ *exists = true;
+ return 0;
+ }
+ }
+
+ if (!it_range->more()) {
+ return 0;
+ }
+
+ // Continue scanning from the next key to keep each transaction short.
+ scan_begin = std::move(last_key);
+ scan_begin.push_back('\x00');
+ }
+}
+
+int InstanceRecycler::check_recycle_and_tmp_rowset_exists(int64_t tablet_id,
+ const std::string& rowset_id,
+ int64_t txn_id, bool* recycle_exists,
+ bool* tmp_exists) {
+ if (recycle_exists == nullptr || tmp_exists == nullptr) {
+ return -1;
+ }
+ *recycle_exists = false;
+ *tmp_exists = false;
+
+ if (txn_id <= 0) {
+ LOG_WARNING("invalid txn id when checking recycle/tmp rowset existence")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("txn_id", txn_id);
+ return -1;
+ }
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to create txn when checking recycle/tmp rowset existence")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("txn_id", txn_id)
+ .tag("err", err);
+ return -1;
+ }
+
+ std::string recycle_key = recycle_rowset_key({instance_id_, tablet_id, rowset_id});
+ auto ret = key_exists(txn.get(), recycle_key, true);
+ if (ret == TxnErrorCode::TXN_OK) {
+ *recycle_exists = true;
+ } else if (ret != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ LOG_WARNING("failed to check recycle rowset existence")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("key", hex(recycle_key))
+ .tag("err", ret);
+ return -1;
+ }
+
+ std::string tmp_key = meta_rowset_tmp_key({instance_id_, txn_id, tablet_id});
+ ret = key_exists(txn.get(), tmp_key, true);
+ if (ret == TxnErrorCode::TXN_OK) {
+ *tmp_exists = true;
+ } else if (ret != TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ LOG_WARNING("failed to check tmp rowset existence")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("txn_id", txn_id)
+ .tag("key", hex(tmp_key))
+ .tag("err", ret);
+ return -1;
+ }
+
+ return 0;
+}
+
+std::pair<std::string, std::shared_ptr<StorageVaultAccessor>>
+InstanceRecycler::resolve_packed_file_accessor(const std::string& hint) {
+ if (!hint.empty()) {
+ if (auto it = accessor_map_.find(hint); it != accessor_map_.end()) {
+ return {hint, it->second};
+ }
+ }
+
+ return {"", nullptr};
+}
+
+int InstanceRecycler::correct_packed_file_info(cloud::PackedFileInfoPB* packed_info, bool* changed,
+ const std::string& packed_file_path,
+ PackedFileRecycleStats* stats) {
+ bool local_changed = false;
+ int64_t left_num = 0;
+ int64_t left_bytes = 0;
+ bool all_small_files_confirmed = true;
+ LOG(INFO) << "begin to correct file: " << packed_file_path;
+
+ auto log_small_file_status = [&](const cloud::PackedSlicePB& file, bool confirmed_this_round) {
+ int64_t tablet_id = file.has_tablet_id() ? file.tablet_id() : int64_t {-1};
+ std::string rowset_id = file.has_rowset_id() ? file.rowset_id() : std::string {};
+ int64_t txn_id = file.has_txn_id() ? file.txn_id() : int64_t {0};
+ LOG_INFO("packed slice correction status")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("small_file_path", file.path())
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("txn_id", txn_id)
+ .tag("size", file.size())
+ .tag("deleted", file.deleted())
+ .tag("corrected", file.corrected())
+ .tag("confirmed_this_round", confirmed_this_round);
+ };
+
+ for (int i = 0; i < packed_info->slices_size(); ++i) {
+ auto* small_file = packed_info->mutable_slices(i);
+ if (small_file->deleted()) {
+ log_small_file_status(*small_file, small_file->corrected());
+ continue;
+ }
+
+ if (small_file->corrected()) {
+ left_num++;
+ left_bytes += small_file->size();
+ log_small_file_status(*small_file, true);
+ continue;
+ }
+
+ if (!small_file->has_tablet_id() || !small_file->has_rowset_id()) {
+ LOG_WARNING("packed file small file missing identifiers during correction")
+ .tag("instance_id", instance_id_)
+ .tag("small_file_path", small_file->path())
+ .tag("index", i);
+ return -1;
+ }
+
+ int64_t tablet_id = small_file->tablet_id();
+ const std::string& rowset_id = small_file->rowset_id();
+ if (!small_file->has_txn_id() || small_file->txn_id() <= 0) {
+ LOG_WARNING("packed file small file missing valid txn id during correction")
+ .tag("instance_id", instance_id_)
+ .tag("small_file_path", small_file->path())
+ .tag("index", i)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("has_txn_id", small_file->has_txn_id())
+ .tag("txn_id", small_file->has_txn_id() ? small_file->txn_id() : 0);
+ return -1;
+ }
+ int64_t txn_id = small_file->txn_id();
+ bool recycle_exists = false;
+ bool tmp_exists = false;
+ if (check_recycle_and_tmp_rowset_exists(tablet_id, rowset_id, txn_id, &recycle_exists,
+ &tmp_exists) != 0) {
+ return -1;
+ }
+
+ bool small_file_confirmed = false;
+ if (tmp_exists) {
+ left_num++;
+ left_bytes += small_file->size();
+ small_file_confirmed = true;
+ } else if (recycle_exists) {
+ left_num++;
+ left_bytes += small_file->size();
+ // keep small_file_confirmed=false so the packed file remains uncorrected
+ } else {
+ bool rowset_exists = false;
+ if (check_rowset_exists(tablet_id, rowset_id, &rowset_exists, stats) != 0) {
+ return -1;
+ }
+
+ if (!rowset_exists) {
+ if (!small_file->deleted()) {
+ small_file->set_deleted(true);
+ local_changed = true;
+ }
+ if (!small_file->corrected()) {
+ small_file->set_corrected(true);
+ local_changed = true;
+ }
+ small_file_confirmed = true;
+ } else {
+ left_num++;
+ left_bytes += small_file->size();
+ small_file_confirmed = true;
+ }
+ }
+
+ if (!small_file_confirmed) {
+ all_small_files_confirmed = false;
+ }
+
+ if (small_file->corrected() != small_file_confirmed) {
+ small_file->set_corrected(small_file_confirmed);
+ local_changed = true;
+ }
+
+ log_small_file_status(*small_file, small_file_confirmed);
+ }
+
+ if (packed_info->remaining_slice_bytes() != left_bytes) {
+ packed_info->set_remaining_slice_bytes(left_bytes);
+ local_changed = true;
+ }
+ if (packed_info->ref_cnt() != left_num) {
+ auto old_ref_cnt = packed_info->ref_cnt();
+ packed_info->set_ref_cnt(left_num);
+ LOG_INFO("corrected packed file ref count")
+ .tag("instance_id", instance_id_)
+ .tag("resource_id", packed_info->resource_id())
+ .tag("packed_file_path", packed_file_path)
+ .tag("old_ref_cnt", old_ref_cnt)
+ .tag("new_ref_cnt", left_num);
+ local_changed = true;
+ }
+ if (packed_info->corrected() != all_small_files_confirmed) {
+ packed_info->set_corrected(all_small_files_confirmed);
+ local_changed = true;
+ }
+ if (left_num == 0 && packed_info->state() != cloud::PackedFileInfoPB::RECYCLING) {
+ packed_info->set_state(cloud::PackedFileInfoPB::RECYCLING);
+ local_changed = true;
+ }
+
+ if (changed != nullptr) {
+ *changed = local_changed;
+ }
+ return 0;
+}
+
+int InstanceRecycler::process_single_packed_file(const std::string& packed_key,
+ const std::string& packed_file_path,
+ PackedFileRecycleStats* stats) {
+ const int max_retry_times = std::max(1, config::packed_file_txn_retry_times);
+ bool correction_ok = false;
+ cloud::PackedFileInfoPB packed_info;
+
+ for (int attempt = 1; attempt <= max_retry_times; ++attempt) {
+ if (stopped()) {
+ LOG_WARNING("recycler stopped before processing packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to create txn when processing packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ std::string packed_val;
+ err = txn->get(packed_key, &packed_val);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ return 0;
+ }
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to get packed file kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ if (!packed_info.ParseFromString(packed_val)) {
+ LOG_WARNING("failed to parse packed file info")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+
+ int64_t now_sec = ::time(nullptr);
+ bool corrected = packed_info.corrected();
+ bool due = config::force_immediate_recycle ||
+ now_sec - packed_info.created_at_sec() >=
+ config::packed_file_correction_delay_seconds;
+
+ if (!corrected && due) {
+ bool changed = false;
+ if (correct_packed_file_info(&packed_info, &changed, packed_file_path, stats) != 0) {
+ LOG_WARNING("correct_packed_file_info failed")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+ if (changed) {
+ std::string updated;
+ if (!packed_info.SerializeToString(&updated)) {
+ LOG_WARNING("failed to serialize packed file info after correction")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+ txn->put(packed_key, updated);
+ err = txn->commit();
+ if (err == TxnErrorCode::TXN_OK) {
+ if (stats) {
+ ++stats->num_corrected;
+ }
+ } else {
+ if (err == TxnErrorCode::TXN_CONFLICT && attempt < max_retry_times) {
+ LOG_WARNING(
+ "failed to commit correction for packed file due to conflict, "
+ "retrying")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ sleep_for_packed_file_retry();
+ packed_info.Clear();
+ continue;
+ }
+ LOG_WARNING("failed to commit correction for packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+ }
+ }
+
+ correction_ok = true;
+ break;
+ }
+
+ if (!correction_ok) {
+ return -1;
+ }
+
+ if (!(packed_info.state() == cloud::PackedFileInfoPB::RECYCLING &&
+ packed_info.ref_cnt() == 0)) {
+ return 0;
+ }
+
+ if (!packed_info.has_resource_id() || packed_info.resource_id().empty()) {
+ LOG_WARNING("packed file missing resource id when recycling")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ return -1;
+ }
+ auto [resource_id, accessor] = resolve_packed_file_accessor(packed_info.resource_id());
+ if (!accessor) {
+ LOG_WARNING("no accessor available to delete packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", packed_info.resource_id());
+ return -1;
+ }
+ int del_ret = accessor->delete_file(packed_file_path);
+ if (del_ret != 0 && del_ret != 1) {
+ LOG_WARNING("failed to delete packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id)
+ .tag("ret", del_ret);
+ return -1;
+ }
+ if (del_ret == 1) {
+ LOG_INFO("packed file already removed")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id);
+ } else {
+ LOG_INFO("deleted packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id);
+ }
+
+ for (int del_attempt = 1; del_attempt <= max_retry_times; ++del_attempt) {
+ std::unique_ptr<Transaction> del_txn;
+ TxnErrorCode err = txn_kv_->create_txn(&del_txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to create txn when removing packed file kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ std::string latest_val;
+ err = del_txn->get(packed_key, &latest_val);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ return 0;
+ }
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to re-read packed file kv before removal")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ cloud::PackedFileInfoPB latest_info;
+ if (!latest_info.ParseFromString(latest_val)) {
+ LOG_WARNING("failed to parse packed file info before removal")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt);
+ return -1;
+ }
+
+ if (!(latest_info.state() == cloud::PackedFileInfoPB::RECYCLING &&
+ latest_info.ref_cnt() == 0)) {
+ LOG_INFO("packed file state changed before removal, skip deleting kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt);
+ return 0;
+ }
+
+ del_txn->remove(packed_key);
+ err = del_txn->commit();
+ if (err == TxnErrorCode::TXN_OK) {
+ if (stats) {
+ ++stats->num_deleted;
+ stats->bytes_deleted += static_cast<int64_t>(packed_key.size()) +
+ static_cast<int64_t>(latest_val.size());
+ if (del_ret == 0 || del_ret == 1) {
+ ++stats->num_object_deleted;
+ int64_t object_size = latest_info.total_slice_bytes();
+ if (object_size <= 0) {
+ object_size = packed_info.total_slice_bytes();
+ }
+ stats->bytes_object_deleted += object_size;
+ }
+ }
+ LOG_INFO("removed packed file metadata")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ return 0;
+ }
+ if (err == TxnErrorCode::TXN_CONFLICT) {
+ if (del_attempt >= max_retry_times) {
+ LOG_WARNING("failed to remove packed file kv due to conflict after max retry")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt);
+ return -1;
+ }
+ LOG_WARNING("failed to remove packed file kv due to conflict, retrying")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt);
+ sleep_for_packed_file_retry();
+ continue;
+ }
+ LOG_WARNING("failed to remove packed file kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("del_attempt", del_attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ return -1;
+}
+
+int InstanceRecycler::handle_packed_file_kv(std::string_view key, std::string_view /*value*/,
+ PackedFileRecycleStats* stats, int* ret) {
+ if (stats) {
+ ++stats->num_scanned;
+ }
+ std::string packed_file_path;
+ if (!decode_packed_file_key(key, &packed_file_path)) {
+ LOG_WARNING("failed to decode packed file key")
+ .tag("instance_id", instance_id_)
+ .tag("key", hex(key));
+ if (stats) {
+ ++stats->num_failed;
+ }
+ if (ret) {
+ *ret = -1;
+ }
+ return 0;
+ }
+
+ std::string packed_key(key);
+ int process_ret = process_single_packed_file(packed_key, packed_file_path, stats);
+ if (process_ret != 0) {
+ if (stats) {
+ ++stats->num_failed;
+ }
+ if (ret) {
+ *ret = -1;
+ }
+ }
+ return 0;
+}
bool is_txn_finished(std::shared_ptr<TxnKv> txn_kv, const std::string& instance_id,
int64_t txn_id) {
std::unique_ptr<Transaction> txn;
@@ -2162,6 +2728,11 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
int64_t num_segments = rs_meta_pb.num_segments();
if (num_segments <= 0) return 0;
+ std::vector<std::string> file_paths;
+ if (decrement_packed_file_ref_counts(rs_meta_pb) != 0) {
+ return -1;
+ }
+
// Process inverted indexes
std::vector<std::pair<int64_t, std::string>> index_ids;
// default format as v1.
@@ -2231,9 +2802,9 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
return -1;
}
auto& accessor = it->second;
+
int64_t tablet_id = rs_meta_pb.tablet_id();
const auto& rowset_id = rs_meta_pb.rowset_id_v2();
- std::vector<std::string> file_paths;
for (int64_t i = 0; i < num_segments; ++i) {
file_paths.push_back(segment_path(tablet_id, rowset_id, i));
if (index_format == InvertedIndexStorageFormatPB::V1) {
@@ -2252,6 +2823,361 @@ int InstanceRecycler::delete_rowset_data(const RowsetMetaCloudPB& rs_meta_pb) {
return accessor->delete_files(file_paths);
}
+int InstanceRecycler::decrement_packed_file_ref_counts(const doris::RowsetMetaCloudPB& rs_meta_pb) {
+ LOG_INFO("begin process_packed_file_location_index")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("index_map_size", rs_meta_pb.packed_slice_locations_size());
+ const auto& index_map = rs_meta_pb.packed_slice_locations();
+ if (index_map.empty()) {
+ LOG_INFO("skip merge file update: empty merge_file_segment_index")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2());
+ return 0;
+ }
+ struct PackedSmallFileInfo {
+ std::string small_file_path;
+ };
+ std::unordered_map<std::string, std::vector<PackedSmallFileInfo>> packed_file_updates;
+ packed_file_updates.reserve(index_map.size());
+ for (const auto& [small_path, index_pb] : index_map) {
+ if (!index_pb.has_packed_file_path() || index_pb.packed_file_path().empty()) {
+ continue;
+ }
+ packed_file_updates[index_pb.packed_file_path()].push_back(
+ PackedSmallFileInfo {small_path});
+ }
+ if (packed_file_updates.empty()) {
+ LOG_INFO("skip packed file update: no valid merge_file_path in merge_file_segment_index")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("index_map_size", index_map.size());
+ return 0;
+ }
+
+ const int max_retry_times = std::max(1, config::decrement_packed_file_ref_counts_retry_times);
+ int ret = 0;
+ for (auto& [packed_file_path, small_files] : packed_file_updates) {
+ if (small_files.empty()) {
+ continue;
+ }
+
+ bool success = false;
+ for (int attempt = 1; attempt <= max_retry_times; ++attempt) {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to create txn when updating packed file ref count")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("err", err);
+ ret = -1;
+ break;
+ }
+
+ std::string packed_key = packed_file_key({instance_id_, packed_file_path});
+ std::string packed_val;
+ err = txn->get(packed_key, &packed_val);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ LOG_WARNING("packed file info not found when recycling rowset")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("key", hex(packed_key))
+ .tag("tablet id", rs_meta_pb.tablet_id());
+ // Skip this packed file entry and continue with others
+ success = true;
+ break;
+ }
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to get packed file info when recycling rowset")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("err", err);
+ ret = -1;
+ break;
+ }
+
+ cloud::PackedFileInfoPB packed_info;
+ if (!packed_info.ParseFromString(packed_val)) {
+ LOG_WARNING("failed to parse packed file info when recycling rowset")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id());
+ ret = -1;
+ break;
+ }
+
+ LOG_INFO("packed file update check")
+ .tag("instance_id", instance_id_)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("merged_file_path", packed_file_path)
+ .tag("requested_small_files", small_files.size())
+ .tag("merge_entries", packed_info.slices_size());
+
+ auto* small_file_entries = packed_info.mutable_slices();
+ int64_t changed_files = 0;
+ int64_t missing_entries = 0;
+ int64_t already_deleted = 0;
+ for (const auto& small_file_info : small_files) {
+ bool found = false;
+ for (auto& small_file_entry : *small_file_entries) {
+ if (small_file_entry.path() == small_file_info.small_file_path) {
+ if (!small_file_entry.deleted()) {
+ small_file_entry.set_deleted(true);
+ if (!small_file_entry.corrected()) {
+ small_file_entry.set_corrected(true);
+ }
+ ++changed_files;
+ } else {
+ ++already_deleted;
+ }
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ ++missing_entries;
+ LOG_WARNING("packed file info missing small file entry")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("small_file_path", small_file_info.small_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id());
+ }
+ }
+
+ if (changed_files == 0) {
+ LOG_INFO("skip merge file update: no merge entries changed")
+ .tag("instance_id", instance_id_)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("merged_file_path", packed_file_path)
+ .tag("missing_entries", missing_entries)
+ .tag("already_deleted", already_deleted)
+ .tag("requested_small_files", small_files.size())
+ .tag("merge_entries", packed_info.slices_size());
+ success = true;
+ break;
+ }
+
+ int64_t left_file_count = 0;
+ int64_t left_file_bytes = 0;
+ for (const auto& small_file_entry : packed_info.slices()) {
+ if (!small_file_entry.deleted()) {
+ ++left_file_count;
+ left_file_bytes += small_file_entry.size();
+ }
+ }
+ packed_info.set_remaining_slice_bytes(left_file_bytes);
+ packed_info.set_ref_cnt(left_file_count);
+ LOG_INFO("updated packed file reference info")
+ .tag("instance_id", instance_id_)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("packed_file_path", packed_file_path)
+ .tag("ref_cnt", left_file_count)
+ .tag("left_file_bytes", left_file_bytes);
+
+ if (left_file_count == 0) {
+ packed_info.set_state(cloud::PackedFileInfoPB::RECYCLING);
+ }
+
+ std::string updated_val;
+ if (!packed_info.SerializeToString(&updated_val)) {
+ LOG_WARNING("failed to serialize packed file info when recycling rowset")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id());
+ ret = -1;
+ break;
+ }
+
+ txn->put(packed_key, updated_val);
+ err = txn->commit();
+ if (err == TxnErrorCode::TXN_OK) {
+ success = true;
+ if (left_file_count == 0) {
+ LOG_INFO("packed file ready to delete, deleting immediately")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ if (delete_packed_file_and_kv(packed_file_path, packed_key, packed_info) != 0) {
+ ret = -1;
+ }
+ }
+ break;
+ }
+ if (err == TxnErrorCode::TXN_CONFLICT) {
+ if (attempt >= max_retry_times) {
+ LOG_WARNING("packed file info update conflict after max retry")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("changed_files", changed_files)
+ .tag("attempt", attempt);
+ ret = -1;
+ break;
+ }
+ LOG_WARNING("packed file info update conflict, retrying")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("changed_files", changed_files)
+ .tag("attempt", attempt);
+ sleep_for_packed_file_retry();
+ continue;
+ }
+
+ LOG_WARNING("failed to commit packed file info update")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("rowset_id", rs_meta_pb.rowset_id_v2())
+ .tag("tablet_id", rs_meta_pb.tablet_id())
+ .tag("err", err)
+ .tag("changed_files", changed_files);
+ ret = -1;
+ break;
+ }
+
+ if (!success) {
+ ret = -1;
+ }
+ }
+
+ return ret;
+}
+
+int InstanceRecycler::delete_packed_file_and_kv(const std::string& packed_file_path,
+ const std::string& packed_key,
+ const cloud::PackedFileInfoPB& packed_info) {
+ if (!packed_info.has_resource_id() || packed_info.resource_id().empty()) {
+ LOG_WARNING("packed file missing resource id when recycling")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ return -1;
+ }
+
+ auto [resource_id, accessor] = resolve_packed_file_accessor(packed_info.resource_id());
+ if (!accessor) {
+ LOG_WARNING("no accessor available to delete packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", packed_info.resource_id());
+ return -1;
+ }
+
+ int del_ret = accessor->delete_file(packed_file_path);
+ if (del_ret != 0 && del_ret != 1) {
+ LOG_WARNING("failed to delete packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id)
+ .tag("ret", del_ret);
+ return -1;
+ }
+ if (del_ret == 1) {
+ LOG_INFO("packed file already removed")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id);
+ } else {
+ LOG_INFO("deleted packed file")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("resource_id", resource_id);
+ }
+
+ const int max_retry_times = std::max(1, config::packed_file_txn_retry_times);
+ for (int attempt = 1; attempt <= max_retry_times; ++attempt) {
+ std::unique_ptr<Transaction> del_txn;
+ TxnErrorCode err = txn_kv_->create_txn(&del_txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to create txn when removing packed file kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ std::string latest_val;
+ err = del_txn->get(packed_key, &latest_val);
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ return 0;
+ }
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG_WARNING("failed to re-read packed file kv before removal")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+
+ cloud::PackedFileInfoPB latest_info;
+ if (!latest_info.ParseFromString(latest_val)) {
+ LOG_WARNING("failed to parse packed file info before removal")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+
+ if (!(latest_info.state() == cloud::PackedFileInfoPB::RECYCLING &&
+ latest_info.ref_cnt() == 0)) {
+ LOG_INFO("packed file state changed before removal, skip deleting kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return 0;
+ }
+
+ del_txn->remove(packed_key);
+ err = del_txn->commit();
+ if (err == TxnErrorCode::TXN_OK) {
+ LOG_INFO("removed packed file metadata")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path);
+ return 0;
+ }
+ if (err == TxnErrorCode::TXN_CONFLICT) {
+ if (attempt >= max_retry_times) {
+ LOG_WARNING("failed to remove packed file kv due to conflict after max retry")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ return -1;
+ }
+ LOG_WARNING("failed to remove packed file kv due to conflict, retrying")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt);
+ sleep_for_packed_file_retry();
+ continue;
+ }
+ LOG_WARNING("failed to remove packed file kv")
+ .tag("instance_id", instance_id_)
+ .tag("packed_file_path", packed_file_path)
+ .tag("attempt", attempt)
+ .tag("err", err);
+ return -1;
+ }
+ return -1;
+}
+
int InstanceRecycler::delete_rowset_data(
const std::map<std::string, doris::RowsetMetaCloudPB>& rowsets, RowsetRecyclingState type,
RecyclerMetricsContext& metrics_context) {
@@ -2267,8 +3193,11 @@ int InstanceRecycler::delete_rowset_data(
// due to aborted schema change.
if (is_formal_rowset) {
std::lock_guard lock(recycled_tablets_mtx_);
- if (recycled_tablets_.count(rs.tablet_id())) {
- continue; // Rowset data has already been deleted
+ if (recycled_tablets_.count(rs.tablet_id()) && rs.packed_slice_locations_size() == 0) {
+ // Tablet has been recycled and this rowset has no packed slices, so file data
+ // should already be gone; skip to avoid redundant deletes. Rowsets with packed
+ // slice info must still run to decrement packed file ref counts.
+ continue;
}
}
@@ -2285,6 +3214,15 @@ int InstanceRecycler::delete_rowset_data(
auto& file_paths = resource_file_paths[rs.resource_id()];
const auto& rowset_id = rs.rowset_id_v2();
int64_t tablet_id = rs.tablet_id();
+ LOG_INFO("recycle rowset merge index size")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rowset_id)
+ .tag("merge_index_size", rs.packed_slice_locations_size());
+ if (decrement_packed_file_ref_counts(rs) != 0) {
+ ret = -1;
+ continue;
+ }
int64_t num_segments = rs.num_segments();
if (num_segments <= 0) {
metrics_context.total_recycled_num++;
@@ -2413,6 +3351,9 @@ int InstanceRecycler::delete_rowset_data(
if (auto pos = str.back().find('_'); pos != std::string::npos) {
rowset_id = str.back().substr(0, pos);
} else {
+ if (path.find("packed_file/") != std::string::npos) {
+ return; // packed files do not have rowset_id encoded
+ }
LOG(WARNING) << "failed to parse rowset_id, path=" << path;
return;
}
@@ -2482,6 +3423,78 @@ int InstanceRecycler::delete_rowset_data(const std::string& resource_id, int64_t
return accessor->delete_prefix(rowset_path_prefix(tablet_id, rowset_id));
}
+bool InstanceRecycler::decode_packed_file_key(std::string_view key, std::string* packed_path) {
+ if (key.empty()) {
+ return false;
+ }
+ std::string_view key_view = key;
+ key_view.remove_prefix(1); // remove keyspace prefix
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> decoded;
+ if (decode_key(&key_view, &decoded) != 0) {
+ return false;
+ }
+ if (decoded.size() < 4) {
+ return false;
+ }
+ try {
+ *packed_path = std::get<std::string>(std::get<0>(decoded.back()));
+ } catch (const std::bad_variant_access&) {
+ return false;
+ }
+ return true;
+}
+
+int InstanceRecycler::recycle_packed_files() {
+ const std::string task_name = "recycle_packed_files";
+ auto start_tp = steady_clock::now();
+ int64_t start_time = duration_cast<seconds>(start_tp.time_since_epoch()).count();
+ int ret = 0;
+ PackedFileRecycleStats stats;
+
+ register_recycle_task(task_name, start_time);
+ DORIS_CLOUD_DEFER {
+ unregister_recycle_task(task_name);
+ int64_t cost =
+ duration_cast<seconds>(steady_clock::now().time_since_epoch()).count() - start_time;
+ int64_t cost_ms = duration_cast<milliseconds>(steady_clock::now() - start_tp).count();
+ g_bvar_recycler_packed_file_recycled_kv_num.put(instance_id_, stats.num_deleted);
+ g_bvar_recycler_packed_file_recycled_kv_bytes.put(instance_id_, stats.bytes_deleted);
+ g_bvar_recycler_packed_file_recycle_cost_ms.put(instance_id_, cost_ms);
+ g_bvar_recycler_packed_file_scanned_kv_num.put(instance_id_, stats.num_scanned);
+ g_bvar_recycler_packed_file_corrected_kv_num.put(instance_id_, stats.num_corrected);
+ g_bvar_recycler_packed_file_recycled_object_num.put(instance_id_, stats.num_object_deleted);
+ g_bvar_recycler_packed_file_bytes_object_deleted.put(instance_id_,
+ stats.bytes_object_deleted);
+ g_bvar_recycler_packed_file_rowset_scanned_num.put(instance_id_, stats.rowset_scan_count);
+ LOG_INFO("recycle packed files finished, cost={}s", cost)
+ .tag("instance_id", instance_id_)
+ .tag("num_scanned", stats.num_scanned)
+ .tag("num_corrected", stats.num_corrected)
+ .tag("num_deleted", stats.num_deleted)
+ .tag("num_failed", stats.num_failed)
+ .tag("num_objects_deleted", stats.num_object_deleted)
+ .tag("bytes_object_deleted", stats.bytes_object_deleted)
+ .tag("rowset_scan_count", stats.rowset_scan_count)
+ .tag("bytes_deleted", stats.bytes_deleted)
+ .tag("ret", ret);
+ };
+
+ auto recycle_func = [this, &stats, &ret](auto&& key, auto&& value) {
+ return handle_packed_file_kv(std::forward<decltype(key)>(key),
+ std::forward<decltype(value)>(value), &stats, &ret);
+ };
+
+ LOG_INFO("begin to recycle packed file").tag("instance_id", instance_id_);
+
+ std::string begin = packed_file_key({instance_id_, ""});
+ std::string end = packed_file_key({instance_id_, "\xff"});
+ if (scan_and_recycle(begin, end, recycle_func) != 0) {
+ ret = -1;
+ }
+
+ return ret;
+}
+
int InstanceRecycler::scan_tablets_and_statistics(int64_t table_id, int64_t index_id,
RecyclerMetricsContext& metrics_context,
int64_t partition_id, bool is_empty_tablet) {
@@ -2724,6 +3737,13 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext&
.tag("rowset meta pb", rs_meta.ShortDebugString());
return -1;
}
+ if (decrement_packed_file_ref_counts(rs_meta) != 0) {
+ LOG_WARNING("failed to update packed file info when recycling tablet")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rs_meta.rowset_id_v2());
+ return -1;
+ }
recycle_rowsets_number += 1;
recycle_segments_number += rs_meta.num_segments();
recycle_rowsets_data_size += rs_meta.data_disk_size();
@@ -2769,6 +3789,13 @@ int InstanceRecycler::recycle_tablet(int64_t tablet_id, RecyclerMetricsContext&
.tag("rowset meta pb", rs_meta.ShortDebugString());
return -1;
}
+ if (decrement_packed_file_ref_counts(rs_meta) != 0) {
+ LOG_WARNING("failed to update packed file info when recycling restore job rowset")
+ .tag("instance_id", instance_id_)
+ .tag("tablet_id", tablet_id)
+ .tag("rowset_id", rs_meta.rowset_id_v2());
+ return -1;
+ }
recycle_restore_job_rowsets_number += 1;
recycle_restore_job_segments_number += rs_meta.num_segments();
recycle_restore_job_rowsets_data_size += rs_meta.data_disk_size();
@@ -4603,8 +5630,8 @@ int InstanceRecycler::recycle_expired_txn_label() {
concurrent_delete_executor.add([&]() {
int ret = delete_recycle_txn_kv(k);
if (ret == 1) {
- constexpr int MAX_RETRY = 10;
- for (size_t i = 1; i <= MAX_RETRY; ++i) {
+ const int max_retry = std::max(1, config::recycle_txn_delete_max_retry_times);
+ for (int i = 1; i <= max_retry; ++i) {
LOG(WARNING) << "txn conflict, retry times=" << i << " key=" << hex(k);
ret = delete_recycle_txn_kv(k);
// clang-format off
diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h
index 4ac9f2edaba..74a09398fe8 100644
--- a/cloud/src/recycler/recycler.h
+++ b/cloud/src/recycler/recycler.h
@@ -29,6 +29,8 @@
#include <string>
#include <string_view>
#include <thread>
+#include <unordered_map>
+#include <unordered_set>
#include <utility>
#include "common/bvars.h"
@@ -243,6 +245,17 @@ public:
class InstanceRecycler {
public:
+ struct PackedFileRecycleStats {
+ int64_t num_scanned = 0; // packed-file kv scanned
+ int64_t num_corrected = 0; // packed-file kv corrected
+ int64_t num_deleted = 0; // packed-file kv deleted
+ int64_t num_failed = 0; // packed-file kv failed
+ int64_t bytes_deleted = 0; // packed-file kv bytes deleted from txn-kv
+ int64_t num_object_deleted = 0; // packed-file objects deleted from storage (vault/HDFS)
+ int64_t bytes_object_deleted = 0; // bytes deleted from storage objects
+ int64_t rowset_scan_count = 0; // rowset metas scanned during correction
+ };
+
explicit InstanceRecycler(std::shared_ptr<TxnKv> txn_kv, const InstanceInfoPB& instance,
RecyclerThreadPoolGroup thread_pool_group,
std::shared_ptr<TxnLazyCommitter> txn_lazy_committer);
@@ -345,6 +358,13 @@ public:
// returns 0 for success otherwise error
int recycle_restore_jobs();
+ /**
+ * Scan packed-file metadata, correct reference counters, and recycle unused packed files.
+ *
+ * @return 0 on success, non-zero error code otherwise
+ */
+ int recycle_packed_files();
+
// scan and recycle snapshots
// returns 0 for success otherwise error
int recycle_cluster_snapshots();
@@ -377,6 +397,15 @@ public:
int scan_and_statistics_restore_jobs();
+ /**
+ * Decode the key of a packed-file metadata record into the persisted object path.
+ *
+ * @param key raw key persisted in txn-kv
+ * @param packed_path output object storage path referenced by the key
+ * @return true if decoding succeeds, false otherwise
+ */
+ static bool decode_packed_file_key(std::string_view key, std::string* packed_path);
+
void TEST_add_accessor(std::string_view id, std::shared_ptr<StorageVaultAccessor> accessor) {
accessor_map_.insert({std::string(id), std::move(accessor)});
}
@@ -416,6 +445,13 @@ private:
int delete_rowset_data(const std::map<std::string, doris::RowsetMetaCloudPB>& rowsets,
RowsetRecyclingState type, RecyclerMetricsContext& metrics_context);
+ // return 0 for success otherwise error
+ int decrement_packed_file_ref_counts(const doris::RowsetMetaCloudPB& rs_meta_pb);
+
+ int delete_packed_file_and_kv(const std::string& packed_file_path,
+ const std::string& packed_key,
+ const cloud::PackedFileInfoPB& packed_info);
+
/**
* Get stage storage info from instance and init StorageVaultAccessor
* @return 0 if accessor is successfully inited, 1 if stage not found, negative for error
@@ -462,6 +498,43 @@ private:
// Whether need to recycle versioned keys
bool should_recycle_versioned_keys() const;
+ /**
+ * Parse the path of a packed-file fragment and output the owning tablet and rowset identifiers.
+ *
+ * @param path packed-file fragment path to decode
+ * @param tablet_id output tablet identifier extracted from the path
+ * @param rowset_id output rowset identifier extracted from the path
+ * @return true if both identifiers are successfully parsed, false otherwise
+ */
+ static bool parse_packed_slice_path(std::string_view path, int64_t* tablet_id,
+ std::string* rowset_id);
+ // Check whether a rowset referenced by a packed file still exists in metadata.
+ // @param stats optional recycle statistics collector.
+ int check_rowset_exists(int64_t tablet_id, const std::string& rowset_id, bool* exists,
+ PackedFileRecycleStats* stats = nullptr);
+ int check_recycle_and_tmp_rowset_exists(int64_t tablet_id, const std::string& rowset_id,
+ int64_t txn_id, bool* recycle_exists, bool* tmp_exists);
+ /**
+ * Resolve which storage accessor should be used for a packed file.
+ *
+ * @param hint preferred storage resource identifier persisted with the file
+ * @return pair of the resolved resource identifier and accessor; the accessor can be null if unavailable
+ */
+ std::pair<std::string, std::shared_ptr<StorageVaultAccessor>> resolve_packed_file_accessor(
+ const std::string& hint);
+ // Recompute packed-file counters and lifecycle state after validating contained fragments.
+ // @param stats optional recycle statistics collector.
+ int correct_packed_file_info(cloud::PackedFileInfoPB* packed_info, bool* changed,
+ const std::string& packed_file_path,
+ PackedFileRecycleStats* stats = nullptr);
+ // Correct and recycle a single packed-file record, updating metadata and accounting statistics.
+ // @param stats optional recycle statistics collector.
+ int process_single_packed_file(const std::string& packed_key,
+ const std::string& packed_file_path,
+ PackedFileRecycleStats* stats);
+ // Process a packed-file KV while scanning and aggregate recycling statistics.
+ int handle_packed_file_kv(std::string_view key, std::string_view value,
+ PackedFileRecycleStats* stats, int* ret);
private:
std::atomic_bool stopped_ {false};
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index aec4908acdd..20ebdbc2fb4 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -12661,4 +12661,160 @@ TEST(MetaServiceTest, CleanTxnLabelVersionedWriteMixedTxns) {
}
}
+TEST(MetaServiceTest, UpdatePackedFileInfoTest) {
+ auto meta_service = get_meta_service();
+
+ // case: normal update merge file info - success case
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_packed_file_path("/path/to/packed/file1");
+
+ // Set packed file info
+ auto* packed_info = req.mutable_packed_file_info();
+ packed_info->set_ref_cnt(5);
+ packed_info->set_total_slice_num(10);
+ packed_info->set_total_slice_bytes(1024000);
+ packed_info->set_remaining_slice_bytes(819200);
+ packed_info->set_created_at_sec(1666666666);
+ packed_info->set_corrected(false);
+ packed_info->set_state(PackedFileInfoPB::NORMAL);
+
+ // Add small files
+ auto* small_file = packed_info->add_slices();
+ small_file->set_path("/small/file1.txt");
+ small_file->set_offset(0);
+ small_file->set_size(512);
+ small_file->set_deleted(false);
+
+ small_file = packed_info->add_slices();
+ small_file->set_path("/small/file2.txt");
+ small_file->set_offset(512);
+ small_file->set_size(1024);
+ small_file->set_deleted(false);
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ }
+
+ // case: missing cloud_unique_id - should fail
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ // req.set_cloud_unique_id("test_cloud_unique_id"); // missing cloud_unique_id
+ req.set_packed_file_path("/path/to/packed/file");
+
+ auto* packed_info = req.mutable_packed_file_info();
+ packed_info->set_ref_cnt(1);
+ packed_info->set_total_slice_num(1);
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ ASSERT_TRUE(res.status().msg().find("cloud_unique_id is required") != std::string::npos);
+ }
+
+ // case: empty cloud_unique_id - should fail
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id(""); // empty cloud_unique_id
+ req.set_packed_file_path("/path/to/merged/file");
+
+ auto* merge_info = req.mutable_packed_file_info();
+ merge_info->set_ref_cnt(1);
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ ASSERT_TRUE(res.status().msg().find("cloud_unique_id is required") != std::string::npos);
+ }
+
+ // case: missing packed_file_path - should fail
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ // req.set_packed_file_path("/path/to/merged/file"); // missing packed_file_path
+
+ auto* merge_info = req.mutable_packed_file_info();
+ merge_info->set_ref_cnt(1);
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ ASSERT_TRUE(res.status().msg().find("packed_file_path is required") != std::string::npos);
+ }
+
+ // case: empty packed_file_path - should fail
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_packed_file_path(""); // empty packed_file_path
+
+ auto* merge_info = req.mutable_packed_file_info();
+ merge_info->set_ref_cnt(1);
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ ASSERT_TRUE(res.status().msg().find("packed_file_path is required") != std::string::npos);
+ }
+
+ // case: missing packed_file_info - should fail
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_packed_file_path("/path/to/merged/file");
+ // No packed_file_info set
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT);
+ ASSERT_TRUE(res.status().msg().find("packed_file_info is required") != std::string::npos);
+ }
+
+ // case: test multiple operations - success case
+ {
+ brpc::Controller cntl;
+ UpdatePackedFileInfoRequest req;
+ req.set_cloud_unique_id("test_cloud_unique_id_multi");
+ req.set_packed_file_path("/path/to/merged/file_multi");
+
+ // Set merge file info with multiple small files
+ auto* merge_info = req.mutable_packed_file_info();
+ merge_info->set_ref_cnt(3);
+ merge_info->set_total_slice_num(5);
+ merge_info->set_total_slice_bytes(2048000);
+ merge_info->set_remaining_slice_bytes(1638400);
+ merge_info->set_created_at_sec(1666666777);
+ merge_info->set_corrected(true);
+ merge_info->set_state(PackedFileInfoPB::NORMAL);
+
+ // Add multiple small files
+ for (int i = 1; i <= 5; ++i) {
+ auto* small_file = merge_info->add_slices();
+ small_file->set_path("/small/file" + std::to_string(i) + ".txt");
+ small_file->set_offset(i * 1000);
+ small_file->set_size(1000);
+ small_file->set_deleted(i > 3); // Mark some as deleted
+ }
+
+ UpdatePackedFileInfoResponse res;
+ meta_service->update_packed_file_info(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ }
+}
+
} // namespace doris::cloud
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index c981c0a58bf..a308a1cfef8 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -18,21 +18,27 @@
#include "recycler/recycler.h"
#include <butil/strings/string_split.h>
+#include <bvar/variable.h>
#include <fmt/core.h>
#include <gen_cpp/cloud.pb.h>
#include <gen_cpp/olap_file.pb.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
+#include <array>
+#include <atomic>
#include <chrono>
#include <cstdint>
#include <cstdlib>
+#include <exception>
#include <future>
#include <memory>
#include <random>
#include <string>
+#include <string_view>
#include <thread>
+#include "common/bvars.h"
#include "common/config.h"
#include "common/defer.h"
#include "common/logging.h"
@@ -62,6 +68,27 @@ int64_t current_time = 0;
static constexpr int64_t db_id = 1000;
static RecyclerMetricsContext ctx;
+namespace {
+
+int64_t read_status_bvar_value(const std::string& module, const std::string& name,
+ const std::string& tag) {
+ const std::string bvar_name = fmt::format("{}_{}_{}", module, name, tag);
+ const std::string value = bvar::Variable::describe_exposed(bvar_name);
+ if (value.empty()) {
+ ADD_FAILURE() << "bvar " << bvar_name << " not exposed";
+ return 0;
+ }
+ try {
+ return std::stoll(value);
+ } catch (const std::exception& e) {
+ ADD_FAILURE() << "failed to parse bvar value for " << bvar_name << ": " << e.what()
+ << ", raw=\"" << value << "\"";
+ return 0;
+ }
+}
+
+} // namespace
+
std::vector<std::string> index_v2_file_path = {
"data/1753202639945/0200000000001a5c92f4e7d9j8f2b4c8a3e6f8b1c9d2e5f8_0.idx",
"data/1753202639947/0200000000001b8d45a74r6c7sf3e9c2b6d4a8e1f7c3d9e2_0.idx",
@@ -157,6 +184,11 @@ static std::string next_rowset_id() {
return std::to_string(++cnt);
}
+static int64_t next_small_file_txn_id() {
+ static std::atomic<int64_t> txn {1000};
+ return ++txn;
+}
+
static doris::RowsetMetaCloudPB create_rowset(const std::string& resource_id, int64_t tablet_id,
int64_t index_id, int num_segments,
const doris::TabletSchemaCloudPB& schema,
@@ -1786,6 +1818,126 @@ TEST(RecyclerTest, recycle_tablet) {
check_delete_bitmap_file_size(accessor, tablet_id, 0);
}
+TEST(RecyclerTest, recycle_tablet_packed_file_ref_count) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ constexpr std::string_view kResourceId = "recycle_tablet_merge_ref";
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id(std::string(kResourceId));
+ obj_info->set_ak(config::test_s3_ak);
+ obj_info->set_sk(config::test_s3_sk);
+ obj_info->set_endpoint(config::test_s3_endpoint);
+ obj_info->set_region(config::test_s3_region);
+ obj_info->set_bucket(config::test_s3_bucket);
+ obj_info->set_prefix(std::string(kResourceId));
+
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.begin()->second;
+
+ constexpr int64_t table_id = 40001;
+ constexpr int64_t index_id = 40002;
+ constexpr int64_t partition_id = 40003;
+ constexpr int64_t tablet_id = 40004;
+ ASSERT_EQ(create_tablet(txn_kv.get(), table_id, index_id, partition_id, tablet_id), 0);
+
+ // Build a rowset whose segments are merged into a shared merged file.
+ doris::RowsetMetaCloudPB merged_rowset;
+ merged_rowset.set_rowset_id(0);
+ merged_rowset.set_rowset_id_v2("merged_rowset");
+ merged_rowset.set_rowset_state(RowsetStatePB::VISIBLE);
+ merged_rowset.set_tablet_id(tablet_id);
+ merged_rowset.set_resource_id(std::string(kResourceId));
+ merged_rowset.set_num_segments(2);
+ merged_rowset.set_creation_time(current_time);
+ merged_rowset.set_schema_version(1);
+ merged_rowset.set_index_id(index_id);
+ merged_rowset.set_start_version(0);
+ merged_rowset.set_end_version(1);
+ const int64_t kSmallFileSize = 64;
+ const std::string packed_file_path =
+ fmt::format("data/merge_file/{}/{}.dat", tablet_id, merged_rowset.rowset_id_v2());
+ auto merged_index_map = merged_rowset.mutable_packed_slice_locations();
+ for (int i = 0; i < merged_rowset.num_segments(); ++i) {
+ std::string small_path = segment_path(tablet_id, merged_rowset.rowset_id_v2(), i);
+ auto& index_pb = (*merged_index_map)[small_path];
+ index_pb.set_packed_file_path(packed_file_path);
+ index_pb.set_offset(i * kSmallFileSize);
+ index_pb.set_size(kSmallFileSize);
+ }
+
+ // Another rowset without merged segments.
+ doris::RowsetMetaCloudPB plain_rowset;
+ plain_rowset.set_rowset_id(0);
+ plain_rowset.set_rowset_id_v2("plain_rowset");
+ plain_rowset.set_rowset_state(RowsetStatePB::VISIBLE);
+ plain_rowset.set_tablet_id(tablet_id);
+ plain_rowset.set_resource_id(std::string(kResourceId));
+ plain_rowset.set_num_segments(1);
+ plain_rowset.set_creation_time(current_time);
+ plain_rowset.set_schema_version(2);
+ plain_rowset.set_index_id(index_id);
+ plain_rowset.set_start_version(2);
+ plain_rowset.set_end_version(2);
+
+ // Put rowset meta and merged file meta into kv.
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+ std::string merged_rowset_key = meta_rowset_key({instance_id, tablet_id, 1});
+ txn->put(merged_rowset_key, merged_rowset.SerializeAsString());
+ std::string plain_rowset_key = meta_rowset_key({instance_id, tablet_id, 2});
+ txn->put(plain_rowset_key, plain_rowset.SerializeAsString());
+
+ PackedFileInfoPB merged_info;
+ merged_info.set_ref_cnt(merged_rowset.num_segments());
+ merged_info.set_total_slice_num(merged_rowset.num_segments());
+ merged_info.set_total_slice_bytes(kSmallFileSize * merged_rowset.num_segments());
+ merged_info.set_remaining_slice_bytes(kSmallFileSize * merged_rowset.num_segments());
+ merged_info.set_state(PackedFileInfoPB::NORMAL);
+ merged_info.set_resource_id(std::string(kResourceId));
+ for (const auto& [small_path, _] : *merged_index_map) {
+ auto* small_file = merged_info.add_slices();
+ small_file->set_path(small_path);
+ small_file->set_offset((*merged_index_map)[small_path].offset());
+ small_file->set_size(kSmallFileSize);
+ small_file->set_deleted(false);
+ small_file->set_txn_id(next_small_file_txn_id());
+ small_file->set_rowset_id(merged_rowset.rowset_id_v2());
+ small_file->set_tablet_id(tablet_id);
+ }
+ std::string merged_key = packed_file_key({instance_id, packed_file_path});
+ txn->put(merged_key, merged_info.SerializeAsString());
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ // Prepare object storage files.
+ ASSERT_EQ(0, accessor->put_file(packed_file_path, "payload"));
+ for (int i = 0; i < merged_rowset.num_segments(); ++i) {
+ auto path = segment_path(tablet_id, merged_rowset.rowset_id_v2(), i);
+ ASSERT_EQ(0, accessor->put_file(path, "segment"));
+ }
+ for (int i = 0; i < plain_rowset.num_segments(); ++i) {
+ auto path = segment_path(tablet_id, plain_rowset.rowset_id_v2(), i);
+ ASSERT_EQ(0, accessor->put_file(path, "segment"));
+ }
+
+ ASSERT_EQ(0, recycler.recycle_tablets(table_id, index_id, ctx));
+
+ // merged file kv should be removed and file deleted as ref_cnt reaches zero.
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+ std::string merged_val;
+ EXPECT_EQ(TxnErrorCode::TXN_KEY_NOT_FOUND, txn->get(merged_key, &merged_val));
+ EXPECT_EQ(1, accessor->exists(packed_file_path));
+
+ // tablet directory should be cleaned.
+ std::unique_ptr<ListIterator> list_iter;
+ ASSERT_EQ(0, accessor->list_directory(tablet_path_prefix(tablet_id), &list_iter));
+ ASSERT_FALSE(list_iter->has_next());
+}
+
TEST(RecyclerTest, recycle_indexes) {
config::retention_seconds = 0;
auto txn_kv = std::make_shared<MemTxnKv>();
@@ -5324,6 +5476,873 @@ TEST(RecyclerTest, delete_rowset_data) {
}
}
+TEST(RecyclerTest, delete_rowset_data_packed_file_single_rowset) {
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ constexpr std::string_view kResourceId = "packed_file_resource_single";
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id(std::string(kResourceId));
+ obj_info->set_ak(config::test_s3_ak);
+ obj_info->set_sk(config::test_s3_sk);
+ obj_info->set_endpoint(config::test_s3_endpoint);
+ obj_info->set_region(config::test_s3_region);
+ obj_info->set_bucket(config::test_s3_bucket);
+ obj_info->set_prefix(std::string(kResourceId));
+
+ InstanceRecycler recycler(txn_kv, instance, thread_group,
+ std::make_shared<TxnLazyCommitter>(txn_kv));
+ ASSERT_EQ(recycler.init(), 0);
+ auto accessor = recycler.accessor_map_.begin()->second;
+
+ doris::TabletSchemaCloudPB schema;
+ schema.set_schema_version(1);
+ schema.set_inverted_index_storage_format(InvertedIndexStorageFormatPB::V1);
+
+ auto rowset = create_rowset(std::string(kResourceId), 9527, 1024, 2, schema, 6001);
+ ASSERT_EQ(0, create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, true));
+
+ auto merged_map = rowset.mutable_packed_slice_locations();
+ std::string packed_file_path =
+ fmt::format("merge/{}/{}.dat", rowset.tablet_id(), rowset.rowset_id_v2());
+ constexpr int64_t kSmallFileSize = 128;
+ for (int i = 0; i < rowset.num_segments(); ++i) {
+ std::string small_path = segment_path(rowset.tablet_id(), rowset.rowset_id_v2(), i);
+ auto& index_pb = (*merged_map)[small_path];
... 3397 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org