You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/09 07:18:52 UTC
(doris) branch branch-4.1 updated: branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 15969aefecc branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)
15969aefecc is described below
commit 15969aefecc3d19d065f2acd8276e9278628023e
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat May 9 15:18:44 2026 +0800
branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)
Cherry-picked from #62878
Co-authored-by: deardeng <de...@selectdb.com>
---
be/src/io/cache/block_file_cache.cpp | 3 +-
be/src/io/cache/block_file_cache.h | 1 +
be/src/io/cache/fs_file_cache_storage.cpp | 5 ++
be/test/io/cache/block_file_cache_test.cpp | 76 ++++++++++++++++++++++++++++++
4 files changed, 84 insertions(+), 1 deletion(-)
diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
index 4bfcd8a157e..3555898b8d9 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -892,9 +892,10 @@ FileBlockCell* BlockFileCache::add_cell(const UInt128Wrapper& hash, const CacheC
<< " tablet_id=" << context.tablet_id;
if (size > 1024 * 1024 * 1024) {
- LOG(WARNING) << "File block size is too large for a block. size=" << size
+ LOG(WARNING) << "File block size is too large for a block, reject. size=" << size
<< " hash=" << hash.to_string() << " offset=" << offset
<< " stack:" << get_stack_trace();
+ return nullptr;
}
auto& offsets = _files[hash];
diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h
index e7a2b9636c2..f4d15f848c2 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -167,6 +167,7 @@ class BlockFileCache {
friend class CacheLRUDumper;
friend class LRUQueueRecorder;
friend struct FileBlockCell;
+ friend class BlockFileCacheTest;
public:
// hash the file_name to uint128
diff --git a/be/src/io/cache/fs_file_cache_storage.cpp b/be/src/io/cache/fs_file_cache_storage.cpp
index 702eb65380b..a9e829329b1 100644
--- a/be/src/io/cache/fs_file_cache_storage.cpp
+++ b/be/src/io/cache/fs_file_cache_storage.cpp
@@ -690,6 +690,11 @@ void FSFileCacheStorage::load_cache_info_into_memory_from_fs(BlockFileCache* mgr
context.expiration_time = expiration_time;
for (; offset_it != std::filesystem::directory_iterator(); ++offset_it) {
size_t size = offset_it->file_size(ec);
+ if (ec) [[unlikely]] {
+ LOG(WARNING) << "skip cache file, file_size failed, file="
+ << offset_it->path().native() << " err=" << ec.message();
+ continue;
+ }
size_t offset = 0;
bool is_tmp = false;
FileCacheType cache_type = FileCacheType::NORMAL;
diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp
index c5b9eb5105d..b857ebe8e2f 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -7895,4 +7895,80 @@ TEST_F(BlockFileCacheTest, set_downloaded_empty_block_branch) {
ASSERT_EQ(block.get_downloader(), 0);
}
+// Reproduces OPENSOURCE-325: when fs_file_cache_storage's directory scan reads
+// a file size via std::filesystem::directory_entry::file_size(ec) and the
+// underlying syscall fails (e.g. file removed concurrently by clear/GC), the
+// returned value is (uintmax_t)-1 == UINT64_MAX. Before the fix, that bogus
+// size flowed into BlockFileCache::add_cell, which only logged a WARNING and
+// then registered the cell, polluting _files, the LRU queue's cache_size, and
+// _cur_cache_size. After the fix, add_cell rejects sizes > 1GiB and returns
+// nullptr without touching any internal state.
+TEST_F(BlockFileCacheTest, add_cell_rejects_oversized_size) {
+ if (fs::exists(cache_base_path)) {
+ fs::remove_all(cache_base_path);
+ }
+ fs::create_directories(cache_base_path);
+
+ io::FileCacheSettings settings;
+ settings.query_queue_size = 30_mb;
+ settings.query_queue_elements = 30;
+ settings.capacity = 30_mb;
+ settings.max_file_block_size = 1_mb;
+ settings.max_query_cache_size = 30;
+
+ io::BlockFileCache cache(cache_base_path, settings);
+ ASSERT_TRUE(cache.initialize());
+ for (int i = 0; i < 100; ++i) {
+ if (cache.get_async_open_success()) {
+ break;
+ }
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+ ASSERT_TRUE(cache.get_async_open_success());
+
+ auto hash = io::BlockFileCache::hash("opensource_325_key");
+ io::CacheContext ctx;
+ TUniqueId qid;
+ qid.hi = 1;
+ qid.lo = 1;
+ ctx.query_id = qid;
+ ctx.cache_type = io::FileCacheType::NORMAL;
+ ReadStatistics rstats;
+ ctx.stats = &rstats;
+
+ const size_t kBadSize = std::numeric_limits<size_t>::max(); // (uintmax_t)-1
+ const size_t kOffset = 1128267776; // matches the JIRA report
+
+ size_t cur_cache_size_before = 0;
+ size_t normal_queue_size_before = 0;
+ {
+ std::lock_guard<std::mutex> cache_lock(cache._mutex);
+ cur_cache_size_before = cache._cur_cache_size;
+ normal_queue_size_before =
+ cache.get_queue(io::FileCacheType::NORMAL).get_capacity(cache_lock);
+ }
+
+ {
+ std::lock_guard<std::mutex> cache_lock(cache._mutex);
+ auto* cell = cache.add_cell(hash, ctx, kOffset, kBadSize, io::FileBlock::State::DOWNLOADED,
+ cache_lock);
+ // Defensive guard must reject oversized blocks instead of polluting state.
+ ASSERT_EQ(cell, nullptr);
+ }
+
+ {
+ std::lock_guard<std::mutex> cache_lock(cache._mutex);
+ // _files index untouched: no entry for (hash, kOffset).
+ ASSERT_EQ(cache._files.find(hash), cache._files.end());
+ // Global and per-queue size counters untouched (no UINT64_MAX added).
+ ASSERT_EQ(cache._cur_cache_size, cur_cache_size_before);
+ ASSERT_EQ(cache.get_queue(io::FileCacheType::NORMAL).get_capacity(cache_lock),
+ normal_queue_size_before);
+ }
+
+ if (fs::exists(cache_base_path)) {
+ fs::remove_all(cache_base_path);
+ }
+}
+
} // namespace doris::io
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org