You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2026/05/09 07:18:52 UTC

(doris) branch branch-4.1 updated: branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.1 by this push:
     new 15969aefecc branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)
15969aefecc is described below

commit 15969aefecc3d19d065f2acd8276e9278628023e
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Sat May 9 15:18:44 2026 +0800

    branch-4.1: [fix](filecache) reject oversized block size in add_cell #62878 (#63046)
    
    Cherry-picked from #62878
    
    Co-authored-by: deardeng <de...@selectdb.com>
---
 be/src/io/cache/block_file_cache.cpp       |  3 +-
 be/src/io/cache/block_file_cache.h         |  1 +
 be/src/io/cache/fs_file_cache_storage.cpp  |  5 ++
 be/test/io/cache/block_file_cache_test.cpp | 76 ++++++++++++++++++++++++++++++
 4 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
index 4bfcd8a157e..3555898b8d9 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -892,9 +892,10 @@ FileBlockCell* BlockFileCache::add_cell(const UInt128Wrapper& hash, const CacheC
                << " tablet_id=" << context.tablet_id;
 
     if (size > 1024 * 1024 * 1024) {
-        LOG(WARNING) << "File block size is too large for a block. size=" << size
+        LOG(WARNING) << "File block size is too large for a block, reject. size=" << size
                      << " hash=" << hash.to_string() << " offset=" << offset
                      << " stack:" << get_stack_trace();
+        return nullptr;
     }
 
     auto& offsets = _files[hash];
diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h
index e7a2b9636c2..f4d15f848c2 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -167,6 +167,7 @@ class BlockFileCache {
     friend class CacheLRUDumper;
     friend class LRUQueueRecorder;
     friend struct FileBlockCell;
+    friend class BlockFileCacheTest;
 
 public:
     // hash the file_name to uint128
diff --git a/be/src/io/cache/fs_file_cache_storage.cpp b/be/src/io/cache/fs_file_cache_storage.cpp
index 702eb65380b..a9e829329b1 100644
--- a/be/src/io/cache/fs_file_cache_storage.cpp
+++ b/be/src/io/cache/fs_file_cache_storage.cpp
@@ -690,6 +690,11 @@ void FSFileCacheStorage::load_cache_info_into_memory_from_fs(BlockFileCache* mgr
             context.expiration_time = expiration_time;
             for (; offset_it != std::filesystem::directory_iterator(); ++offset_it) {
                 size_t size = offset_it->file_size(ec);
+                if (ec) [[unlikely]] {
+                    LOG(WARNING) << "skip cache file, file_size failed, file="
+                                 << offset_it->path().native() << " err=" << ec.message();
+                    continue;
+                }
                 size_t offset = 0;
                 bool is_tmp = false;
                 FileCacheType cache_type = FileCacheType::NORMAL;
diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp
index c5b9eb5105d..b857ebe8e2f 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -7895,4 +7895,80 @@ TEST_F(BlockFileCacheTest, set_downloaded_empty_block_branch) {
     ASSERT_EQ(block.get_downloader(), 0);
 }
 
+// Reproduces OPENSOURCE-325: when fs_file_cache_storage's directory scan reads
+// a file size via std::filesystem::directory_entry::file_size(ec) and the
+// underlying syscall fails (e.g. file removed concurrently by clear/GC), the
+// returned value is (uintmax_t)-1 == UINT64_MAX. Before the fix, that bogus
+// size flowed into BlockFileCache::add_cell, which only logged a WARNING and
+// then registered the cell, polluting _files, the LRU queue's cache_size, and
+// _cur_cache_size. After the fix, add_cell rejects sizes > 1GiB and returns
+// nullptr without touching any internal state.
+TEST_F(BlockFileCacheTest, add_cell_rejects_oversized_size) {
+    if (fs::exists(cache_base_path)) {
+        fs::remove_all(cache_base_path);
+    }
+    fs::create_directories(cache_base_path);
+
+    io::FileCacheSettings settings;
+    settings.query_queue_size = 30_mb;
+    settings.query_queue_elements = 30;
+    settings.capacity = 30_mb;
+    settings.max_file_block_size = 1_mb;
+    settings.max_query_cache_size = 30;
+
+    io::BlockFileCache cache(cache_base_path, settings);
+    ASSERT_TRUE(cache.initialize());
+    for (int i = 0; i < 100; ++i) {
+        if (cache.get_async_open_success()) {
+            break;
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+    ASSERT_TRUE(cache.get_async_open_success());
+
+    auto hash = io::BlockFileCache::hash("opensource_325_key");
+    io::CacheContext ctx;
+    TUniqueId qid;
+    qid.hi = 1;
+    qid.lo = 1;
+    ctx.query_id = qid;
+    ctx.cache_type = io::FileCacheType::NORMAL;
+    ReadStatistics rstats;
+    ctx.stats = &rstats;
+
+    const size_t kBadSize = std::numeric_limits<size_t>::max(); // (uintmax_t)-1
+    const size_t kOffset = 1128267776;                          // matches the JIRA report
+
+    size_t cur_cache_size_before = 0;
+    size_t normal_queue_size_before = 0;
+    {
+        std::lock_guard<std::mutex> cache_lock(cache._mutex);
+        cur_cache_size_before = cache._cur_cache_size;
+        normal_queue_size_before =
+                cache.get_queue(io::FileCacheType::NORMAL).get_capacity(cache_lock);
+    }
+
+    {
+        std::lock_guard<std::mutex> cache_lock(cache._mutex);
+        auto* cell = cache.add_cell(hash, ctx, kOffset, kBadSize, io::FileBlock::State::DOWNLOADED,
+                                    cache_lock);
+        // Defensive guard must reject oversized blocks instead of polluting state.
+        ASSERT_EQ(cell, nullptr);
+    }
+
+    {
+        std::lock_guard<std::mutex> cache_lock(cache._mutex);
+        // _files index untouched: no entry for (hash, kOffset).
+        ASSERT_EQ(cache._files.find(hash), cache._files.end());
+        // Global and per-queue size counters untouched (no UINT64_MAX added).
+        ASSERT_EQ(cache._cur_cache_size, cur_cache_size_before);
+        ASSERT_EQ(cache.get_queue(io::FileCacheType::NORMAL).get_capacity(cache_lock),
+                  normal_queue_size_before);
+    }
+
+    if (fs::exists(cache_base_path)) {
+        fs::remove_all(cache_base_path);
+    }
+}
+
 } // namespace doris::io


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org