You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2025/04/25 10:10:43 UTC

(doris) branch branch-3.0.5 updated (e277cfb83f4 -> de3cb382fbe)

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a change to branch branch-3.0.5
in repository https://gitbox.apache.org/repos/asf/doris.git


    from e277cfb83f4 branch-3.0: [fix](connect) fix wrong format causing connection fail #50214 (#50216)
     new c251728f7a1 branch-3.0: [Fix](vault) Remove double slashes when s3 vault prefix is empty #50352 (#50377)
     new 268d6716dad branch-3.0: [fix](serDe) global function could not deserialization from image #50279 (#50347)
     new de3cb382fbe [fix](cloud) batch process ttl cache block gc to limit lock held time once in a time (pick#50387) (#50402)

The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 be/src/common/config.cpp                           |  3 +
 be/src/common/config.h                             |  4 ++
 be/src/io/cache/block_file_cache.cpp               | 18 ++++--
 be/src/io/cache/block_file_cache.h                 |  1 +
 be/test/io/cache/block_file_cache_test.cpp         | 64 ++++++++++++++++++++++
 cloud/src/recycler/s3_accessor.cpp                 |  8 ++-
 .../apache/doris/catalog/GlobalFunctionMgr.java    |  2 +-
 .../doris/nereids/trees/expressions/UdfTest.java   | 10 +++-
 8 files changed, 101 insertions(+), 9 deletions(-)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 02/03: branch-3.0: [fix](serDe) global function could not deserialization from image #50279 (#50347)

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0.5
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 268d6716dadd38a3c1b59b6faf9b56f0109ed349
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu Apr 24 19:41:39 2025 +0800

    branch-3.0: [fix](serDe) global function could not deserialization from image #50279 (#50347)
    
    Cherry-picked from #50279
    
    Co-authored-by: morrySnow <zh...@selectdb.com>
---
 .../main/java/org/apache/doris/catalog/GlobalFunctionMgr.java  |  2 +-
 .../org/apache/doris/nereids/trees/expressions/UdfTest.java    | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
index fbb8a525a36..2e3f81d93ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/GlobalFunctionMgr.java
@@ -61,7 +61,7 @@ public class GlobalFunctionMgr extends MetaObject implements GsonPostProcessable
     @Override
     public void write(DataOutput out) throws IOException {
         // write functions
-        Text.writeString(out, GsonUtils.GSON.toJson(name2Function));
+        Text.writeString(out, GsonUtils.GSON.toJson(this, GlobalFunctionMgr.class));
     }
 
     @Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
index 635a3300dfe..39fd3a22336 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/UdfTest.java
@@ -18,6 +18,8 @@
 package org.apache.doris.nereids.trees.expressions;
 
 import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.Function;
+import org.apache.doris.catalog.GlobalFunctionMgr;
 import org.apache.doris.common.Config;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.DateFormat;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
@@ -49,6 +51,7 @@ import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
 import java.io.DataOutputStream;
+import java.util.List;
 
 public class UdfTest extends TestWithFeService implements PlanPatternMatchSupported {
     @Override
@@ -192,8 +195,13 @@ public class UdfTest extends TestWithFeService implements PlanPatternMatchSuppor
         Env.getCurrentEnv().getGlobalFunctionMgr().write(new DataOutputStream(outputStream));
         byte[] buffer = outputStream.toByteArray();
         ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer);
-        Env.getCurrentEnv().getGlobalFunctionMgr().readFields(new DataInputStream(inputStream));
+        GlobalFunctionMgr newMgr = GlobalFunctionMgr.read(new DataInputStream(inputStream));
 
+        List<Function> functions = newMgr.getFunctions();
+        Assertions.assertEquals(1, functions.stream()
+                .map(f -> f.getFunctionName().getFunction())
+                .filter(name -> name.equals("f8"))
+                .count());
         Assertions.assertEquals(1, Env.getCurrentEnv().getFunctionRegistry()
                 .findUdfBuilder(connectContext.getDatabase(), "f8").size());
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 01/03: branch-3.0: [Fix](vault) Remove double slashes when s3 vault prefix is empty #50352 (#50377)

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0.5
in repository https://gitbox.apache.org/repos/asf/doris.git

commit c251728f7a19ab8ddece88dbafb7b827017aabb4
Author: github-actions[bot] <41...@users.noreply.github.com>
AuthorDate: Thu Apr 24 19:42:49 2025 +0800

    branch-3.0: [Fix](vault) Remove double slashes when s3 vault prefix is empty #50352 (#50377)
    
    Cherry-picked from #50352
    
    Co-authored-by: abmdocrt <li...@selectdb.com>
---
 cloud/src/recycler/s3_accessor.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp
index aa37f9a983e..0c024111da6 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -204,7 +204,7 @@ S3Accessor::S3Accessor(S3Conf conf)
 S3Accessor::~S3Accessor() = default;
 
 std::string S3Accessor::get_key(const std::string& relative_path) const {
-    return conf_.prefix + '/' + relative_path;
+    return conf_.prefix.empty() ? relative_path : conf_.prefix + '/' + relative_path;
 }
 
 std::string S3Accessor::to_uri(const std::string& relative_path) const {
@@ -268,7 +268,11 @@ int S3Accessor::init() {
 #endif
     }
     default: {
-        uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix;
+        if (conf_.prefix.empty()) {
+            uri_ = conf_.endpoint + '/' + conf_.bucket;
+        } else {
+            uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix;
+        }
 
         static S3Environment s3_env;
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org


(doris) 03/03: [fix](cloud) batch process ttl cache block gc to limit lock held time once in a time (pick#50387) (#50402)

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0.5
in repository https://gitbox.apache.org/repos/asf/doris.git

commit de3cb382fbe2dac4467a45c210118a8fc2e0af42
Author: zhengyu <zh...@selectdb.com>
AuthorDate: Fri Apr 25 17:53:51 2025 +0800

    [fix](cloud) batch process ttl cache block gc to limit lock held time once in a time (pick#50387) (#50402)
    
    pick#50387
    
    too many ttl cache blocks gc will burst the cache lock latency and thus
    affect the query latency. limit them into batches to unleash the lock.
---
 be/src/common/config.cpp                   |  3 ++
 be/src/common/config.h                     |  4 ++
 be/src/io/cache/block_file_cache.cpp       | 18 ++++++---
 be/src/io/cache/block_file_cache.h         |  1 +
 be/test/io/cache/block_file_cache_test.cpp | 64 ++++++++++++++++++++++++++++++
 5 files changed, 85 insertions(+), 5 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index f36bd991e09..452e93a3d49 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1097,6 +1097,9 @@ DEFINE_mInt64(cache_lock_held_long_tail_threshold_us, "30000000");
 DEFINE_mBool(enable_file_cache_keep_base_compaction_output, "false");
 DEFINE_mInt64(file_cache_remove_block_qps_limit, "1000");
 DEFINE_mInt64(file_cache_background_gc_interval_ms, "100");
+DEFINE_mInt64(file_cache_background_monitor_interval_ms, "5000");
+DEFINE_mInt64(file_cache_background_ttl_gc_interval_ms, "3000");
+DEFINE_mInt64(file_cache_background_ttl_gc_batch, "1000");
 
 DEFINE_mInt32(index_cache_entry_stay_time_after_lookup_s, "1800");
 DEFINE_mInt32(inverted_index_cache_stale_sweep_time_sec, "600");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index f927c0ae704..3294dd5e7df 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1138,6 +1138,10 @@ DECLARE_mInt64(cache_lock_held_long_tail_threshold_us);
 DECLARE_mBool(enable_file_cache_keep_base_compaction_output);
 DECLARE_mInt64(file_cache_remove_block_qps_limit);
 DECLARE_mInt64(file_cache_background_gc_interval_ms);
+DECLARE_mInt64(file_cache_background_monitor_interval_ms);
+DECLARE_mInt64(file_cache_background_ttl_gc_interval_ms);
+DECLARE_mInt64(file_cache_background_ttl_gc_batch);
+
 // inverted index searcher cache
 // cache entry stay time after lookup
 DECLARE_mInt32(index_cache_entry_stay_time_after_lookup_s);
diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp
index 52e36e42f9f..d38e9670f2f 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -219,6 +219,8 @@ BlockFileCache::BlockFileCache(const std::string& cache_base_path,
 
     _recycle_keys_length_recorder = std::make_shared<bvar::LatencyRecorder>(
             _cache_base_path.c_str(), "file_cache_recycle_keys_length");
+    _ttl_gc_latency_us = std::make_shared<bvar::LatencyRecorder>(_cache_base_path.c_str(),
+                                                                 "file_cache_ttl_gc_latency_us");
 
     _disposable_queue = LRUQueue(cache_settings.disposable_queue_size,
                                  cache_settings.disposable_queue_elements, 60 * 60);
@@ -1829,28 +1831,34 @@ void BlockFileCache::run_background_monitor() {
     }
 }
 
-void BlockFileCache::run_background_ttl_gc() { // TODO(zhengyu): fix!
-    int64_t interval_time_seconds = 20;
+void BlockFileCache::run_background_ttl_gc() {
     while (!_close) {
-        TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time", &interval_time_seconds);
+        int64_t interval_ms = config::file_cache_background_ttl_gc_interval_ms;
+        int64_t batch_size = config::file_cache_background_ttl_gc_batch;
+        TEST_SYNC_POINT_CALLBACK("BlockFileCache::set_sleep_time", &interval_ms);
         {
             std::unique_lock close_lock(_close_mtx);
-            _close_cv.wait_for(close_lock, std::chrono::seconds(interval_time_seconds));
+            _close_cv.wait_for(close_lock, std::chrono::milliseconds(interval_ms));
             if (_close) {
                 break;
             }
         }
+        int64_t duration_ns = 0;
         {
             int64_t cur_time = UnixSeconds();
+            int64_t count = 0;
             SCOPED_CACHE_LOCK(_mutex, this);
+            SCOPED_RAW_TIMER(&duration_ns);
             while (!_time_to_key.empty()) {
                 auto begin = _time_to_key.begin();
-                if (cur_time < begin->first) {
+                if (cur_time < begin->first || count > batch_size) {
                     break;
                 }
                 remove_if_ttl_file_blocks(begin->second, false, cache_lock, false);
+                ++count;
             }
         }
+        *_ttl_gc_latency_us << (duration_ns / 1000);
     }
 }
 
diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h
index 5634621eb41..a64d339d081 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -574,6 +574,7 @@ private:
     std::shared_ptr<bvar::LatencyRecorder> _storage_async_remove_latency_us;
     std::shared_ptr<bvar::LatencyRecorder> _evict_in_advance_latency_us;
     std::shared_ptr<bvar::LatencyRecorder> _recycle_keys_length_recorder;
+    std::shared_ptr<bvar::LatencyRecorder> _ttl_gc_latency_us;
 };
 
 } // namespace doris::io
diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp
index 6aad66de5cc..2c9bfaf2093 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -2898,6 +2898,70 @@ TEST_F(BlockFileCacheTest, remove_directly_when_normal_change_to_ttl) {
     }
 }
 
+TEST_F(BlockFileCacheTest, ttl_gc) {
+    if (fs::exists(cache_base_path)) {
+        fs::remove_all(cache_base_path);
+    }
+    fs::create_directories(cache_base_path);
+    auto sp = SyncPoint::get_instance();
+    SyncPoint::CallbackGuard guard1;
+    sp->enable_processing();
+    TUniqueId query_id;
+    query_id.hi = 1;
+    query_id.lo = 1;
+    io::FileCacheSettings settings;
+    settings.query_queue_size = 50;
+    settings.query_queue_elements = 5;
+    settings.ttl_queue_size = 500;
+    settings.ttl_queue_elements = 500;
+    settings.capacity = 100;
+    settings.max_file_block_size = 30;
+    settings.max_query_cache_size = 30;
+
+    config::file_cache_background_ttl_gc_batch = 6;
+    config::file_cache_background_ttl_gc_interval_ms =
+            3000; // make it big enough to disable auto ttl_gc
+
+    io::BlockFileCache cache(cache_base_path, settings);
+    ASSERT_TRUE(cache.initialize());
+    for (int i = 0; i < 100; i++) {
+        if (cache.get_async_open_success()) {
+            break;
+        };
+        std::this_thread::sleep_for(std::chrono::milliseconds(1));
+    }
+
+    io::CacheContext context;
+    ReadStatistics rstats;
+    context.stats = &rstats;
+    context.cache_type = io::FileCacheType::TTL;
+    context.query_id = query_id;
+    int64_t cur_time = UnixSeconds();
+    context.expiration_time = cur_time + 2;
+
+    for (int64_t i = 0; i < 12; ++i) {
+        auto key = io::BlockFileCache::hash(fmt::format("key{}", i));
+        auto holder = cache.get_or_set(key, 0, 5, context);
+        auto blocks = fromHolder(holder);
+        ASSERT_EQ(blocks.size(), 1);
+        assert_range(1, blocks[0], io::FileBlock::Range(0, 4), io::FileBlock::State::EMPTY);
+        ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id());
+        download(blocks[0]);
+        assert_range(1, blocks[0], io::FileBlock::Range(0, 4), io::FileBlock::State::DOWNLOADED);
+    }
+    ASSERT_EQ(cache._time_to_key.size(), 12);
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+    ASSERT_GT(cache._time_to_key.size(), 0);
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(3000));
+    ASSERT_EQ(cache._time_to_key.size(), 0);
+
+    if (fs::exists(cache_base_path)) {
+        fs::remove_all(cache_base_path);
+    }
+}
+
 TEST_F(BlockFileCacheTest, recyle_cache_async) {
     if (fs::exists(cache_base_path)) {
         fs::remove_all(cache_base_path);


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org