[kudu] branch master updated: KUDU-3181: put a bound on the queue of compilation manager

[al...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git


The following commit(s) were added to refs/heads/master by this push:
     new c444e8d  KUDU-3181: put a bound on the queue of compilation manager
c444e8d is described below

commit c444e8dcbc9dbdef3d599fe4858d740ff91b2045
Author: Li Zhiming <li...@thinkingdata.cn>
AuthorDate: Wed Aug 12 10:46:00 2020 +0800

    KUDU-3181: put a bound on the queue of compilation manager
    
    Add flag codegen_queue_capacity to config max queue size.
    Without the bound, when codegen cache hit rate is low, the queue
    could accumulate tons of entries, each of which retains
    a copy of schema meta data, thus consuming a lot of memeory.
    
    Change-Id: Ia85b5818d1a5ea3ea978808900dd9c9e0cc0784e
    Reviewed-on: http://gerrit.cloudera.org:8080/16330
    Reviewed-by: Andrew Wong <aw...@cloudera.com>
    Tested-by: Andrew Wong <aw...@cloudera.com>
    Reviewed-by: Alexey Serbin <as...@cloudera.com>
---
 src/kudu/codegen/compilation_manager.cc | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/kudu/codegen/compilation_manager.cc b/src/kudu/codegen/compilation_manager.cc
index 1f76392..191263c 100644
--- a/src/kudu/codegen/compilation_manager.cc
+++ b/src/kudu/codegen/compilation_manager.cc
@@ -35,6 +35,7 @@
 #include "kudu/gutil/ref_counted.h"
 #include "kudu/util/faststring.h"
 #include "kudu/util/flag_tags.h"
+#include "kudu/util/logging.h"
 #include "kudu/util/metrics.h"
 #include "kudu/util/monotime.h"
 #include "kudu/util/slice.h"
@@ -55,6 +56,10 @@ DEFINE_int32(codegen_cache_capacity, 100, "Number of entries which may be stored
              "code generation cache.");
 TAG_FLAG(codegen_cache_capacity, experimental);
 
+DEFINE_int32(codegen_queue_capacity, 100, "Number of tasks which may be put in the code "
+             "generation task queue.");
+TAG_FLAG(codegen_queue_capacity, experimental);
+
 METRIC_DEFINE_gauge_int64(server, code_cache_hits, "Codegen Cache Hits",
                           kudu::MetricUnit::kCacheHits,
                           "Number of codegen cache hits since start",
@@ -132,6 +137,7 @@ CompilationManager::CompilationManager()
   CHECK_OK(ThreadPoolBuilder("compiler_manager_pool")
            .set_min_threads(0)
            .set_max_threads(1)
+           .set_max_queue_size(FLAGS_codegen_queue_capacity)
            .set_idle_timeout(MonoDelta::FromMilliseconds(kThreadTimeoutMs))
            .Build(&pool_));
   // We call std::atexit after the implicit default construction of
@@ -174,7 +180,7 @@ bool CompilationManager::RequestRowProjector(const Schema* base_schema,
                                              unique_ptr<RowProjector>* out) {
   faststring key;
   Status s = RowProjectorFunctions::EncodeKey(*base_schema, *projection, &key);
-  WARN_NOT_OK(s, "RowProjector compilation request failed");
+  WARN_NOT_OK(s, "RowProjector compilation request encode key failed");
   if (!s.ok()) return false;
   query_counter_.Increment();
 
@@ -185,8 +191,8 @@ bool CompilationManager::RequestRowProjector(const Schema* base_schema,
   if (!cached) {
     shared_ptr<CompilationTask> task(make_shared<CompilationTask>(
         *base_schema, *projection, &cache_, &generator_));
-    WARN_NOT_OK(pool_->Submit([task]() { task->Run(); }),
-                "RowProjector compilation request failed");
+    WARN_NOT_OK_EVERY_N_SECS(pool_->Submit([task]() { task->Run(); }),
+                    "RowProjector compilation request submit failed", 10);
     return false;
   }