[couchdb] branch prototype/fdb-layer updated: Improve load handling in couch_jobs and couch_views

[va...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vatamane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/prototype/fdb-layer by this push:
     new 5bade6f  Improve load handling in couch_jobs and couch_views
5bade6f is described below

commit 5bade6fde84c46c472ab64967563cb07a239f026
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu May 21 18:40:55 2020 -0400

    Improve load handling in couch_jobs and couch_views
    
    Increase couch_views job timeout by 20 seconds. This will set a larger jitter
    when multiple nodes concurrently check and re-equeue jobs. It would reduce the
    chance of them bumping into each other and conflicting.
    
    If they do conflict in activity monitor, catch the error and emit an error log.
    We gain some more robustness under load for a longer timeout for jobs with
    workers that have suddenly died getting re-enqueued.
---
 src/couch_jobs/src/couch_jobs_activity_monitor.erl | 9 ++++++++-
 src/couch_views/src/couch_views_jobs.erl           | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/couch_jobs/src/couch_jobs_activity_monitor.erl b/src/couch_jobs/src/couch_jobs_activity_monitor.erl
index ef82e6b..6f50d96 100644
--- a/src/couch_jobs/src/couch_jobs_activity_monitor.erl
+++ b/src/couch_jobs/src/couch_jobs_activity_monitor.erl
@@ -65,7 +65,14 @@ handle_cast(Msg, St) ->
 
 
 handle_info(check_activity, St) ->
-    St1 = check_activity(St),
+    St1 = try
+        check_activity(St)
+    catch
+        {error, {erlfdb_error, 1020}} ->
+            LogMsg = "~p : type:~p got 1020 error, possibly from overload",
+            couch_log:error(LogMsg, [?MODULE, St#st.type]),
+            St
+    end,
     St2 = schedule_check(St1),
     {noreply, St2};
 
diff --git a/src/couch_views/src/couch_views_jobs.erl b/src/couch_views/src/couch_views_jobs.erl
index 909e923..a9ca168 100644
--- a/src/couch_views/src/couch_views_jobs.erl
+++ b/src/couch_views/src/couch_views_jobs.erl
@@ -31,7 +31,7 @@
 
 
 set_timeout() ->
-    couch_jobs:set_type_timeout(?INDEX_JOB_TYPE, 6).
+    couch_jobs:set_type_timeout(?INDEX_JOB_TYPE, 26).
 
 
 build_view(TxDb, Mrst, UpdateSeq) ->