[26/50] fabric commit: updated refs/heads/2971-count-distinct to 5d18415

[ko...@apache.org]

Track open_shard timeouts with a counter

The open_shard RPC endpoint is used to grab security docs. There are
fairly aggressive timeouts on these requests so that when a node is too
busy it'll try the next shard. Rather than log everytime these fail
(which can be substantial under load) lets just use a counter that can
be graphed and alerted on.

COUCHDB-3234


Project: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/commit/4fcfecf7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/tree/4fcfecf7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-fabric/diff/4fcfecf7

Branch: refs/heads/2971-count-distinct
Commit: 4fcfecf720791f400169c6823aac1abc2d85c47e
Parents: 7cfabb5
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Nov 11 11:34:11 2016 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Fri Nov 11 11:38:00 2016 -0600

----------------------------------------------------------------------
 priv/stats_descriptions.cfg | 4 ++++
 src/fabric_rpc.erl          | 6 +++++-
 2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/4fcfecf7/priv/stats_descriptions.cfg
----------------------------------------------------------------------
diff --git a/priv/stats_descriptions.cfg b/priv/stats_descriptions.cfg
index 0c2351b..d12aa0c 100644
--- a/priv/stats_descriptions.cfg
+++ b/priv/stats_descriptions.cfg
@@ -2,6 +2,10 @@
     {type, counter},
     {desc, <<"number of worker timeouts">>}
 ]}.
+{[fabric, open_shard, timeouts], [
+    {type, counter},
+    {desc, <<"number of open shard timeouts">>}
+]}.
 {[fabric, read_repairs, success], [
     {type, counter},
     {desc, <<"number of successful read repair operations">>}

http://git-wip-us.apache.org/repos/asf/couchdb-fabric/blob/4fcfecf7/src/fabric_rpc.erl
----------------------------------------------------------------------
diff --git a/src/fabric_rpc.erl b/src/fabric_rpc.erl
index 3a21d83..9c7d518 100644
--- a/src/fabric_rpc.erl
+++ b/src/fabric_rpc.erl
@@ -253,7 +253,11 @@ reset_validation_funs(DbName) ->
 
 open_shard(Name, Opts) ->
     set_io_priority(Name, Opts),
-    rexi:reply(couch_db:open(Name, Opts)).
+    try
+        rexi:reply(couch_db:open(Name, Opts))
+    catch exit:{timeout, _} ->
+        couch_stats:increment_counter([fabric, open_shard, timeouts])
+    end.
 
 compact(DbName) ->
     with_db(DbName, [], {couch_db, start_compact, []}).