You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by fd...@apache.org on 2011/10/04 16:41:56 UTC
git commit: Replicator: configurable # of retries per request
Updated Branches:
refs/heads/master fb5d36fd5 -> 37afbce7c
Replicator: configurable # of retries per request
Instead of using an hardcoded value of 10, this parameter is
now configurable via .ini configuration or per replication
in the replication document/object (field "retries_per_request").
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/37afbce7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/37afbce7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/37afbce7
Branch: refs/heads/master
Commit: 37afbce7cf80289783b2eb813d023fbde4ad69fd
Parents: fb5d36f
Author: Filipe David Manana <fd...@apache.org>
Authored: Tue Oct 4 15:39:28 2011 -0700
Committer: Filipe David Manana <fd...@apache.org>
Committed: Tue Oct 4 15:39:28 2011 -0700
----------------------------------------------------------------------
etc/couchdb/default.ini.tpl.in | 2 ++
src/couchdb/couch_api_wrap_httpc.erl | 4 +++-
src/couchdb/couch_replicator.erl | 2 ++
src/couchdb/couch_replicator_utils.erl | 11 ++++++++---
4 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/etc/couchdb/default.ini.tpl.in
----------------------------------------------------------------------
diff --git a/etc/couchdb/default.ini.tpl.in b/etc/couchdb/default.ini.tpl.in
index 1e8c88a..12bc20f 100644
--- a/etc/couchdb/default.ini.tpl.in
+++ b/etc/couchdb/default.ini.tpl.in
@@ -173,6 +173,8 @@ http_connections = 20
; Even for very fast/reliable networks it might need to be increased if a remote
; database is too busy.
connection_timeout = 30000
+; If a request fails, the replicator will retry it up to N times.
+retries_per_request = 10
; Some socket options that might boost performance in some scenarios:
; {nodelay, boolean()}
; {sndbuf, integer()}
http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_api_wrap_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_api_wrap_httpc.erl b/src/couchdb/couch_api_wrap_httpc.erl
index bff581f..f1ef7d3 100644
--- a/src/couchdb/couch_api_wrap_httpc.erl
+++ b/src/couchdb/couch_api_wrap_httpc.erl
@@ -26,6 +26,7 @@
]).
-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
+-define(MAX_WAIT, 5 * 60 * 1000).
setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
@@ -156,7 +157,8 @@ maybe_retry(Error, Worker, #httpdb{retries = Retries, wait = Wait} = HttpDb,
?LOG_INFO("Retrying ~s request to ~s in ~p seconds due to error ~s",
[Method, Url, Wait / 1000, error_cause(Error)]),
ok = timer:sleep(Wait),
- send_req(HttpDb#httpdb{retries = Retries - 1, wait = Wait * 2}, Params, Cb).
+ Wait2 = erlang:min(Wait * 2, ?MAX_WAIT),
+ send_req(HttpDb#httpdb{retries = Retries - 1, wait = Wait2}, Params, Cb).
report_error(Worker, HttpDb, Params, Error) ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_replicator.erl b/src/couchdb/couch_replicator.erl
index efea373..40cb9a4 100644
--- a/src/couchdb/couch_replicator.erl
+++ b/src/couchdb/couch_replicator.erl
@@ -306,9 +306,11 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
"~ca worker batch size of ~p~n"
"~c~p HTTP connections~n"
"~ca connection timeout of ~p milliseconds~n"
+ "~c~p retries per request~n"
"~csocket options are: ~s~s",
[BaseId ++ Ext, $\t, NumWorkers, $\t, BatchSize, $\t,
MaxConns, $\t, get_value(connection_timeout, Options),
+ $\t, get_value(retries, Options),
$\t, io_lib:format("~p", [get_value(socket_options, Options)]),
case StartSeq of
?LOWEST_SEQ ->
http://git-wip-us.apache.org/repos/asf/couchdb/blob/37afbce7/src/couchdb/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_replicator_utils.erl b/src/couchdb/couch_replicator_utils.erl
index 3c61400..07630e7 100644
--- a/src/couchdb/couch_replicator_utils.erl
+++ b/src/couchdb/couch_replicator_utils.erl
@@ -193,7 +193,8 @@ parse_rep_db({Props}, ProxyParams, Options) ->
[{socket_options, get_value(socket_options, Options)} |
ProxyParams ++ ssl_params(Url)]),
timeout = get_value(connection_timeout, Options),
- http_connections = get_value(http_connections, Options)
+ http_connections = get_value(http_connections, Options),
+ retries = get_value(retries, Options)
};
parse_rep_db(<<"http://", _/binary>> = Url, ProxyParams, Options) ->
parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
@@ -220,16 +221,18 @@ make_options(Props) ->
DefBatchSize = couch_config:get("replicator", "worker_batch_size", "500"),
DefConns = couch_config:get("replicator", "http_connections", "20"),
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
+ DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
- lists:ukeymerge(1, Options, [
+ lists:ukeymerge(1, Options, lists:keysort(1, [
{connection_timeout, list_to_integer(DefTimeout)},
+ {retries, list_to_integer(DefRetries)},
{http_connections, list_to_integer(DefConns)},
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
{worker_processes, list_to_integer(DefWorkers)}
- ]).
+ ])).
convert_options([])->
@@ -261,6 +264,8 @@ convert_options([{<<"http_connections">>, V} | R]) ->
[{http_connections, couch_util:to_integer(V)} | convert_options(R)];
convert_options([{<<"connection_timeout">>, V} | R]) ->
[{connection_timeout, couch_util:to_integer(V)} | convert_options(R)];
+convert_options([{<<"retries_per_request">>, V} | R]) ->
+ [{retries, couch_util:to_integer(V)} | convert_options(R)];
convert_options([{<<"socket_options">>, V} | R]) ->
{ok, SocketOptions} = couch_util:parse_term(V),
[{socket_options, SocketOptions} | convert_options(R)];