couch-replicator commit: updated refs/heads/master to f195535

[rn...@apache.org]

Repository: couchdb-couch-replicator
Updated Branches:
  refs/heads/master a6fd29328 -> f195535e7


Use connection pools for _changes requests

In high-throughput scenarios the process of setting up new HTTPS connections can
bottleneck replication performance. The #htttpdb.httpc_pool construct alleviates
most of this pain, but _changes requests don't use them.

The decision to not use connection pools for _changes requests is an old one. As
far as I can discern it is/was related to the use of HTTP pipelining in the
original implementation of couch_replicator. Pipelining is long-gone, but
_changes requests still set up their own connections.

This patch reconfigures replications to use the available connection pool for
_changes requests, along with associated cleanup of related code now rendered
unnecessary.


Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/f195535e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/f195535e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/f195535e

Branch: refs/heads/master
Commit: f195535e72383468428a5bafbd558cd9e565d9c4
Parents: a6fd293
Author: Benjamin Anderson <b...@banjiewen.net>
Authored: Wed Sep 24 11:04:46 2014 -0700
Committer: Robert Newson <rn...@apache.org>
Committed: Thu Oct 30 18:18:21 2014 +0000

----------------------------------------------------------------------
 src/couch_replicator_httpc.erl | 29 ++++++++++-------------------
 1 file changed, 10 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/f195535e/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index 985f0c7..9a6213a 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -40,14 +40,17 @@ send_req(HttpDb, Params1, Callback) ->
         [{K, ?b2l(iolist_to_binary(V))} || {K, V} <- get_value(qs, Params1, [])]),
     Params = ?replace(Params2, ibrowse_options,
         lists:keysort(1, get_value(ibrowse_options, Params2, []))),
-    {Worker, Response, IsChanges} = send_ibrowse_req(HttpDb, Params),
+    {Worker, Response} = send_ibrowse_req(HttpDb, Params),
     Ret = try
         process_response(Response, Worker, HttpDb, Params, Callback)
     catch
         throw:{retry, NewHttpDb0, NewParams0} ->
             {retry, NewHttpDb0, NewParams0}
     after
-        release_worker(Worker, HttpDb, IsChanges),
+        ok = couch_replicator_httpc_pool:release_worker(
+            HttpDb#httpdb.httpc_pool,
+            Worker
+        ),
         clean_mailbox(Response)
     end,
     % This is necessary to keep this tail-recursive. Calling
@@ -68,17 +71,16 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb, Params) ->
     Headers2 = oauth_header(HttpDb, Params) ++ Headers1,
     Url = full_url(HttpDb, Params),
     Body = get_value(body, Params, []),
-    IsChanges = get_value(path, Params) == "_changes",
-    if IsChanges ->
-        {ok, Worker} = ibrowse:spawn_link_worker_process(Url),
-        Timeout = infinity;
+    case get_value(path, Params) == "_changes" of
     true ->
-        {ok, Worker} = couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool),
+        Timeout = infinity;
+    false ->
         Timeout = case config:get("replicator", "request_timeout", "infinity") of
             "infinity" -> infinity;
             Milliseconds -> list_to_integer(Milliseconds)
         end
     end,
+    {ok, Worker} = couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool),
     IbrowseOptions = [
         {response_format, binary}, {inactivity_timeout, HttpDb#httpdb.timeout} |
         lists:ukeymerge(1, get_value(ibrowse_options, Params, []),
@@ -86,7 +88,7 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb, Params) ->
     ],
     Response = ibrowse:send_req_direct(
         Worker, Url, Headers2, Method, Body, IbrowseOptions, Timeout),
-    {Worker, Response, IsChanges}.
+    {Worker, Response}.
 
 
 process_response({error, sel_conn_closed}, _Worker, HttpDb, Params, _Cb) ->
@@ -177,17 +179,6 @@ clean_mailbox(_) ->
     ok.
 
 
-release_worker(Worker, _, true) ->
-    true = unlink(Worker),
-    ibrowse_http_client:stop(Worker),
-    receive
-        {'EXIT', Worker, _} -> ok
-        after 0 -> ok
-    end;
-release_worker(Worker, #httpdb{httpc_pool = Pool}, false) ->
-    ok = couch_replicator_httpc_pool:release_worker(Pool, Worker).
-
-
 maybe_retry(Error, Worker, #httpdb{retries = 0} = HttpDb, Params) ->
     report_error(Worker, HttpDb, Params, {error, Error});