You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2017/03/14 19:26:24 UTC
[29/50] couch-replicator commit: updated refs/heads/63012-scheduler
to 27a5eae
Share replicator connections by proxy
Prior to this commit, proxied and unproxied replication connections
would be shared based on their source or target. This caused a bug where
a proxied connection could function as an unproxied connection if an
idle unproxied connection was shared with it.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/58ddc265
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/58ddc265
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/58ddc265
Branch: refs/heads/63012-scheduler
Commit: 58ddc26531789320e00d497c4fa3b95dea618d36
Parents: 1413a6f
Author: Benjamin Bastian <be...@gmail.com>
Authored: Mon Nov 21 09:57:30 2016 -0800
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Wed Nov 23 13:49:52 2016 -0800
----------------------------------------------------------------------
src/couch_replicator_api_wrap.hrl | 3 ++-
src/couch_replicator_connection.erl | 7 ++++++-
src/couch_replicator_docs.erl | 32 ++++++++++++++++++--------------
src/couch_replicator_httpc.erl | 14 ++++++++++++--
src/couch_replicator_scheduler.erl | 9 ++++++++-
5 files changed, 46 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_api_wrap.hrl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.hrl b/src/couch_replicator_api_wrap.hrl
index d15d214..fc94054 100644
--- a/src/couch_replicator_api_wrap.hrl
+++ b/src/couch_replicator_api_wrap.hrl
@@ -25,7 +25,8 @@
wait = 250, % milliseconds
httpc_pool = nil,
http_connections,
- first_error_timestamp = nil
+ first_error_timestamp = nil,
+ proxy_url
}).
-record(oauth, {
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_connection.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_connection.erl b/src/couch_replicator_connection.erl
index b776624..e580663 100644
--- a/src/couch_replicator_connection.erl
+++ b/src/couch_replicator_connection.erl
@@ -55,6 +55,9 @@ init([]) ->
{ok, #state{close_interval=Interval, timer=Timer}}.
+acquire(URL) when is_binary(URL) ->
+ acquire(binary_to_list(URL));
+
acquire(URL) ->
case gen_server:call(?MODULE, {acquire, URL}) of
{ok, Worker} ->
@@ -85,7 +88,9 @@ handle_call({acquire, URL}, From, State) ->
couch_stats:increment_counter([couch_replicator, connection, acquires]),
ets:insert(?MODULE, Worker#connection{mref=monitor(process, Pid)}),
{reply, {ok, Worker#connection.worker}, State}
- end
+ end;
+ {error, invalid_uri} ->
+ {reply, {error, invalid_uri}, State}
end;
handle_call({create, URL, Worker}, From, State) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_docs.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_docs.erl b/src/couch_replicator_docs.erl
index 07344bb..204cb39 100644
--- a/src/couch_replicator_docs.erl
+++ b/src/couch_replicator_docs.erl
@@ -259,17 +259,15 @@ parse_rep_doc(Doc, UserCtx) ->
-spec parse_rep_doc_without_id({[_]}, #user_ctx{}) -> {ok, #rep{}}.
parse_rep_doc_without_id({Props}, UserCtx) ->
- ProxyParams = parse_proxy_params(get_value(<<"proxy">>, Props, <<>>)),
+ Proxy = get_value(<<"proxy">>, Props, <<>>),
Opts = make_options(Props),
case get_value(cancel, Opts, false) andalso
(get_value(id, Opts, nil) =/= nil) of
true ->
{ok, #rep{options = Opts, user_ctx = UserCtx}};
false ->
- Source = parse_rep_db(get_value(<<"source">>, Props),
- ProxyParams, Opts),
- Target = parse_rep_db(get_value(<<"target">>, Props),
- ProxyParams, Opts),
+ Source = parse_rep_db(get_value(<<"source">>, Props), Proxy, Opts),
+ Target = parse_rep_db(get_value(<<"target">>, Props), Proxy, Opts),
{Type, View} = case couch_replicator_filters:view_type(Props, Opts) of
{error, Error} ->
throw({bad_request, Error});
@@ -380,8 +378,13 @@ rep_user_ctx({RepDoc}) ->
}
end.
--spec parse_rep_db({[_]} | binary(), [_], [_]) -> #httpd{} | binary().
-parse_rep_db({Props}, ProxyParams, Options) ->
+-spec parse_rep_db({[_]} | binary(), binary(), [_]) -> #httpd{} | binary().
+parse_rep_db({Props}, Proxy, Options) ->
+ ProxyParams = parse_proxy_params(Proxy),
+ ProxyURL = case ProxyParams of
+ [] -> undefined;
+ _ -> binary_to_list(Proxy)
+ end,
Url = maybe_add_trailing_slash(get_value(<<"url">>, Props)),
{AuthProps} = get_value(<<"auth">>, Props, {[]}),
{BinHeaders} = get_value(<<"headers">>, Props, {[]}),
@@ -414,15 +417,16 @@ parse_rep_db({Props}, ProxyParams, Options) ->
ProxyParams ++ ssl_params(Url)]),
timeout = get_value(connection_timeout, Options),
http_connections = get_value(http_connections, Options),
- retries = get_value(retries, Options)
+ retries = get_value(retries, Options),
+ proxy_url = ProxyURL
};
-parse_rep_db(<<"http://", _/binary>> = Url, ProxyParams, Options) ->
- parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
-parse_rep_db(<<"https://", _/binary>> = Url, ProxyParams, Options) ->
- parse_rep_db({[{<<"url">>, Url}]}, ProxyParams, Options);
-parse_rep_db(<<DbName/binary>>, _ProxyParams, _Options) ->
+parse_rep_db(<<"http://", _/binary>> = Url, Proxy, Options) ->
+ parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options);
+parse_rep_db(<<"https://", _/binary>> = Url, Proxy, Options) ->
+ parse_rep_db({[{<<"url">>, Url}]}, Proxy, Options);
+parse_rep_db(<<DbName/binary>>, _Proxy, _Options) ->
DbName;
-parse_rep_db(undefined, _ProxyParams, _Options) ->
+parse_rep_db(undefined, _Proxy, _Options) ->
throw({error, <<"Missing replicator database">>}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index 5fb7842..58fb0e1 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -40,8 +40,18 @@
-define(MAX_DISCARDED_MESSAGES, 16).
-setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
- {ok, Pid} = couch_replicator_httpc_pool:start_link(Url, [{max_connections, MaxConns}]),
+setup(Db) ->
+ #httpdb{
+ httpc_pool = nil,
+ url = Url,
+ http_connections = MaxConns,
+ proxy_url = ProxyURL
+ } = Db,
+ HttpcURL = case ProxyURL of
+ undefined -> Url;
+ _ when is_list(ProxyURL) -> ProxyURL
+ end,
+ {ok, Pid} = couch_replicator_httpc_pool:start_link(HttpcURL, [{max_connections, MaxConns}]),
{ok, Db#httpdb{httpc_pool = Pid}}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/58ddc265/src/couch_replicator_scheduler.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_scheduler.erl b/src/couch_replicator_scheduler.erl
index 440458e..f08e829 100644
--- a/src/couch_replicator_scheduler.erl
+++ b/src/couch_replicator_scheduler.erl
@@ -119,6 +119,12 @@ job_summary(JobId, HealthThreshold) ->
{Pid, ErrorCount} when is_pid(Pid) ->
{running, null}
end,
+ StrippedProxyURL = case (Rep#rep.source)#httpdb.proxy_url of
+ undefined ->
+ null;
+ ProxyURL when is_list(ProxyURL) ->
+ list_to_binary(couch_util:url_strip_password(ProxyURL))
+ end,
[
{source, iolist_to_binary(ejson_url(Rep#rep.source))},
{target, iolist_to_binary(ejson_url(Rep#rep.target))},
@@ -126,7 +132,8 @@ job_summary(JobId, HealthThreshold) ->
{info, Info},
{error_count, ErrorCount},
{last_updated, last_updated(History)},
- {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)}
+ {start_time, couch_replicator_utils:iso8601(Rep#rep.start_time)},
+ {proxy, StrippedProxyURL}
];
{error, not_found} ->
nil % Job might have just completed