You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2021/06/03 18:28:13 UTC
[couchdb] branch
improve-basic-auth-creds-passing-in-replicator-main created (now 787e35b)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch improve-basic-auth-creds-passing-in-replicator-main
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 787e35b Improve basic auth credentials handling in replicator
This branch includes the following new commits:
new 787e35b Improve basic auth credentials handling in replicator
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Improve basic auth credentials handling in
replicator
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch improve-basic-auth-creds-passing-in-replicator-main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 787e35b5de45b9a2e9d519d2b5864c59fe7d5689
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Jun 3 14:09:43 2021 -0400
Improve basic auth credentials handling in replicator
This is a port of commit ecd266b0e87f44e1080cabdb4c28e4758f5a4406 from 3.x to
main. Including the same commit message from there for completeness and then
towards the end, there is a description of changes required to port the PR to
main.
Previously, there were two ways to pass in basic auth credentials for endpoints
-- using URL's userinfo part, and encoding them in an `"Authorization": "basic
..."` header. Neither one is ideal for these reasons:
* Passwords in userinfo don't allow using ":", "@" and other characters.
However, even after switching to always unquoting them like we did recently
[1], it could break authentication for usernames or passwords previously
containing "+" or "%HH" patterns, as "+" might now be decoded to a " ".
* Base64 encoded headers need an extra step to encode them. Also, quite often
these encoded headers are confused as being "encrypted" and shared in a
clear channel.
To improve this, revert the recent commit to unquote URL userinfo parts to
restore backwards compatibility, and introduce a way to pass in basic auth
credentials in the "auth" object. The "auth" object was already added a while
back to allow authentication plugins to store their credentials in it. The
format is:
```
"source": {
"url": "https://host/db",
"auth": {
"basic": {
"username":"myuser",
"password":"mypassword"
}
}
}
```
{"auth" : "basic" : {...}} object is checked first, and if credentials are
provided, they will be used. If they are not then userinfo and basic auth
header will be parsed.
Internally, there was a good amount duplication related to parsing credentials
from userinfo and headers in replication ID generation logic and in the auth
session plugin. As a cleanup, consolidate that logic in the
`couch_replicator_parse` module.
The commit is quite different from the 3.x one for these two reasons:
* `main` uses two types of replication endpoint "objects": `#httpdb` records
and `HttpDb` maps. In most cases it uses maps which can be serialized and
deserialized to and from json. But in lower level, connection handling code
in couch_replicator_httpc, it uses `#httpdb` records. This explain the need
to still handle both representations. Auth session plugin, for instance,
uses the lower level #httpdb records while replicator ID handling code uses
the map based one.
* `main` has all the parsing of replication documents and `_replicate` request
bodies in a separate `couch_replicator_parse`. So, most of the code which
handles normalizing basic auth creds is there instead of
`couch_replicator_docs` or `couch_replicator_utils` like it is in 3.x
---
.../src/couch_replicator_auth_session.erl | 238 ++++----------
.../src/couch_replicator_httpc.erl | 8 +-
src/couch_replicator/src/couch_replicator_ids.erl | 114 ++++---
src/couch_replicator/src/couch_replicator_job.erl | 4 +-
.../src/couch_replicator_parse.erl | 355 ++++++++++++++++++++-
.../src/couch_replicator_utils.erl | 121 +++----
6 files changed, 521 insertions(+), 319 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_auth_session.erl b/src/couch_replicator/src/couch_replicator_auth_session.erl
index 6ca30c8..b2bf317 100644
--- a/src/couch_replicator/src/couch_replicator_auth_session.erl
+++ b/src/couch_replicator/src/couch_replicator_auth_session.erl
@@ -79,7 +79,6 @@
-type headers() :: [{string(), string()}].
-type code() :: non_neg_integer().
--type creds() :: {string() | undefined, string() | undefined}.
-type time_sec() :: non_neg_integer().
-type age() :: time_sec() | undefined.
@@ -246,59 +245,15 @@ init_state(#httpdb{} = HttpDb) ->
-spec extract_creds(#httpdb{}) ->
{ok, string(), string(), #httpdb{}} | {error, term()}.
-extract_creds(#httpdb{url = Url, headers = Headers} = HttpDb) ->
- {{HeadersUser, HeadersPass}, HeadersNoCreds} =
- couch_replicator_utils:remove_basic_auth_from_headers(Headers),
- case extract_creds_from_url(Url) of
- {ok, UrlUser, UrlPass, UrlNoCreds} ->
- case pick_creds({UrlUser, UrlPass}, {HeadersUser, HeadersPass}) of
- {ok, User, Pass} ->
- HttpDb1 = HttpDb#httpdb{
- url = UrlNoCreds,
- headers = HeadersNoCreds
- },
- {ok, User, Pass, HttpDb1};
- {error, Error} ->
- {error, Error}
- end;
- {error, Error} ->
- {error, Error}
- end.
-
-
-% Credentials could be specified in the url and/or in the headers.
-% * If no credentials specified return error.
-% * If specified in url but not in headers, pick url creds.
-% * Otherwise pick headers creds.
-%
--spec pick_creds(creds(), creds()) ->
- {ok, string(), string()} | {error, missing_credentials}.
-pick_creds({undefined, _}, {undefined, _}) ->
- {error, missing_credentials};
-pick_creds({UrlUser, UrlPass}, {undefined, _}) ->
- {ok, UrlUser, UrlPass};
-pick_creds({_, _}, {HeadersUser, HeadersPass}) ->
- {ok, HeadersUser, HeadersPass}.
-
-
--spec extract_creds_from_url(string()) ->
- {ok, string() | undefined, string() | undefined, string()} |
- {error, term()}.
-extract_creds_from_url(Url) ->
- case ibrowse_lib:parse_url(Url) of
- {error, Error} ->
- {error, Error};
- #url{username = undefined, password = undefined} ->
- {ok, undefined, undefined, Url};
- #url{protocol = Proto, username = User, password = Pass} ->
- % Excise user and pass parts from the url. Try to keep the host,
- % port and path as they were in the original.
- Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
- Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
- NoCreds = lists:concat([Proto, "://", Suffix]),
- User1 = chttpd:unquote(User),
- Pass1 = chttpd:unquote(Pass),
- {ok, User1, Pass1, NoCreds}
+extract_creds(#httpdb{} = HttpDb) ->
+ case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+ {undefined, undefined} ->
+ % Return error. Session plugin should ignore this replication
+ % endpoint as there are no valid creds which can be used
+ {error, missing_credentials};
+ {User, Pass} when is_list(User), is_list(Pass) ->
+ HttpDb1 = remove_basic_auth_creds(HttpDb),
+ {ok, User, Pass, HttpDb1}
end.
@@ -567,6 +522,11 @@ b64creds(User, Pass) ->
base64:encode_to_string(User ++ ":" ++ Pass).
+remove_basic_auth_creds(#httpdb{auth_props = Props} = HttpDb) ->
+ Props1 = maps:remove(<<"basic">>, Props),
+ HttpDb#httpdb{auth_props = Props1}.
+
+
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
@@ -582,116 +542,15 @@ get_session_url_test_() ->
]].
-extract_creds_success_test_() ->
- DefaultHeaders = (#httpdb{})#httpdb.headers,
- [?_assertEqual({ok, User, Pass, HttpDb2}, extract_creds(HttpDb1)) ||
- {HttpDb1, {User, Pass, HttpDb2}} <- [
- {
- #httpdb{url = "http://u:p@x.y/db"},
- {"u", "p", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40:p%40@x.y/db"},
- {"u@", "p@", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40u:p%40p@x.y/db"},
- {"u@u", "p@p", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40%401:p%40%401@x.y/db"},
- {"u@@1", "p@@1", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u%40%2540:p%40%2540@x.y/db"},
- {"u@%40", "p@%40", #httpdb{url = "http://x.y/db"}}
- },
- {
- #httpdb{url = "http://u:p@h:80/db"},
- {"u", "p", #httpdb{url = "http://h:80/db"}}
- },
- {
- #httpdb{url = "http://u%3A:p%3A@h:80/db"},
- {"u:", "p:", #httpdb{url = "http://h:80/db"}}
- },
- {
- #httpdb{url = "https://u:p@h/db"},
- {"u", "p", #httpdb{url = "https://h/db"}}
- },
- {
- #httpdb{url = "https://u%2F:p%2F@h/db"},
- {"u/", "p/", #httpdb{url = "https://h/db"}}
- },
- {
- #httpdb{url = "http://u:p@127.0.0.1:5984/db"},
- {"u", "p", #httpdb{url = "http://127.0.0.1:5984/db"}}
- },
- {
- #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]/db"},
- {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]/db"}}
- },
- {
- #httpdb{url = "http://u:p@[2001:db8:a1b:12f9::1]:81/db"},
- {"u", "p", #httpdb{url = "http://[2001:db8:a1b:12f9::1]:81/db"}}
- },
- {
- #httpdb{url = "http://u:p%3A%2F%5B%5D%40@[2001:db8:a1b:12f9::1]:81/db"},
- {"u", "p:/[]@", #httpdb{url = "http://[2001:db8:a1b:12f9::1]:81/db"}}
- },
- {
- #httpdb{url = "http://u:p@x.y/db/other?query=Z&query=w"},
- {"u", "p", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
- },
- {
- #httpdb{url = "http://u:p%3F@x.y/db/other?query=Z&query=w"},
- {"u", "p?", #httpdb{url = "http://x.y/db/other?query=Z&query=w"}}
- },
- {
- #httpdb{
- url = "http://h/db",
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p")}
- ]
- },
- {"u", "p", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db",
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p@")}
- ]
- },
- {"u", "p@", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db",
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u", "p@%40")}
- ]
- },
- {"u", "p@%40", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://h/db",
- headers = DefaultHeaders ++ [
- {"aUthoriZation", "bASIC " ++ b64creds("U", "p")}
- ]
- },
- {"U", "p", #httpdb{url = "http://h/db"}}
- },
- {
- #httpdb{
- url = "http://u1:p1@h/db",
- headers = DefaultHeaders ++ [
- {"Authorization", "Basic " ++ b64creds("u2", "p2")}
- ]
- },
- {"u2", "p2", #httpdb{url = "http://h/db"}}
+extract_creds_success_test() ->
+ HttpDb = #httpdb{auth_props = #{
+ <<"basic">> => #{
+ <<"username">> => <<"u2">>,
+ <<"password">> => <<"p2">>
}
- ]].
+ }},
+ ?assertEqual({ok, "u2", "p2", #httpdb{}}, extract_creds(HttpDb)),
+ ?assertEqual({error, missing_credentials}, extract_creds(#httpdb{})).
cookie_update_test_() ->
@@ -808,7 +667,7 @@ t_process_ok_no_cookie() ->
t_init_state_fails_on_401() ->
?_test(begin
mock_http_401_response(),
- {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+ {error, Error} = init_state(httpdb("http://u:p@h")),
SessionUrl = "http://h/_session",
?assertEqual({session_request_unauthorized, SessionUrl, "u"}, Error)
end).
@@ -818,32 +677,42 @@ t_init_state_401_with_require_valid_user() ->
?_test(begin
mock_http_401_response_with_require_valid_user(),
?assertMatch({ok, #httpdb{}, #state{cookie = "Cookie"}},
- init_state(#httpdb{url = "http://u:p@h"}))
+ init_state(httpdb("http://u:p@h")))
end).
t_init_state_404() ->
?_test(begin
mock_http_404_response(),
- ?assertEqual(ignore, init_state(#httpdb{url = "http://u:p@h"}))
+ ?assertEqual(ignore, init_state(httpdb("http://u:p@h")))
end).
t_init_state_no_creds() ->
?_test(begin
- ?_assertEqual(ignore, init_state(#httpdb{url = "http://h"}))
+ ?_assertEqual(ignore, init_state(httpdb("http://h")))
end).
t_init_state_http_error() ->
?_test(begin
mock_http_error_response(),
- {error, Error} = init_state(#httpdb{url = "http://u:p@h"}),
+ {error, Error} = init_state(httpdb("http://u:p@h")),
SessionUrl = "http://h/_session",
?assertEqual({session_request_failed, SessionUrl, "u", x}, Error)
end).
+httpdb(Url) when is_list(Url) ->
+ FakeDoc = #{
+ <<"source">> => list_to_binary(Url),
+ <<"target">> => <<"http://somehost">>
+ },
+ {ok, Rep} = couch_replicator_parse:parse_rep(FakeDoc, null),
+ HttpDb = maps:get(<<"source">>, Rep),
+ couch_replicator_api_wrap:db_from_json(HttpDb).
+
+
setup_all() ->
meck:expect(couch_replicator_httpc_pool, get_worker, 1, {ok, worker}),
meck:expect(couch_replicator_httpc_pool, release_worker_sync, 2, ok),
@@ -898,14 +767,6 @@ mock_http_error_response() ->
meck:expect(ibrowse, send_req_direct, 7, {error, x}).
-extract_creds_error_test_() ->
- [?_assertMatch({error, Error}, extract_creds(HttpDb)) ||
- {HttpDb, Error} <- [
- {#httpdb{url = "some_junk"}, invalid_uri},
- {#httpdb{url = "http://h/db"}, missing_credentials}
- ]].
-
-
parse_max_age_test_() ->
[?_assertEqual(R, parse_max_age(mochiweb_headers:make([{"Max-Age", A}])))
|| {A, R} <- [
@@ -922,4 +783,29 @@ parse_max_age_test_() ->
].
+remove_basic_auth_creds_test() ->
+ Check = fun(Props) ->
+ HttpDb = remove_basic_auth_creds(#httpdb{auth_props = Props}),
+ HttpDb#httpdb.auth_props
+ end,
+
+ ?assertEqual(#{}, Check(#{})),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(#{<<"other">> => #{}})),
+
+ ?assertEqual(#{}, Check(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ })),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ },
+ <<"other">> => #{}
+ })).
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_httpc.erl b/src/couch_replicator/src/couch_replicator_httpc.erl
index 28b0f38..59a79cf 100644
--- a/src/couch_replicator/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator/src/couch_replicator_httpc.erl
@@ -119,7 +119,13 @@ send_ibrowse_req(#httpdb{headers = BaseHeaders} = HttpDb0, Params) ->
end
end,
{ok, Worker} = couch_replicator_httpc_pool:get_worker(HttpDb#httpdb.httpc_pool),
- IbrowseOptions = [
+ BasicAuthOpts = case couch_replicator_utils:get_basic_auth_creds(HttpDb) of
+ {undefined, undefined} ->
+ [];
+ {User, Pass} when is_list(User), is_list(Pass) ->
+ [{basic_auth, {User, Pass}}]
+ end,
+ IbrowseOptions = BasicAuthOpts ++ [
{response_format, binary}, {inactivity_timeout, HttpDb#httpdb.timeout} |
lists:ukeymerge(1, get_value(ibrowse_options, Params, []),
HttpDb#httpdb.ibrowse_options)
diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl
index 44b9e47..b781404 100644
--- a/src/couch_replicator/src/couch_replicator_ids.erl
+++ b/src/couch_replicator/src/couch_replicator_ids.erl
@@ -174,22 +174,12 @@ get_rep_endpoint(#{<<"url">> := Url0, <<"headers">> := Headers0}) ->
get_v4_endpoint(#{} = HttpDb) ->
{remote, Url, Headers} = get_rep_endpoint(HttpDb),
- {{UserFromHeaders, _}, HeadersWithoutBasicAuth} =
- couch_replicator_utils:remove_basic_auth_from_headers(Headers),
- {UserFromUrl, Host, NonDefaultPort, Path} = get_v4_url_info(Url),
- User = pick_defined_value([UserFromUrl, UserFromHeaders]),
+ {User, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+ {Host, NonDefaultPort, Path} = get_v4_url_info(Url),
OAuth = undefined, % Keep this to ensure checkpoints don't change
- {remote, User, Host, NonDefaultPort, Path, HeadersWithoutBasicAuth, OAuth}.
+ {remote, User, Host, NonDefaultPort, Path, Headers, OAuth}.
-pick_defined_value(Values) ->
- case [V || V <- Values, V /= undefined] of
- [] ->
- undefined;
- DefinedValues ->
- hd(DefinedValues)
- end.
-
get_v4_url_info(Url) when is_binary(Url) ->
get_v4_url_info(binary_to_list(Url));
@@ -198,16 +188,16 @@ get_v4_url_info(Url) ->
{error, invalid_uri} ->
% Tolerate errors here to avoid a bad user document
% crashing the replicator
- {undefined, Url, undefined, undefined};
+ {Url, undefined, undefined};
#url{
protocol = Schema,
- username = User,
+
host = Host,
port = Port,
path = Path
} ->
NonDefaultPort = get_non_default_port(Schema, Port),
- {User, Host, NonDefaultPort, Path}
+ {Host, NonDefaultPort, Path}
end.
@@ -239,82 +229,82 @@ replication_id_convert_test_() ->
http_v4_endpoint_test_() ->
[?_assertMatch({remote, User, Host, Port, Path, HeadersNoAuth, undefined},
- get_v4_endpoint(#{<<"url">> => Url, <<"headers">> => Headers})) ||
- {{User, Host, Port, Path, HeadersNoAuth}, {Url, Headers}} <- [
+ get_v4_endpoint(HttpDb)) ||
+ {{User, Host, Port, Path, HeadersNoAuth}, HttpDb} <- [
{
{undefined, "host", default, "/", []},
- {<<"http://host">>, #{}}
+ httpdb("http://host")
},
{
{undefined, "host", default, "/", []},
- {<<"https://host">>, #{}}
+ httpdb("https://host")
},
{
{undefined, "host", default, "/", []},
- {<<"http://host:5984">>, #{}}
+ httpdb("http://host:5984")
},
{
{undefined, "host", 1, "/", []},
- {<<"http://host:1">>, #{}}
+ httpdb("http://host:1")
},
{
{undefined, "host", 2, "/", []},
- {<<"https://host:2">>, #{}}
+ httpdb("https://host:2")
},
{
{undefined, "host", default, "/", [{"h", "v"}]},
- {<<"http://host">>, #{<<"h">> => <<"v">>}}
+ httpdb("http://host", undefined, undefined, #{"h" => "v"})
},
{
{undefined, "host", default, "/a/b", []},
- {<<"http://host/a/b">>, #{}}
- },
- {
- {"user", "host", default, "/", []},
- {<<"http://user:pass@host">>, #{}}
- },
- {
- {"user", "host", 3, "/", []},
- {<<"http://user:pass@host:3">>, #{}}
+ httpdb("http://host/a/b")
},
{
{"user", "host", default, "/", []},
- {<<"http://user:newpass@host">>, #{}}
+ httpdb("http://host", "user", "pass")
},
{
{"user", "host", default, "/", []},
- {<<"http://host">>, basic_auth(<<"user">>, <<"pass">>)}
- },
- {
- {"user", "host", default, "/", []},
- {<<"http://host">>, basic_auth(<<"user">>, <<"newpass">>)}
- },
- {
- {"user1", "host", default, "/", []},
- {<<"http://user1:pass1@host">>, basic_auth(<<"user2">>,
- <<"pass2">>)}
- },
- {
- {"user", "host", default, "/", [{"h", "v"}]},
- {<<"http://host">>, maps:merge(#{<<"h">> => <<"v">>},
- basic_auth(<<"user">>, <<"pass">>))}
+ httpdb("http://host", "user", "newpass")
},
{
- {undefined, "random_junk", undefined, undefined},
- {<<"random_junk">>, #{}}
- },
- {
- {undefined, "host", default, "/", []},
- {<<"http://host">>, #{<<"Authorization">> =>
- <<"Basic bad">>}}
+ {"user2", "host", default, "/", [{"h", "v"}]},
+ httpdb("http://host", "user2", "pass2", #{"h" => "v"})
}
]
].
-basic_auth(User, Pass) ->
- B64Auth = base64:encode(<<User/binary, ":", Pass/binary>>),
- #{<<"Authorization">> => <<"Basic ", B64Auth/binary>>}.
+httpdb(Url) ->
+ #{
+ <<"url">> => list_to_binary(Url),
+ <<"auth_props">> => #{},
+ <<"headers">> => #{}
+ }.
+
+
+httpdb(Url, User, Pass) ->
+ #{
+ <<"url">> => list_to_binary(Url),
+ <<"auth_props">> => #{
+ <<"basic">> => #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ }
+ },
+ <<"headers">> => #{}
+ }.
+
+
+httpdb(Url, User, Pass, #{} = Headers) ->
+ HttpDb1 = case {User, Pass} of
+ {undefined, undefined} -> httpdb(Url);
+ {User, Pass} -> httpdb(Url, User, Pass)
+ end,
+ Headers1 = maps:fold(fun(K, V, Acc) ->
+ Acc#{list_to_binary(K) => list_to_binary(V)}
+ end, #{}, Headers),
+ HttpDb1#{<<"headers">> => Headers1}.
version4_matches_couchdb3_test_() ->
@@ -352,4 +342,10 @@ id_matches_couchdb3(_) ->
?assertEqual(BaseId3x, BaseId).
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+ [{<<"basic">>, {[
+ {<<"username">>, list_to_binary(User)},
+ {<<"password">>, list_to_binary(Pass)}
+ ]}}].
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_job.erl b/src/couch_replicator/src/couch_replicator_job.erl
index 951471a..381fe57 100644
--- a/src/couch_replicator/src/couch_replicator_job.erl
+++ b/src/couch_replicator/src/couch_replicator_job.erl
@@ -1727,8 +1727,8 @@ t_format_status(_) ->
},
Format = format_status(opts_ignored, [pdict, State]),
FmtOptions = proplists:get_value(options, Format),
- ?assertEqual("http://u:*****@h1/d1/", proplists:get_value(source, Format)),
- ?assertEqual("http://u:*****@h2/d2/", proplists:get_value(target, Format)),
+ ?assertEqual("http://h1/d1/", proplists:get_value(source, Format)),
+ ?assertEqual("http://h2/d2/", proplists:get_value(target, Format)),
?assertEqual(<<"base+ext">>, proplists:get_value(rep_id, Format)),
?assertEqual(true, maps:get(<<"create_target">>, FmtOptions)),
?assertEqual(<<"mydoc">>, proplists:get_value(doc_id, Format)),
diff --git a/src/couch_replicator/src/couch_replicator_parse.erl b/src/couch_replicator/src/couch_replicator_parse.erl
index ac25bee..38d50d4 100644
--- a/src/couch_replicator/src/couch_replicator_parse.erl
+++ b/src/couch_replicator/src/couch_replicator_parse.erl
@@ -205,7 +205,7 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} = Options) ->
}, ProxyParams),
SslParams = ssl_params(Url),
- #{
+ HttpDb = #{
<<"url">> => Url,
<<"auth_props">> => AuthProps,
<<"headers">> => Headers,
@@ -214,7 +214,8 @@ parse_rep_db(#{} = Endpoint, #{} = ProxyParams, #{} = Options) ->
<<"http_connections">> => maps:get(<<"http_connections">>, Options),
<<"retries">> => maps:get(<<"retries_per_request">>, Options),
<<"proxy_url">> => ProxyUrl
- };
+ },
+ normalize_basic_auth(HttpDb);
parse_rep_db(<<"http://", _/binary>> = Url, Proxy, Options) ->
parse_rep_db(#{<<"url">> => Url}, Proxy, Options);
@@ -475,6 +476,117 @@ ssl_verify_options(false) ->
}.
+-spec set_basic_auth_creds(string(), string(), map()) -> map().
+set_basic_auth_creds(undefined, undefined, #{}= HttpDb) ->
+ HttpDb;
+set_basic_auth_creds(User, Pass, #{} = HttpDb)
+ when is_list(User), is_list(Pass) ->
+ #{<<"auth_props">> := AuthProps} = HttpDb,
+ UserPass = #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ },
+ AuthProps1 = AuthProps#{<<"basic">> => UserPass},
+ HttpDb#{<<"auth_props">> := AuthProps1}.
+
+
+-spec extract_creds_from_url(binary()) ->
+ {ok, {string() | undefined, string() | undefined}, string()} |
+ {error, term()}.
+extract_creds_from_url(Url0) ->
+ Url = binary_to_list(Url0),
+ case ibrowse_lib:parse_url(Url) of
+ {error, Error} ->
+ {error, Error};
+ #url{username = undefined, password = undefined} ->
+ {ok, {undefined, undefined}, Url0};
+ #url{protocol = Proto, username = User, password = Pass} ->
+ % Excise user and pass parts from the url. Try to keep the host,
+ % port and path as they were in the original.
+ Prefix = lists:concat([Proto, "://", User, ":", Pass, "@"]),
+ Suffix = lists:sublist(Url, length(Prefix) + 1, length(Url) + 1),
+ NoCreds = lists:concat([Proto, "://", Suffix]),
+ {ok, {User, Pass}, list_to_binary(NoCreds)}
+ end.
+
+
+% Normalize basic auth credentials so they are set only in the auth props
+% object. If multiple basic auth credentials are provided, the resulting
+% credentials are picked in the following order.
+% 1) {"auth": "basic": {"username":.., "password": ...} ...}
+% 2) URL userinfo part
+% 3) "Authentication" : "basic $base64" headers
+%
+-spec normalize_basic_auth(map()) -> map().
+normalize_basic_auth(#{} = HttpDb) ->
+ #{
+ <<"url">> := Url,
+ <<"headers">> := Headers
+ } = HttpDb,
+ {HeaderCreds, HeadersNoCreds} = remove_basic_auth_from_headers(Headers),
+ {UrlCreds, UrlWithoutCreds} = case extract_creds_from_url(Url) of
+ {ok, Creds = {_, _}, UrlNoCreds} ->
+ {Creds, UrlNoCreds};
+ {error, _Error} ->
+ % Don't crash replicator if user provided an invalid
+ % userinfo part
+ {undefined, undefined}
+ end,
+ AuthCreds = {_, _} = couch_replicator_utils:get_basic_auth_creds(HttpDb),
+ HttpDb1 = HttpDb#{
+ <<"url">> := UrlWithoutCreds,
+ <<"headers">> := HeadersNoCreds
+ },
+ {User, Pass} = case {AuthCreds, UrlCreds, HeaderCreds} of
+ {{U, P}, {_, _}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+ {{_, _}, {U, P}, {_, _}} when is_list(U), is_list(P) -> {U, P};
+ {{_, _}, {_, _}, {U, P}} -> {U, P}
+ end,
+ set_basic_auth_creds(User, Pass, HttpDb1).
+
+
+remove_basic_auth_from_headers(#{} = HeadersMap) ->
+ % Headers are passed in a map however mochiweb_headers expects them to be
+ % lists so we transform them to lists first, then back to maps
+ Headers = maps:fold(fun(K, V, Acc) ->
+ [{binary_to_list(K), binary_to_list(V)} | Acc]
+ end, [], HeadersMap),
+ Headers1 = mochiweb_headers:make(Headers),
+ case mochiweb_headers:get_value("Authorization", Headers1) of
+ undefined ->
+ {{undefined, undefined}, HeadersMap};
+ Auth ->
+ {Basic, B64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
+ BasicLower = string:to_lower(Basic),
+ Result = maybe_remove_basic_auth(BasicLower, B64, Headers1),
+ {{User, Pass}, Headers2} = Result,
+ HeadersMapResult = lists:foldl(fun({K, V}, Acc) ->
+ Acc#{list_to_binary(K) => list_to_binary(V)}
+ end, #{}, Headers2),
+ {{User, Pass}, HeadersMapResult}
+ end.
+
+
+maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
+ Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
+ {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
+maybe_remove_basic_auth(_, _, Headers) ->
+ {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
+
+
+decode_basic_creds(Base64) ->
+ try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
+ [User, Pass] ->
+ {User, Pass};
+ _ ->
+ {undefined, undefined}
+ catch
+ % Tolerate invalid B64 values here to avoid crashing replicator
+ error:function_clause ->
+ {undefined, undefined}
+ end.
+
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
@@ -565,4 +677,243 @@ t_error_on_local_endpoint(_) ->
?assertThrow({bad_rep_doc, Expect}, parse_rep_doc(RepDoc)).
+remove_basic_auth_from_headers_test_() ->
+ B64 = list_to_binary(b64creds("user", "pass")),
+ [?_assertEqual({{User, Pass}, NoAuthHeaders},
+ remove_basic_auth_from_headers(Headers)) ||
+ {{User, Pass, NoAuthHeaders}, Headers} <- [
+ {
+ {undefined, undefined, #{}},
+ #{}
+ },
+ {
+ {undefined, undefined, #{<<"h">> => <<"v">>}},
+ #{<<"h">> => <<"v">>}
+ },
+ {
+ {undefined, undefined, #{<<"Authorization">> => <<"junk">>}},
+ #{<<"Authorization">> => <<"junk">>}
+ },
+ {
+ {undefined, undefined, #{}},
+ #{<<"Authorization">> => <<"basic X">>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"Authorization">> => <<"Basic ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"AuThorization">> => <<"Basic ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{}},
+ #{<<"Authorization">> => <<"bAsIc ", B64/binary>>}
+ },
+ {
+ {"user", "pass", #{<<"h">> => <<"v">>}},
+ #{
+ <<"Authorization">> => <<"Basic ", B64/binary>>,
+ <<"h">> => <<"v">>
+ }
+ }
+ ]
+ ].
+
+
+b64creds(User, Pass) ->
+ base64:encode_to_string(User ++ ":" ++ Pass).
+
+
+set_basic_auth_creds_test() ->
+ Check = fun(User, Pass, Props) ->
+ HttpDb = #{<<"auth_props">> => Props},
+ HttpDb1 = set_basic_auth_creds(User, Pass, HttpDb),
+ maps:get(<<"auth_props">>, HttpDb1)
+ end,
+
+ ?assertEqual(#{}, Check(undefined, undefined, #{})),
+
+ ?assertEqual(#{<<"other">> => #{}}, Check(undefined, undefined,
+ #{<<"other">> => #{}})),
+
+ ?assertEqual(#{
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ }, Check("u", "p", #{})),
+
+ ?assertEqual(#{
+ <<"other">> => #{},
+ <<"basic">> => #{
+ <<"username">> => <<"u">>,
+ <<"password">> => <<"p">>
+ }
+ }, Check("u", "p", #{<<"other">> => #{}})).
+
+
+normalize_basic_creds_test_() ->
+ DefaultHeaders = couch_replicator_utils:default_headers_map(),
+ [?_assertEqual(Expect, normalize_basic_auth(Input)) || {Input, Expect} <- [
+ {
+ #{
+ <<"url">> => <<"http://u:p@x.y/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://x.y/db">>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u:p@h:80/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://h:80/db">>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"https://u:p@h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"https://h/db">>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u:p@[2001:db8:a1b:12f9::1]/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://[2001:db8:a1b:12f9::1]/db">>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p@")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u", "p@"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"authorization">> => basic_b64("u", "p@%40")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u", "p@%40"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"aUthoriZation">> => basic_b64("U", "p")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("U", "p"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db">>,
+ <<"auth_props">> => #{},
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"Authorization">> => basic_b64("u2", "p2")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u1", "p1"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db">>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ }
+ },
+ {
+ #{
+ <<"url">> => <<"http://u1:p1@h/db">>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => maps:merge(DefaultHeaders, #{
+ <<"Authorization">> => basic_b64("u3", "p3")
+ })
+ },
+ #{
+ <<"url">> => <<"http://h/db">>,
+ <<"auth_props">> => auth_props("u2", "p2"),
+ <<"headers">> => DefaultHeaders
+ }
+ }
+ ]].
+
+
+basic_b64(User, Pass) when is_list(User), is_list(Pass) ->
+ B64Creds = list_to_binary(b64creds(User, Pass)),
+ <<"basic ", B64Creds/binary>>.
+
+
+auth_props(User, Pass) when is_list(User), is_list(Pass) ->
+ #{
+ <<"basic">> => #{
+ <<"username">> => list_to_binary(User),
+ <<"password">> => list_to_binary(Pass)
+ }
+ }.
+
-endif.
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl
index 523de5f..c60cf56 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -18,12 +18,12 @@
iso8601/1,
rfc1123_local/0,
rfc1123_local/1,
- remove_basic_auth_from_headers/1,
normalize_rep/1,
compare_reps/2,
default_headers_map/0,
parse_replication_states/1,
parse_int_param/5,
+ get_basic_auth_creds/1,
proplist_options/1
]).
@@ -71,37 +71,6 @@ rfc1123_local(Sec) ->
list_to_binary(httpd_util:rfc1123_date(Local)).
-remove_basic_auth_from_headers(Headers) ->
- Headers1 = mochiweb_headers:make(Headers),
- case mochiweb_headers:get_value("Authorization", Headers1) of
- undefined ->
- {{undefined, undefined}, Headers};
- Auth ->
- {Basic, Base64} = lists:splitwith(fun(X) -> X =/= $\s end, Auth),
- maybe_remove_basic_auth(string:to_lower(Basic), Base64, Headers1)
- end.
-
-
-maybe_remove_basic_auth("basic", " " ++ Base64, Headers) ->
- Headers1 = mochiweb_headers:delete_any("Authorization", Headers),
- {decode_basic_creds(Base64), mochiweb_headers:to_list(Headers1)};
-maybe_remove_basic_auth(_, _, Headers) ->
- {{undefined, undefined}, mochiweb_headers:to_list(Headers)}.
-
-
-decode_basic_creds(Base64) ->
- try re:split(base64:decode(Base64), ":", [{return, list}, {parts, 2}]) of
- [User, Pass] ->
- {User, Pass};
- _ ->
- {undefined, undefined}
- catch
- % Tolerate invalid B64 values here to avoid crashing replicator
- error:function_clause ->
- {undefined, undefined}
- end.
-
-
-spec compare_reps(#{} | null, #{} | null) -> boolean().
compare_reps(Rep1, Rep2) ->
NormRep1 = normalize_rep(Rep1),
@@ -199,56 +168,30 @@ unix_sec_to_timestamp(Sec) when is_integer(Sec) ->
{MegaSecPart, SecPart, 0}.
--ifdef(TEST).
+-spec get_basic_auth_creds(#httpdb{} | map()) ->
+ {string(), string()} | {undefined, undefined}.
+get_basic_auth_creds(#httpdb{auth_props = AuthProps}) ->
+ get_basic_auth_creds(#{<<"auth_props">> => AuthProps});
+
+get_basic_auth_creds(#{<<"auth_props">> := Props}) ->
+ case Props of
+ #{<<"basic">> := Basic} ->
+ User = maps:get(<<"username">>, Basic, undefined),
+ Pass = maps:get(<<"password">>, Basic, undefined),
+ case {User, Pass} of
+ _ when is_binary(User), is_binary(Pass) ->
+ {binary_to_list(User), binary_to_list(Pass)};
+ _Other ->
+ {undefined, undefined}
+ end;
+ _Other ->
+ {undefined, undefined}
+ end.
--include_lib("eunit/include/eunit.hrl").
-remove_basic_auth_from_headers_test_() ->
- [?_assertMatch({{User, Pass}, NoAuthHeaders},
- remove_basic_auth_from_headers(Headers)) ||
- {{User, Pass, NoAuthHeaders}, Headers} <- [
- {
- {undefined, undefined, []},
- []
- },
- {
- {undefined, undefined, [{"h", "v"}]},
- [{"h", "v"}]
- },
- {
- {undefined, undefined, [{"Authorization", "junk"}]},
- [{"Authorization", "junk"}]
- },
- {
- {undefined, undefined, []},
- [{"Authorization", "basic X"}]
- },
- {
- {"user", "pass", []},
- [{"Authorization", "Basic " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", []},
- [{"AuThorization", "Basic " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", []},
- [{"Authorization", "bAsIc " ++ b64creds("user", "pass")}]
- },
- {
- {"user", "pass", [{"h", "v"}]},
- [
- {"Authorization", "Basic " ++ b64creds("user", "pass")},
- {"h", "v"}
- ]
- }
- ]
- ].
-
-
-b64creds(User, Pass) ->
- base64:encode_to_string(User ++ ":" ++ Pass).
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
normalize_rep_test_() ->
{
@@ -299,4 +242,24 @@ normalize_endpoint() ->
?assertEqual(<<"local">>, normalize_endpoint(<<"local">>)).
+get_basic_auth_creds_from_httpdb_test() ->
+ Check = fun(Props) ->
+ get_basic_auth_creds(#{<<"auth_props">> => Props})
+ end,
+
+ ?assertEqual({undefined, undefined}, Check(#{})),
+
+ ?assertEqual({undefined, undefined}, Check(#{a => b})),
+
+ ?assertEqual({undefined, undefined}, Check(#{<<"other">> => <<"x">>})),
+
+ ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => #{}})),
+
+ UserPass1 = #{<<"username">> => <<"u">>, <<"password">> => <<"p">>},
+ ?assertEqual({"u", "p"}, Check(#{<<"basic">> => UserPass1})),
+
+ UserPass2 = #{<<"username">> => <<"u">>, <<"password">> => null},
+ ?assertEqual({undefined, undefined}, Check(#{<<"basic">> => UserPass2})).
+
+
-endif.