You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2020/02/06 17:08:02 UTC
[couchdb] 01/01: Simplify couch_auth_cache
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch simplify-couch-auth-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2df0c327e3792774fd2771551853a50ec043b36f
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Thu Feb 6 12:03:28 2020 -0500
Simplify couch_auth_cache
Since the backend port is closed by default, remove the cache service
since it was shown to cause monitor leaks, and it just consumes resources
running in the background when almost nobody uses it. The few utility scripts
that might use the backend port can just get the auth docs directly from the
db.
Issue: https://github.com/apache/couchdb/issues/2493
---
src/couch/src/couch_auth_cache.erl | 383 +++---------------------
src/couch/src/couch_secondary_sup.erl | 3 +-
src/couch/test/eunit/couch_auth_cache_tests.erl | 28 +-
3 files changed, 41 insertions(+), 373 deletions(-)
diff --git a/src/couch/src/couch_auth_cache.erl b/src/couch/src/couch_auth_cache.erl
index 157b090..c564cee 100644
--- a/src/couch/src/couch_auth_cache.erl
+++ b/src/couch/src/couch_auth_cache.erl
@@ -11,37 +11,22 @@
% the License.
-module(couch_auth_cache).
--behaviour(gen_server).
--vsn(3).
--behaviour(config_listener).
-% public API
--export([get_user_creds/1, get_user_creds/2, update_user_creds/3]).
--export([get_admin/1, add_roles/2, auth_design_doc/1]).
-% gen_server API
--export([start_link/0, init/1, handle_call/3, handle_info/2, handle_cast/2]).
--export([code_change/3, terminate/2]).
+-export([
+ get_user_creds/1,
+ get_user_creds/2,
+ update_user_creds/3,
+ get_admin/1,
+ add_roles/2,
+ auth_design_doc/1,
+ ensure_users_db_exists/0
+]).
--export([handle_config_change/5, handle_config_terminate/3]).
--export([handle_db_event/3]).
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch/include/couch_js_functions.hrl").
--define(STATE, auth_state_ets).
--define(BY_USER, auth_by_user_ets).
--define(BY_ATIME, auth_by_atime_ets).
-
--define(RELISTEN_DELAY, 5000).
-
--record(state, {
- max_cache_size = 0,
- cache_size = 0,
- db_notifier = nil,
- event_listener = nil
-}).
-
-spec get_user_creds(UserName::string() | binary()) ->
{ok, Credentials::list(), term()} | nil.
@@ -58,9 +43,9 @@ get_user_creds(Req, UserName) when is_list(UserName) ->
get_user_creds(_Req, UserName) ->
UserCreds = case get_admin(UserName) of
nil ->
- get_from_cache(UserName);
+ get_from_db(UserName);
Props ->
- case get_from_cache(UserName) of
+ case get_from_db(UserName) of
nil ->
Props;
UserProps when is_list(UserProps) ->
@@ -70,8 +55,8 @@ get_user_creds(_Req, UserName) ->
validate_user_creds(UserCreds).
update_user_creds(_Req, UserDoc, _AuthCtx) ->
- DbNameList = config:get("couch_httpd_auth", "authentication_db", "_users"),
- couch_util:with_db(?l2b(DbNameList), fun(UserDb) ->
+ ok = ensure_users_db_exists(),
+ couch_util:with_db(users_db(), fun(UserDb) ->
{ok, _NewRev} = couch_db:update_doc(UserDb, UserDoc, []),
ok
end).
@@ -109,21 +94,20 @@ make_admin_doc(DerivedKey, Salt, Iterations) ->
{<<"password_scheme">>, <<"pbkdf2">>},
{<<"derived_key">>, ?l2b(DerivedKey)}].
-get_from_cache(UserName) ->
- exec_if_auth_db(
- fun(_AuthDb) ->
- maybe_refresh_cache(),
- case ets:lookup(?BY_USER, UserName) of
- [] ->
- gen_server:call(?MODULE, {fetch, UserName}, infinity);
- [{UserName, {Credentials, _ATime}}] ->
- couch_stats:increment_counter([couchdb, auth_cache_hits]),
- gen_server:cast(?MODULE, {cache_hit, UserName}),
- Credentials
- end
- end,
- nil
- ).
+
+get_from_db(UserName) ->
+ ok = ensure_users_db_exists(),
+ couch_util:with_db(users_db(), fun(Db) ->
+ DocId = <<"org.couchdb.user:", UserName/binary>>,
+ try
+ {ok, Doc} = couch_db:open_doc(Db, DocId, [conflicts]),
+ {DocProps} = couch_doc:to_json_obj(Doc, []),
+ DocProps
+ catch
+ _:_Error ->
+ nil
+ end
+ end).
validate_user_creds(nil) ->
@@ -141,313 +125,24 @@ validate_user_creds(UserCreds) ->
{ok, UserCreds, nil}.
-start_link() ->
- gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-
-init(_) ->
- ?STATE = ets:new(?STATE, [set, protected, named_table]),
- ?BY_USER = ets:new(?BY_USER, [set, protected, named_table]),
- ?BY_ATIME = ets:new(?BY_ATIME, [ordered_set, private, named_table]),
- AuthDbName = config:get("couch_httpd_auth", "authentication_db"),
- process_flag(trap_exit, true),
- ok = config:listen_for_changes(?MODULE, nil),
- {ok, Listener} = couch_event:link_listener(
- ?MODULE, handle_db_event, nil, [{dbname, AuthDbName}]
- ),
- State = #state{
- event_listener = Listener,
- max_cache_size = list_to_integer(
- config:get("couch_httpd_auth", "auth_cache_size", "50")
- )
- },
- {ok, reinit_cache(State)}.
-
-
-handle_db_event(_DbName, created, St) ->
- gen_server:call(?MODULE, reinit_cache, infinity),
- {ok, St};
-handle_db_event(_DbName, compacted, St) ->
- gen_server:call(?MODULE, auth_db_compacted, infinity),
- {ok, St};
-handle_db_event(_, _, St) ->
- {ok, St}.
-
-
-handle_call(reinit_cache, _From, State) ->
- exec_if_auth_db(fun(AuthDb) -> catch couch_db:close(AuthDb) end),
- {reply, ok, reinit_cache(State)};
-
-handle_call(auth_db_compacted, _From, State) ->
- exec_if_auth_db(
- fun(AuthDb) ->
- true = ets:insert(?STATE, {auth_db, reopen_auth_db(AuthDb)})
- end
- ),
- {reply, ok, State};
-
-handle_call({new_max_cache_size, NewSize},
- _From, #state{cache_size = Size} = State) when NewSize >= Size ->
- {reply, ok, State#state{max_cache_size = NewSize}};
-
-handle_call({new_max_cache_size, NewSize}, _From, State) ->
- free_mru_cache_entries(State#state.cache_size - NewSize),
- {reply, ok, State#state{max_cache_size = NewSize, cache_size = NewSize}};
-
-handle_call({fetch, UserName}, _From, State) ->
- {Credentials, NewState} = case ets:lookup(?BY_USER, UserName) of
- [{UserName, {Creds, ATime}}] ->
- couch_stats:increment_counter([couchdb, auth_cache_hits]),
- cache_hit(UserName, Creds, ATime),
- {Creds, State};
- [] ->
- couch_stats:increment_counter([couchdb, auth_cache_misses]),
- Creds = get_user_props_from_db(UserName),
- ATime = couch_util:unique_monotonic_integer(),
- State1 = add_cache_entry(UserName, Creds, ATime, State),
- {Creds, State1}
- end,
- {reply, Credentials, NewState};
-
-handle_call(refresh, _From, State) ->
- exec_if_auth_db(fun refresh_entries/1),
- {reply, ok, State}.
-
-
-handle_cast({cache_hit, UserName}, State) ->
- case ets:lookup(?BY_USER, UserName) of
- [{UserName, {Credentials, ATime}}] ->
- cache_hit(UserName, Credentials, ATime);
- _ ->
- ok
- end,
- {noreply, State}.
-
-
-handle_info({'EXIT', LPid, _Reason}, #state{event_listener=LPid}=State) ->
- erlang:send_after(5000, self(), restart_event_listener),
- {noreply, State#state{event_listener=undefined}};
-handle_info(restart_event_listener, State) ->
- [{auth_db_name, AuthDbName}] = ets:lookup(?STATE, auth_db_name),
- {ok, NewListener} = couch_event:link_listener(
- ?MODULE, handle_db_event, nil, [{dbname, AuthDbName}]
- ),
- {noreply, State#state{event_listener=NewListener}};
-handle_info({'DOWN', _Ref, _, _, shutdown}, State) ->
- {stop, shutdown, State};
-handle_info({'DOWN', _Ref, _, _, _}, State) ->
- {noreply, reinit_cache(State)};
-handle_info(restart_config_listener, State) ->
- ok = config:listen_for_changes(?MODULE, nil),
- {noreply, State}.
-
-
-
-terminate(_Reason, #state{event_listener = Listener}) ->
- couch_event:stop_listener(Listener),
- exec_if_auth_db(fun(AuthDb) -> catch couch_db:close(AuthDb) end),
- true = ets:delete(?BY_USER),
- true = ets:delete(?BY_ATIME),
- true = ets:delete(?STATE).
-
-
-code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
-
-
-handle_config_change("couch_httpd_auth", "auth_cache_size", SizeList, _, _) ->
- Size = list_to_integer(SizeList),
- {ok, gen_server:call(?MODULE, {new_max_cache_size, Size}, infinity)};
-handle_config_change("couch_httpd_auth", "authentication_db", _DbName, _, _) ->
- {ok, gen_server:call(?MODULE, reinit_cache, infinity)};
-handle_config_change(_, _, _, _, _) ->
- {ok, nil}.
-
-handle_config_terminate(_, stop, _) ->
- ok;
-handle_config_terminate(_Server, _Reason, _State) ->
- erlang:send_after(?RELISTEN_DELAY, whereis(?MODULE), restart_config_listener).
-
-clear_cache(State) ->
- exec_if_auth_db(fun(AuthDb) -> catch couch_db:close(AuthDb) end),
- true = ets:delete_all_objects(?BY_USER),
- true = ets:delete_all_objects(?BY_ATIME),
- State#state{cache_size = 0}.
-
-
-reinit_cache(#state{} = State) ->
- NewState = clear_cache(State),
- AuthDbName = ?l2b(config:get("couch_httpd_auth", "authentication_db")),
- true = ets:insert(?STATE, {auth_db_name, AuthDbName}),
- AuthDb = open_auth_db(),
- true = ets:insert(?STATE, {auth_db, AuthDb}),
- couch_db:monitor(AuthDb),
- NewState.
-
-
-add_cache_entry(_, _, _, #state{max_cache_size = 0} = State) ->
- State;
-add_cache_entry(UserName, Credentials, ATime, State) ->
- case State#state.cache_size >= State#state.max_cache_size of
- true ->
- free_mru_cache_entry();
- false ->
- ok
- end,
- true = ets:insert(?BY_ATIME, {ATime, UserName}),
- true = ets:insert(?BY_USER, {UserName, {Credentials, ATime}}),
- State#state{cache_size = couch_util:get_value(size, ets:info(?BY_USER))}.
-
-free_mru_cache_entries(0) ->
- ok;
-free_mru_cache_entries(N) when N > 0 ->
- free_mru_cache_entry(),
- free_mru_cache_entries(N - 1).
-
-free_mru_cache_entry() ->
- MruTime = ets:last(?BY_ATIME),
- [{MruTime, UserName}] = ets:lookup(?BY_ATIME, MruTime),
- true = ets:delete(?BY_ATIME, MruTime),
- true = ets:delete(?BY_USER, UserName).
-
-
-cache_hit(UserName, Credentials, ATime) ->
- NewATime = couch_util:unique_monotonic_integer(),
- true = ets:delete(?BY_ATIME, ATime),
- true = ets:insert(?BY_ATIME, {NewATime, UserName}),
- true = ets:insert(?BY_USER, {UserName, {Credentials, NewATime}}).
-
-
-refresh_entries(AuthDb) ->
- case reopen_auth_db(AuthDb) of
- nil ->
- ok;
- AuthDb2 ->
- AuthDbSeq = couch_db:get_update_seq(AuthDb),
- AuthDb2Seq = couch_db:get_update_seq(AuthDb2),
- case AuthDb2Seq > AuthDbSeq of
- true ->
- Fun = fun(DocInfo, _) -> refresh_entry(AuthDb2, DocInfo) end,
- {ok, _} = couch_db:fold_changes(AuthDb2, AuthDbSeq, Fun, nil),
- true = ets:insert(?STATE, {auth_db, AuthDb2});
- false ->
- ok
- end
- end.
-
-
-refresh_entry(Db, #full_doc_info{} = FDI) ->
- refresh_entry(Db, couch_doc:to_doc_info(FDI));
-refresh_entry(Db, #doc_info{high_seq = DocSeq} = DocInfo) ->
- case is_user_doc(DocInfo) of
- {true, UserName} ->
- case ets:lookup(?BY_USER, UserName) of
- [] ->
- ok;
- [{UserName, {_OldCreds, ATime}}] ->
- {ok, Doc} = couch_db:open_doc(Db, DocInfo, [conflicts, deleted]),
- NewCreds = user_creds(Doc),
- true = ets:insert(?BY_USER, {UserName, {NewCreds, ATime}})
- end;
- false ->
- ok
- end,
- {ok, DocSeq}.
-
-
-user_creds(#doc{deleted = true}) ->
- nil;
-user_creds(#doc{} = Doc) ->
- {Creds} = couch_doc:to_json_obj(Doc, []),
- Creds.
-
-
-is_user_doc(#doc_info{id = <<"org.couchdb.user:", UserName/binary>>}) ->
- {true, UserName};
-is_user_doc(_) ->
- false.
-
-
-maybe_refresh_cache() ->
- case cache_needs_refresh() of
- true ->
- ok = gen_server:call(?MODULE, refresh, infinity);
- false ->
- ok
- end.
-
-
-cache_needs_refresh() ->
- exec_if_auth_db(
- fun(AuthDb) ->
- case reopen_auth_db(AuthDb) of
- nil ->
- false;
- AuthDb2 ->
- AuthDbSeq = couch_db:get_update_seq(AuthDb),
- AuthDb2Seq = couch_db:get_update_seq(AuthDb2),
- AuthDb2Seq > AuthDbSeq
- end
- end,
- false
- ).
-
-
-reopen_auth_db(AuthDb) ->
- case (catch couch_db:reopen(AuthDb)) of
- {ok, AuthDb2} ->
- AuthDb2;
- _ ->
- nil
- end.
-
-
-exec_if_auth_db(Fun) ->
- exec_if_auth_db(Fun, ok).
-
-exec_if_auth_db(Fun, DefRes) ->
- case ets:lookup(?STATE, auth_db) of
- [{auth_db, AuthDb}] ->
- Fun(AuthDb);
- _ ->
- DefRes
- end.
-
-
-open_auth_db() ->
- [{auth_db_name, DbName}] = ets:lookup(?STATE, auth_db_name),
- {ok, AuthDb} = ensure_users_db_exists(DbName, [sys_db]),
- AuthDb.
-
+users_db() ->
+ DbNameList = config:get("couch_httpd_auth", "authentication_db", "_users"),
+ ?l2b(DbNameList).
-get_user_props_from_db(UserName) ->
- exec_if_auth_db(
- fun(AuthDb) ->
- Db = reopen_auth_db(AuthDb),
- DocId = <<"org.couchdb.user:", UserName/binary>>,
- try
- {ok, Doc} = couch_db:open_doc(Db, DocId, [conflicts]),
- {DocProps} = couch_doc:to_json_obj(Doc, []),
- DocProps
- catch
- _:_Error ->
- nil
- end
- end,
- nil
- ).
-ensure_users_db_exists(DbName, Options) ->
- Options1 = [?ADMIN_CTX, nologifmissing | Options],
- case couch_db:open(DbName, Options1) of
+ensure_users_db_exists() ->
+ Options = [?ADMIN_CTX, nologifmissing],
+ case couch_db:open(users_db(), Options) of
{ok, Db} ->
ensure_auth_ddoc_exists(Db, <<"_design/_auth">>),
- {ok, Db};
+ couch_db:close(Db);
_Error ->
- {ok, Db} = couch_db:create(DbName, Options1),
+ {ok, Db} = couch_db:create(users_db(), Options),
ok = ensure_auth_ddoc_exists(Db, <<"_design/_auth">>),
- {ok, Db}
- end.
+ couch_db:close(Db)
+ end,
+ ok.
+
ensure_auth_ddoc_exists(Db, DDocId) ->
case couch_db:open_doc(Db, DDocId) of
diff --git a/src/couch/src/couch_secondary_sup.erl b/src/couch/src/couch_secondary_sup.erl
index 9c7d414..bb78215 100644
--- a/src/couch/src/couch_secondary_sup.erl
+++ b/src/couch/src/couch_secondary_sup.erl
@@ -30,8 +30,7 @@ init([]) ->
{index_server, {couch_index_server, start_link, []}},
{query_servers, {couch_proc_manager, start_link, []}},
{vhosts, {couch_httpd_vhost, start_link, []}},
- {uuids, {couch_uuids, start, []}},
- {auth_cache, {couch_auth_cache, start_link, []}}
+ {uuids, {couch_uuids, start, []}}
],
MaybeHttp = case http_enabled() of
diff --git a/src/couch/test/eunit/couch_auth_cache_tests.erl b/src/couch/test/eunit/couch_auth_cache_tests.erl
index 5439dd7..71faf77 100644
--- a/src/couch/test/eunit/couch_auth_cache_tests.erl
+++ b/src/couch/test/eunit/couch_auth_cache_tests.erl
@@ -52,7 +52,6 @@ couch_auth_cache_test_() ->
fun should_drop_cache_on_auth_db_change/1,
fun should_restore_cache_on_auth_db_change/1,
fun should_recover_cache_after_shutdown/1,
- fun should_close_old_db_on_auth_db_change/1,
fun should_get_admin_from_config/1
]
}
@@ -226,13 +225,6 @@ should_recover_cache_after_shutdown(DbName) ->
?assertEqual(PasswordHash, get_user_doc_password_sha(DbName, "joe"))
end).
-should_close_old_db_on_auth_db_change(DbName) ->
- {timeout, ?DB_TIMEOUT, ?_test(begin
- ?assertEqual(ok, wait_db(DbName, fun is_opened/1)),
- config:set("couch_httpd_auth", "authentication_db",
- ?b2l(?tempdb()), false),
- ?assertEqual(ok, wait_db(DbName, fun is_closed/1))
- end)}.
should_get_admin_from_config(_DbName) ->
?_test(begin
@@ -251,6 +243,7 @@ update_user_doc(DbName, UserName, Password) ->
update_user_doc(DbName, UserName, Password, nil).
update_user_doc(DbName, UserName, Password, Rev) ->
+ ok = couch_auth_cache:ensure_users_db_exists(),
User = iolist_to_binary(UserName),
Doc = couch_doc:from_json_obj({[
{<<"_id">>, <<"org.couchdb.user:", User/binary>>},
@@ -269,17 +262,6 @@ update_user_doc(DbName, UserName, Password, Rev) ->
ok = couch_db:close(AuthDb),
{ok, couch_doc:rev_to_str(NewRev)}.
-wait_db(Db, DbFun) ->
- test_util:wait(fun() ->
- case DbFun(Db) of
- true ->
- ok;
- false ->
- wait
- end
- end, ?DB_TIMEOUT, 500).
-
-
hash_password(Password) ->
?l2b(couch_util:to_hex(crypto:hash(sha, iolist_to_binary([Password, ?SALT])))).
@@ -324,14 +306,6 @@ delete_user_doc(DbName, UserName) ->
{ok, _} = couch_db:update_doc(AuthDb, DeletedDoc, []),
ok = couch_db:close(AuthDb).
-is_opened(DbName) ->
- {ok, AuthDb} = couch_db:open_int(DbName, [?ADMIN_CTX]),
- Monitors = couch_db:monitored_by(AuthDb) -- [self()],
- ok = couch_db:close(AuthDb),
- Monitors /= [].
-
-is_closed(DbName) ->
- not is_opened(DbName).
make_validate_test({Old, New, "ok"} = Case) ->
{test_id(Case), ?_assertEqual(ok, validate(doc(Old), doc(New)))};