You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by bb...@apache.org on 2017/03/08 20:39:27 UTC
[3/6] couch-index commit: updated refs/heads/8409-view-cache to
5a9f05c
Make index cache only have a soft limit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/commit/78a65a3f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/tree/78a65a3f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/diff/78a65a3f
Branch: refs/heads/8409-view-cache
Commit: 78a65a3f5eefd5669b096fcbc4806cd9029299fb
Parents: 19a3a49
Author: Benjamin Bastian <be...@gmail.com>
Authored: Thu Mar 2 14:23:24 2017 -0800
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Mar 2 14:23:24 2017 -0800
----------------------------------------------------------------------
src/couch_index_monitor.erl | 20 ++---
src/couch_index_server.erl | 133 +++++++++++++-------------------
test/couch_index_monitor_tests.erl | 5 +-
3 files changed, 61 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/src/couch_index_monitor.erl
----------------------------------------------------------------------
diff --git a/src/couch_index_monitor.erl b/src/couch_index_monitor.erl
index 4cc4bc7..fc2fef1 100644
--- a/src/couch_index_monitor.erl
+++ b/src/couch_index_monitor.erl
@@ -14,7 +14,7 @@
-export([
- spawn_link/2,
+ spawn_link/1,
close/1,
set_pid/2,
@@ -24,7 +24,7 @@
]).
-export([
- init/2
+ init/1
]).
@@ -34,15 +34,14 @@
-record(st, {
name,
type,
- is_sys,
ref,
client_refs,
closing
}).
-spawn_link(Name, IsSys) ->
- erlang:spawn_link(?MODULE, init, [Name, IsSys]).
+spawn_link(Name) ->
+ erlang:spawn_link(?MODULE, init, [Name]).
close(Monitor) ->
@@ -67,11 +66,11 @@ notify(Monitor, {Client, _}) when is_pid(Client) ->
notify(Monitor, Client).
-cancel(Name, {Client, Monitor, IsSys})
+cancel(Name, {Client, Monitor})
when Client == self(), is_pid(Monitor) ->
Monitor ! {cancel, self()},
case (catch ets:update_counter(?BY_COUNTERS, Name, -1)) of
- 0 when not IsSys ->
+ 0 ->
true = ets:insert(?BY_IDLE, {Name}),
ok;
_ ->
@@ -79,11 +78,10 @@ cancel(Name, {Client, Monitor, IsSys})
end.
-init(Name, IsSys) ->
+init(Name) ->
{ok, CRefs} = khash:new(),
loop(#st{
name = Name,
- is_sys = IsSys,
ref = undefined,
client_refs = CRefs,
closing = false
@@ -153,10 +151,6 @@ handle_info(Msg, St) ->
maybe_set_idle(St) ->
case khash:size(St#st.client_refs) of
- 0 when St#st.is_sys ->
- % System dbs don't go idle so they're
- % never a candidate to get closed
- ok;
0 ->
% We're now idle
ets:insert(?BY_IDLE, {St#st.name});
http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/src/couch_index_server.erl
----------------------------------------------------------------------
diff --git a/src/couch_index_server.erl b/src/couch_index_server.erl
index 0350492..21028a1 100644
--- a/src/couch_index_server.erl
+++ b/src/couch_index_server.erl
@@ -16,7 +16,7 @@
-vsn(2).
--export([start_link/0, validate/2, get_index/4, get_index/3, get_index_from_state/3]).
+-export([start_link/0, validate/2, get_index/4, get_index/3, get_index/2]).
-export([init/1, terminate/2, code_change/3]).
-export([handle_call/3, handle_cast/2, handle_info/2]).
@@ -37,9 +37,8 @@
-record(st, {
root_dir,
- max_indexes,
- open,
- sys_open
+ soft_max_indexes,
+ open
}).
start_link() ->
@@ -99,24 +98,20 @@ get_index(Module, Db, DDoc, Fun) when is_binary(DDoc) ->
get_index(Module, Db, DDoc, Fun) when is_function(Fun, 1) ->
{ok, InitState} = Module:init(Db, DDoc),
{ok, FunResp} = Fun(InitState),
- case get_index_from_state(Module, InitState, couch_db:is_system_db(Db)) of
- {ok, Pid} ->
- {ok, Pid, FunResp};
- {error, all_active} ->
- {error, all_active}
- end;
+ {ok, Pid} = get_index(Module, InitState),
+ {ok, Pid, FunResp};
get_index(Module, Db, DDoc, _Fun) ->
{ok, InitState} = Module:init(Db, DDoc),
- get_index_from_state(Module, InitState, couch_db:is_system_db(Db)).
+ get_index(Module, InitState).
-get_index_from_state(Module, IdxState, SysOwned) ->
+get_index(Module, IdxState) ->
DbName = Module:get(db_name, IdxState),
Sig = Module:get(signature, IdxState),
- Args = {Module, IdxState, DbName, Sig, SysOwned},
+ Args = {Module, IdxState, DbName, Sig},
case incref({DbName, Sig}) of
ok ->
- [{_, {Pid, Monitor, _SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+ [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
ok = couch_index_monitor:notify(Monitor),
{ok, Pid};
_ ->
@@ -136,8 +131,8 @@ init([]) ->
RootDir = couch_index_util:root_dir(),
couch_file:init_delete_dir(RootDir),
MaxIndexes = list_to_integer(
- config:get("couchdb", "max_indexes_open", integer_to_list(?MAX_INDEXES_OPEN))),
- {ok, #st{root_dir=RootDir, max_indexes=MaxIndexes, open=0, sys_open=0}}.
+ config:get("couchdb", "soft_max_indexes_open", integer_to_list(?MAX_INDEXES_OPEN))),
+ {ok, #st{root_dir=RootDir, soft_max_indexes=MaxIndexes, open=0}}.
terminate(_Reason, _State) ->
@@ -145,77 +140,60 @@ terminate(_Reason, _State) ->
lists:map(fun couch_util:shutdown_sync/1, Pids),
ok.
-make_room(State, false) ->
- case maybe_close_idle(State) of
- {ok, NewState} ->
- {ok, NewState};
- Other ->
- Other
- end;
-make_room(State, true) ->
- {ok, State}.
--spec maybe_close_idle(#st{}) -> {ok, #st{}} | {error, all_active}.
-maybe_close_idle(#st{open=Open, max_indexes=Max}=State) when Open < Max ->
+-spec maybe_close_idle(#st{}) -> {ok, #st{}}.
+maybe_close_idle(#st{open=Open, soft_max_indexes=Max}=State) when Open < Max ->
{ok, State};
maybe_close_idle(State) ->
- try
- {ok, close_idle(State)}
- catch error:all_active ->
- {error, all_active}
- end.
-
--spec close_idle(#st{}) -> #st{}.
-close_idle(State) ->
ets:safe_fixtable(?BY_IDLE, true),
try
- close_idle(State, ets:first(?BY_IDLE))
+ {ok, close_idle(State, ets:first(?BY_IDLE))}
after
ets:safe_fixtable(?BY_IDLE, false)
end.
-spec close_idle(#st{}, term()) -> #st{}.
-close_idle(_State, '$end_of_table') ->
- erlang:error(all_active);
+close_idle(State, '$end_of_table') ->
+ State;
close_idle(State, Name) ->
+ true = ets:delete(?BY_IDLE, Name),
case ets:lookup(?BY_SIG, Name) of
- [{_, {Pid, _Monitor, SysOwned}}] ->
- true = ets:delete(?BY_IDLE, Name),
+ [{_, {Pid, _Monitor}}] ->
couch_index:stop(Pid),
- closed(State, SysOwned);
+ case closed(State) of
+ NewState when NewState#st.open > State#st.soft_max_indexes ->
+ close_idle(NewState, ets:next(?BY_IDLE, Name));
+ NewState ->
+ NewState
+ end;
[] ->
- true = ets:delete(?BY_IDLE, Name),
close_idle(State, ets:next(?BY_IDLE, Name))
end.
-handle_call({get_index, {_Mod, _IdxState, DbName, Sig, SysOwned}=Args}, From, State) ->
+handle_call({get_index, {_Mod, _IdxState, DbName, Sig}=Args}, From, State) ->
case ets:lookup(?BY_SIG, {DbName, Sig}) of
[] ->
- case make_room(State, SysOwned) of
- {ok, NewState} ->
- spawn_link(fun() -> new_index(Args) end),
- Monitor = couch_index_monitor:spawn_link({DbName, Sig}, SysOwned),
- ets:insert(?BY_SIG, {{DbName, Sig}, {[From], Monitor, SysOwned}}),
- {noreply, NewState};
- {error, all_active} ->
- {reply, {error, all_active}, State}
- end;
- [{_, {Waiters, Monitor, SysOwned}}] when is_list(Waiters) ->
- ets:insert(?BY_SIG, {{DbName, Sig}, {[From | Waiters], Monitor, SysOwned}}),
+ {ok, NewState} = maybe_close_idle(State),
+ spawn_link(fun() -> new_index(Args) end),
+ Monitor = couch_index_monitor:spawn_link({DbName, Sig}),
+ ets:insert(?BY_SIG, {{DbName, Sig}, {[From], Monitor}}),
+ {noreply, NewState};
+ [{_, {Waiters, Monitor}}] when is_list(Waiters) ->
+ ets:insert(?BY_SIG, {{DbName, Sig}, {[From | Waiters], Monitor}}),
{noreply, State};
- [{_, {Pid, Monitor, _SysOwned}}] when is_pid(Pid) ->
+ [{_, {Pid, Monitor}}] when is_pid(Pid) ->
ok = incref({DbName, Sig}),
ok = couch_index_monitor:notify(Monitor, From),
{reply, {ok, Pid}, State}
end;
handle_call({async_open, {DbName, DDocId, Sig}, {ok, Pid}}, _From, State) ->
- [{_, {Waiters, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+ [{_, {Waiters, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
link(Pid),
- ets:insert(?BY_SIG, {{DbName, Sig}, {Pid, Monitor, SysOwned}}),
+ ets:insert(?BY_SIG, {{DbName, Sig}, {Pid, Monitor}}),
ets:insert(?BY_PID, {Pid, {DbName, Sig}}),
ets:insert(?BY_COUNTERS, {{DbName, Sig}, 0}),
ets:insert(?BY_DB, {DbName, {DDocId, Sig}}),
@@ -225,7 +203,7 @@ handle_call({async_open, {DbName, DDocId, Sig}, {ok, Pid}}, _From, State) ->
ok = couch_index_monitor:notify(Monitor, Client),
gen_server:reply(From, {ok, Pid})
end, Waiters),
- {reply, ok, opened(State, SysOwned)};
+ {reply, ok, opened(State)};
handle_call({async_error, {DbName, _DDocId, Sig}, Error}, {FromPid, _}, State) ->
[{_, {Waiters, Monitor, _SO}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
[gen_server:reply(From, Error) || From <- Waiters],
@@ -235,12 +213,12 @@ handle_call({async_error, {DbName, _DDocId, Sig}, Error}, {FromPid, _}, State) -
true = ets:delete(?BY_IDLE, {DbName, Sig}),
ok = couch_index_monitor:close(Monitor),
{reply, ok, State};
-handle_call({set_max_indexes_open, Max}, _From, State) ->
- {reply, ok, State#st{max_indexes=Max}};
+handle_call({set_soft_max_indexes_open, Max}, _From, State) ->
+ {reply, ok, State#st{soft_max_indexes=Max}};
handle_call({reset_indexes, DbName}, _From, State) ->
{reply, ok, reset_indexes(DbName, State)};
handle_call(open_index_count, _From, State) ->
- {reply, {State#st.open, State#st.sys_open}, State};
+ {reply, State#st.open, State};
handle_call(get_server, _From, State) ->
{reply, State, State}.
@@ -251,11 +229,10 @@ handle_cast({reset_indexes, DbName}, State) ->
handle_info({'EXIT', Pid, Reason}, Server) ->
case ets:lookup(?BY_PID, Pid) of
[{_, {DbName, Sig}}] ->
- [{_, {_W, _M, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
[{DbName, {DDocId, Sig}}] =
ets:match_object(?BY_DB, {DbName, {'$1', Sig}}),
rem_from_ets(DbName, Sig, DDocId, Pid),
- {noreply, closed(Server, SysOwned)};
+ {noreply, closed(Server)};
[] when Reason /= normal ->
exit(Reason);
_Else ->
@@ -283,8 +260,8 @@ handle_config_change("couchdb", "index_dir", _, _, _) ->
handle_config_change("couchdb", "view_index_dir", _, _, _) ->
exit(whereis(couch_index_server), config_change),
remove_handler;
-handle_config_change("couchdb", "max_indexes_open", Max, _, _) when is_list(Max) ->
- {ok, gen_server:call(?MODULE, {set_max_indexes_open, list_to_integer(Max)})};
+handle_config_change("couchdb", "soft_max_indexes_open", Max, _, _) when is_list(Max) ->
+ {ok, gen_server:call(?MODULE, {set_soft_max_indexes_open, list_to_integer(Max)})};
handle_config_change(_, _, _, _, _) ->
{ok, nil}.
@@ -295,7 +272,7 @@ handle_config_terminate(_Server, _Reason, _State) ->
{ok, couch_index_util:root_dir()}.
-new_index({Mod, IdxState, DbName, Sig, _SysOwned}) ->
+new_index({Mod, IdxState, DbName, Sig}) ->
DDocId = Mod:get(idx_name, IdxState),
case couch_index:start_link({Mod, IdxState}) of
{ok, Pid} ->
@@ -312,13 +289,13 @@ reset_indexes(DbName, State) ->
#st{root_dir=Root} = State,
% shutdown all the updaters and clear the files, the db got changed
Fun = fun({_, {DDocId, Sig}}, StateAcc) ->
- [{_, {Pid, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+ [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
couch_index_monitor:close(Monitor),
MRef = erlang:monitor(process, Pid),
gen_server:cast(Pid, delete),
receive {'DOWN', MRef, _, _, _} -> ok end,
rem_from_ets(DbName, Sig, DDocId, Pid),
- closed(StateAcc, SysOwned)
+ closed(StateAcc)
end,
NewState = lists:foldl(Fun, State, ets:lookup(?BY_DB, DbName)),
Path = couch_index_util:index_dir("", DbName),
@@ -343,7 +320,7 @@ handle_db_event(DbName, deleted, St) ->
handle_db_event(DbName, {ddoc_updated, DDocId}, St) ->
lists:foreach(fun({_DbName, {_DDocId, Sig}}) ->
case ets:lookup(?BY_SIG, {DbName, Sig}) of
- [{_, {IndexPid, _Monitor, _SysOwned}}] ->
+ [{_, {IndexPid, _Monitor}}] ->
(catch gen_server:cast(IndexPid, ddoc_updated));
[] ->
ok
@@ -354,20 +331,14 @@ handle_db_event(_DbName, _Event, St) ->
{ok, St}.
--spec opened(#st{}, boolean()) -> #st{}.
-opened(State, IsSysOwned) ->
- case IsSysOwned of
- true -> State#st{sys_open=State#st.sys_open + 1};
- false -> State#st{open=State#st.open + 1}
- end.
+-spec opened(#st{}) -> #st{}.
+opened(State) ->
+ State#st{open=State#st.open + 1}.
--spec closed(#st{}, boolean()) -> #st{}.
-closed(State, IsSysOwned) ->
- case IsSysOwned of
- true -> State#st{sys_open=State#st.sys_open - 1};
- false -> State#st{open=State#st.open - 1}
- end.
+-spec closed(#st{}) -> #st{}.
+closed(State) ->
+ State#st{open=State#st.open - 1}.
incref(Name) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-index/blob/78a65a3f/test/couch_index_monitor_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_index_monitor_tests.erl b/test/couch_index_monitor_tests.erl
index d4fd2c9..9e0d8e0 100644
--- a/test/couch_index_monitor_tests.erl
+++ b/test/couch_index_monitor_tests.erl
@@ -60,10 +60,9 @@ test_basic({Db, DDoc}) ->
{ok, IdxState} = couch_mrview_index:init(Db, DDoc),
DbName = couch_mrview_index:get(db_name, IdxState),
Sig = couch_mrview_index:get(signature, IdxState),
- [{_, {Pid, Monitor, SysOwned}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
+ [{_, {Pid, Monitor}}] = ets:lookup(?BY_SIG, {DbName, Sig}),
?assert(is_pid(Pid)),
?assert(is_pid(Monitor)),
- ?assertNot(SysOwned),
?assertEqual(1, get_count({DbName, Sig})),
Acq1 = spawn_monitor_acquirer(DbName, DDoc#doc.id),
?assertEqual(2, get_count({DbName, Sig})),
@@ -73,7 +72,7 @@ test_basic({Db, DDoc}) ->
?assertEqual(2, get_count({DbName, Sig})),
wait_down(Acq1),
?assertEqual(1, get_count({DbName, Sig})),
- couch_index_monitor:cancel({DbName, Sig}, {self(), Monitor, SysOwned}),
+ couch_index_monitor:cancel({DbName, Sig}, {self(), Monitor}),
?assertEqual(0, get_count({DbName, Sig})),
?assertEqual(1, length(ets:lookup(?BY_IDLE, {DbName, Sig}))),
ok