You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/06/28 22:22:51 UTC
[couchdb] branch optimize-ddoc-cache updated (0422143 -> 629c2cf)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch optimize-ddoc-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 0422143 Rewrite ddoc_cache to minimize evictions
new 2443c2f Tweaks for performance
new 3c2b480 TMP: Simple benchmark script
new 629c2cf Rewrite the rewrite
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/ddoc_cache/src/ddoc_cache.erl | 8 +-
src/ddoc_cache/src/ddoc_cache.hrl | 2 +-
src/ddoc_cache/src/ddoc_cache_entry.erl | 264 +++++++++++++++++++---
src/ddoc_cache/src/ddoc_cache_lru.erl | 257 +++++++++------------
src/ddoc_cache/src/ddoc_cache_opener.erl | 81 +------
src/ddoc_cache/src/ddoc_cache_refresher.erl | 102 ---------
src/ddoc_cache/src/ddoc_cache_speed.erl | 64 ++++++
src/ddoc_cache/src/ddoc_cache_tables.erl | 3 +-
src/ddoc_cache/test/ddoc_cache_basic_test.erl | 3 +-
src/ddoc_cache/test/ddoc_cache_coverage_test.erl | 28 +--
src/ddoc_cache/test/ddoc_cache_opener_test.erl | 114 +++++-----
src/ddoc_cache/test/ddoc_cache_refresher_test.erl | 167 --------------
src/ddoc_cache/test/ddoc_cache_remove_test.erl | 6 +-
src/ddoc_cache/test/ddoc_cache_tutil.erl | 2 +-
14 files changed, 486 insertions(+), 615 deletions(-)
delete mode 100644 src/ddoc_cache/src/ddoc_cache_refresher.erl
create mode 100644 src/ddoc_cache/src/ddoc_cache_speed.erl
delete mode 100644 src/ddoc_cache/test/ddoc_cache_refresher_test.erl
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].
[couchdb] 03/03: Rewrite the rewrite
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch optimize-ddoc-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 629c2cf50dcf405ce80dc66b406c6af1c61b8172
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Jun 28 17:12:28 2017 -0500
Rewrite the rewrite
Russel managed to nerd snipe me pretty good with his comments on the PR
so I've spent a lot of time trying to figure out a more parallel method
for managing the LRU.
This approach changes things so that every entry in the cache has an
associated pid that manages all of the ets table entries. This way we
can insert and delete all of our LRU timestamps in parallel.
Initial testing shows throughput in the millions of operations per
second.
Also for an aside, I don't think the original suggested appraoch for
having individual clients write to ets tables is going to work as well.
It worries me that we'll end up with some strange combination of
operations that ends up orphaning entries. But it seems like the
parallelism of the maximum cache size should be fairly acceptable.
---
src/ddoc_cache/src/ddoc_cache.erl | 8 +-
src/ddoc_cache/src/ddoc_cache.hrl | 2 +-
src/ddoc_cache/src/ddoc_cache_entry.erl | 264 +++++++++++++++++++---
src/ddoc_cache/src/ddoc_cache_lru.erl | 259 +++++++++------------
src/ddoc_cache/src/ddoc_cache_opener.erl | 81 +------
src/ddoc_cache/src/ddoc_cache_refresher.erl | 102 ---------
src/ddoc_cache/src/ddoc_cache_speed.erl | 9 +-
src/ddoc_cache/src/ddoc_cache_tables.erl | 3 +-
src/ddoc_cache/test/ddoc_cache_basic_test.erl | 3 +-
src/ddoc_cache/test/ddoc_cache_coverage_test.erl | 28 +--
src/ddoc_cache/test/ddoc_cache_opener_test.erl | 114 +++++-----
src/ddoc_cache/test/ddoc_cache_refresher_test.erl | 167 --------------
src/ddoc_cache/test/ddoc_cache_remove_test.erl | 6 +-
src/ddoc_cache/test/ddoc_cache_tutil.erl | 2 +-
14 files changed, 428 insertions(+), 620 deletions(-)
diff --git a/src/ddoc_cache/src/ddoc_cache.erl b/src/ddoc_cache/src/ddoc_cache.erl
index ff87258..50cac30 100644
--- a/src/ddoc_cache/src/ddoc_cache.erl
+++ b/src/ddoc_cache/src/ddoc_cache.erl
@@ -27,22 +27,22 @@
open_doc(DbName, DocId) ->
Key = {ddoc_cache_entry_ddocid, {DbName, DocId}},
- ddoc_cache_opener:open(Key).
+ ddoc_cache_lru:open(Key).
open_doc(DbName, DocId, RevId) ->
Key = {ddoc_cache_entry_ddocid_rev, {DbName, DocId, RevId}},
- ddoc_cache_opener:open(Key).
+ ddoc_cache_lru:open(Key).
open_validation_funs(DbName) ->
Key = {ddoc_cache_entry_validation_funs, DbName},
- ddoc_cache_opener:open(Key).
+ ddoc_cache_lru:open(Key).
open_custom(DbName, Mod) ->
Key = {ddoc_cache_entry_custom, {DbName, Mod}},
- ddoc_cache_opener:open(Key).
+ ddoc_cache_lru:open(Key).
refresh(ShardDbName, DDocIds) when is_list(DDocIds) ->
diff --git a/src/ddoc_cache/src/ddoc_cache.hrl b/src/ddoc_cache/src/ddoc_cache.hrl
index e11b350..dba0d37 100644
--- a/src/ddoc_cache/src/ddoc_cache.hrl
+++ b/src/ddoc_cache/src/ddoc_cache.hrl
@@ -17,8 +17,8 @@
-define(CACHE, ddoc_cache_entries).
-define(LRU, ddoc_cache_lru).
--define(OPENERS, ddoc_cache_openers).
-define(REFRESH_TIMEOUT, 67000).
+-define(SHUTDOWN_TIMEOUT, 1000).
-record(entry, {
key,
diff --git a/src/ddoc_cache/src/ddoc_cache_entry.erl b/src/ddoc_cache/src/ddoc_cache_entry.erl
index 33f2b4c..4689390 100644
--- a/src/ddoc_cache/src/ddoc_cache_entry.erl
+++ b/src/ddoc_cache/src/ddoc_cache_entry.erl
@@ -14,21 +14,43 @@
-export([
- spawn_opener/1,
- spawn_refresher/1,
+ dbname/1,
+ ddocid/1,
+ recover/1,
- open/1,
- handle_resp/1,
+ start_link/1,
+ shutdown/1,
+ open/2,
+ accessed/1,
+ refresh/1
+]).
- dbname/1,
- ddocid/1
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
]).
-export([
- do_open/2
+ do_open/1
]).
+-include("ddoc_cache.hrl").
+
+
+-record(st, {
+ key,
+ val,
+ opener,
+ waiters,
+ ts
+}).
+
+
dbname({Mod, Arg}) ->
Mod:dbname(Arg).
@@ -37,40 +59,226 @@ ddocid({Mod, Arg}) ->
Mod:ddocid(Arg).
-spawn_opener(Key) ->
- erlang:spawn_link(?MODULE, do_open, [Key, true]).
+recover({Mod, Arg}) ->
+ Mod:recover(Arg).
+
+
+start_link(Key) ->
+ Pid = proc_lib:spawn_link(?MODULE, init, [Key]),
+ {ok, Pid}.
+
+
+shutdown(Pid) ->
+ ok = gen_server:call(Pid, shutdown).
+
+
+open(Pid, Key) ->
+ try
+ Resp = gen_server:call(Pid, open),
+ case Resp of
+ {open_ok, Val} ->
+ Val;
+ {open_error, {T, R, S}} ->
+ erlang:raise(T, R, S)
+ end
+ catch exit:_ ->
+ % Its possible that this process was evicted just
+ % before we tried talking to it. Just fallback
+ % to a standard recovery
+ recover(Key)
+ end.
+
+
+accessed(Pid) ->
+ gen_server:cast(Pid, accessed).
+
+
+refresh(Pid) ->
+ gen_server:cast(Pid, refresh).
+
+
+init(Key) ->
+ Entry = #entry{
+ key = Key,
+ val = undefined,
+ pid = self()
+ },
+ true = ets:insert(?CACHE, Entry),
+ St = #st{
+ key = Key,
+ opener = spawn_opener(Key),
+ waiters = []
+ },
+ gen_server:enter_loop(?MODULE, [], St).
+
+
+terminate(_Reason, St) ->
+ #st{
+ key = Key,
+ opener = Pid,
+ ts = Ts
+ } = St,
+ % We may have already deleted our cache entry
+ % during shutdown
+ Pattern = #entry{key = Key, pid = self(), _ = '_'},
+ CacheMSpec = [{Pattern, [], [true]}],
+ true = ets:select_delete(?CACHE, CacheMSpec) < 2,
+ % We may have already deleted our LRU entry
+ % during shutdown
+ if Ts == undefined -> ok; true ->
+ LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+ true = ets:select_delete(?LRU, LruMSpec) < 2
+ end,
+ % Blow away any current opener if it exists
+ if not is_pid(Pid) -> ok; true ->
+ catch exit(Pid, kill)
+ end,
+ ok.
+
+handle_call(open, From, #st{val = undefined} = St) ->
+ NewSt = St#st{
+ waiters = [From | St#st.waiters]
+ },
+ {noreply, NewSt};
-spawn_refresher(Key) ->
- erlang:spawn_monitor(?MODULE, do_open, [Key, false]).
+handle_call(open, _From, St) ->
+ {reply, St#st.val, St};
+handle_call(shutdown, _From, St) ->
+ remove_from_cache(St),
+ {stop, normal, ok, St};
-handle_resp({open_ok, _Key, Resp}) ->
- Resp;
+handle_call(Msg, _From, St) ->
+ {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
-handle_resp({open_error, _Key, Type, Reason, Stack}) ->
- erlang:raise(Type, Reason, Stack);
-handle_resp(Other) ->
- erlang:error({ddoc_cache_error, Other}).
+handle_cast(accessed, St) ->
+ ?EVENT(accessed, St#st.key),
+ drain_accessed(),
+ {noreply, update_lru(St)};
+handle_cast(refresh, #st{opener = Ref} = St) when is_reference(Ref) ->
+ #st{
+ key = Key
+ } = St,
+ erlang:cancel_timer(Ref),
+ NewSt = St#st{
+ opener = spawn_opener(Key)
+ },
+ {noreply, NewSt};
-open(Key) ->
- {_Pid, Ref} = erlang:spawn_monitor(?MODULE, do_open, [Key, false]),
+handle_cast(refresh, #st{opener = Pid} = St) when is_pid(Pid) ->
+ catch exit(Pid, kill),
receive
- {'DOWN', Ref, _, _, Resp} ->
- handle_resp(Resp)
- end.
+ {'DOWN', _, _, Pid, _} -> ok
+ end,
+ NewSt = St#st{
+ opener = spawn_opener(St#st.key)
+ },
+ {noreply, NewSt};
+
+handle_cast(Msg, St) ->
+ {stop, {bad_cast, Msg}, St}.
+
+
+handle_info({'DOWN', _, _, Pid, Resp}, #st{key = Key, opener = Pid} = St) ->
+ case Resp of
+ {open_ok, Key, {ok, Val}} ->
+ if not is_list(St#st.waiters) -> ok; true ->
+ respond(St#st.waiters, {open_ok, {ok, Val}})
+ end,
+ update_cache(St, Val),
+ Msg = {'$gen_cast', refresh},
+ Timer = erlang:send_after(?REFRESH_TIMEOUT, self(), Msg),
+ NewSt = St#st{
+ val = {open_ok, {ok, Val}},
+ opener = Timer
+ },
+ {noreply, update_lru(NewSt)};
+ {Status, Key, Other} ->
+ NewSt = St#st{
+ val = {Status, Other},
+ opener = undefined,
+ waiters = undefined
+ },
+ remove_from_cache(NewSt),
+ if not is_list(St#st.waiters) -> ok; true ->
+ respond(St#st.waiters, {Status, Other})
+ end,
+ {stop, normal, NewSt}
+ end;
+
+handle_info(Msg, St) ->
+ {stop, {bad_info, Msg}, St}.
+
+
+code_change(_, St, _) ->
+ {ok, St}.
-do_open({Mod, Arg} = Key, DoInsert) ->
- try Mod:recover(Arg) of
- {ok, Resp} when DoInsert ->
- ddoc_cache_lru:insert(Key, Resp),
- erlang:exit({open_ok, Key, {ok, Resp}});
+spawn_opener(Key) ->
+ {Pid, _} = erlang:spawn_monitor(?MODULE, do_open, [Key]),
+ Pid.
+
+
+do_open(Key) ->
+ try recover(Key) of
Resp ->
erlang:exit({open_ok, Key, Resp})
catch T:R ->
S = erlang:get_stacktrace(),
- erlang:exit({open_error, Key, T, R, S})
+ erlang:exit({open_error, Key, {T, R, S}})
end.
+
+
+update_lru(#st{key = Key, ts = Ts} = St) ->
+ if Ts == undefined -> ok; true ->
+ MSpec = [{{{Ts, Key, self()}}, [], [true]}],
+ 1 = ets:select_delete(?LRU, MSpec)
+ end,
+ NewTs = os:timestamp(),
+ true = ets:insert(?LRU, {{NewTs, Key, self()}}),
+ St#st{ts = NewTs}.
+
+
+update_cache(#st{val = undefined} = St, Val) ->
+ true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+ ?EVENT(inserted, St#st.key);
+
+update_cache(#st{val = V1} = _St, V2) when {open_ok, {ok, V2}} == V1 ->
+ ?EVENT(update_noop, _St#st.key);
+
+update_cache(St, Val) ->
+ true = ets:update_element(?CACHE, St#st.key, {#entry.val, Val}),
+ ?EVENT(updated, {St#st.key, Val}).
+
+
+remove_from_cache(St) ->
+ #st{
+ key = Key,
+ ts = Ts
+ } = St,
+ Pattern = #entry{key = Key, pid = self(), _ = '_'},
+ CacheMSpec = [{Pattern, [], [true]}],
+ 1 = ets:select_delete(?CACHE, CacheMSpec),
+ if Ts == undefined -> ok; true ->
+ LruMSpec = [{{{Ts, Key, self()}}, [], [true]}],
+ 1 = ets:select_delete(?LRU, LruMSpec)
+ end,
+ ?EVENT(removed, St#st.key),
+ ok.
+
+
+drain_accessed() ->
+ receive
+ {'$gen_cast', accessed} ->
+ drain_accessed()
+ after 0 ->
+ ok
+ end.
+
+
+respond(Waiters, Resp) ->
+ [gen_server:reply(W, Resp) || W <- Waiters].
+
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl
index 5b10959..5988f03 100644
--- a/src/ddoc_cache/src/ddoc_cache_lru.erl
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -17,11 +17,7 @@
-export([
start_link/0,
-
- insert/2,
- accessed/1,
- update/2,
- remove/1,
+ open/1,
refresh/2
]).
@@ -43,9 +39,9 @@
-record(st, {
- atimes, % key -> time
- dbs, % dbname -> docid -> key -> []
- time,
+ pids, % pid -> key
+ dbs, % dbname -> docid -> key -> pid
+ size,
evictor
}).
@@ -54,20 +50,28 @@ start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-insert(Key, Val) ->
- gen_server:call(?MODULE, {insert, Key, Val}, infinity).
-
-
-accessed(Key) ->
- gen_server:call(?MODULE, {accessed, Key}).
-
-
-update(Key, Val) ->
- gen_server:call(?MODULE, {update, Key, Val}, infinity).
-
-
-remove(Key) ->
- gen_server:call(?MODULE, {remove, Key}, infinity).
+open(Key) ->
+ try ets:lookup(?CACHE, Key) of
+ [] ->
+ couch_stats:increment_counter([ddoc_cache, miss]),
+ case gen_server:call(?MODULE, {start, Key}, infinity) of
+ {ok, Pid} ->
+ ddoc_cache_entry:open(Pid, Key);
+ full ->
+ couch_stats:increment_counter([ddoc_cache, recovery]),
+ ddoc_cache_entry:recover(Key)
+ end;
+ [#entry{val = undefined, pid = Pid}] ->
+ couch_stats:increment_counter([ddoc_cache, miss]),
+ ddoc_cache_entry:open(Pid, Key);
+ [#entry{val = Val, pid = Pid}] ->
+ couch_stats:increment_counter([ddoc_cache, hit]),
+ ddoc_cache_entry:accessed(Pid),
+ {ok, Val}
+ catch _:_ ->
+ couch_stats:increment_counter([ddoc_cache, recovery]),
+ ddoc_cache_entry:recover(Key)
+ end.
refresh(DbName, DDocIds) ->
@@ -76,115 +80,48 @@ refresh(DbName, DDocIds) ->
init(_) ->
process_flag(trap_exit, true),
- {ok, ATimes} = khash:new(),
+ {ok, Pids} = khash:new(),
{ok, Dbs} = khash:new(),
{ok, Evictor} = couch_event:link_listener(
?MODULE, handle_db_event, nil, [all_dbs]
),
{ok, #st{
- atimes = ATimes,
+ pids = Pids,
dbs = Dbs,
- time = 0,
+ size = 0,
evictor = Evictor
}}.
terminate(_Reason, St) ->
case is_pid(St#st.evictor) of
- true -> exit(St#st.evictor, kill);
+ true -> catch exit(St#st.evictor, kill);
false -> ok
end,
ok.
-handle_call({insert, Key, Val}, _From, St) ->
+handle_call({start, Key}, _From, St) ->
#st{
- atimes = ATimes,
+ pids = Pids,
dbs = Dbs,
- time = Time
- } = St,
- NewTime = Time + 1,
- NewSt = St#st{time = NewTime},
- Pid = ddoc_cache_refresher:spawn_link(Key, ?REFRESH_TIMEOUT),
- true = ets:insert(?CACHE, #entry{key = Key, val = Val, pid = Pid}),
- true = ets:insert(?LRU, {{NewTime, Key}}),
- ok = khash:put(ATimes, Key, NewTime),
- store_key(Dbs, Key),
- trim(NewSt),
- ?EVENT(inserted, {Key, Val}),
- {reply, ok, NewSt};
-
-handle_call({accessed, Key}, _From, St) ->
- {noreply, NewSt} = handle_cast({accessed, Key}, St),
- {reply, ok, NewSt};
-
-handle_call({update, Key, Val}, _From, St) ->
- #st{
- atimes = ATimes
+ size = CurSize
} = St,
- case khash:lookup(ATimes, Key) of
- {value, _} ->
- ets:update_element(?CACHE, Key, {#entry.val, Val}),
- ?EVENT(updated, {Key, Val}),
- {reply, ok, St};
- not_found ->
- {reply, evicted, St}
+ MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 500)),
+ case trim(St, CurSize, MaxSize) of
+ {ok, N} ->
+ {ok, Pid} = ddoc_cache_entry:start_link(Key),
+ ok = khash:put(Pids, Pid, Key),
+ store_key(Dbs, Key, Pid),
+ {reply, {ok, Pid}, St#st{size = CurSize - N + 1}};
+ full ->
+ {reply, full, St}
end;
-handle_call({remove, Key}, _From, St) ->
- #st{
- atimes = ATimes,
- dbs = Dbs
- } = St,
- case khash:lookup(ATimes, Key) of
- {value, ATime} ->
- [#entry{pid = Pid}] = ets:lookup(?CACHE, Key),
- ddoc_cache_refresher:stop(Pid),
- remove_key(St, Key, ATime),
-
- DbName = ddoc_cache_entry:dbname(Key),
- DDocId = ddoc_cache_entry:ddocid(Key),
- {value, DDocIds} = khash:lookup(Dbs, DbName),
- {value, Keys} = khash:lookup(DDocIds, DDocId),
- ok = khash:del(Keys, Key),
- case khash:size(Keys) of
- 0 -> khash:del(DDocIds, DDocId);
- _ -> ok
- end,
- case khash:size(DDocIds) of
- 0 -> khash:del(Dbs, DDocId);
- _ -> ok
- end,
-
- ?EVENT(removed, Key);
- not_found ->
- ok
- end,
- {reply, ok, St};
-
handle_call(Msg, _From, St) ->
{stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
-handle_cast({accessed, Key}, St) ->
- #st{
- atimes = ATimes,
- time = Time
- } = St,
- NewTime = Time + 1,
- case khash:lookup(ATimes, Key) of
- {value, OldTime} ->
- true = ets:delete(?LRU, {OldTime, Key}),
- true = ets:insert(?LRU, {{NewTime, Key}}),
- ok = khash:put(ATimes, Key, NewTime),
- ?EVENT(accessed, Key);
- not_found ->
- % Likely a client read from the cache while an
- % eviction message was in our mailbox
- ok
- end,
- {noreply, St};
-
handle_cast({evict, DbName}, St) ->
gen_server:abcast(mem3:nodes(), ?MODULE, {do_evict, DbName}),
{noreply, St};
@@ -197,21 +134,21 @@ handle_cast({do_evict, DbName}, St) ->
#st{
dbs = Dbs
} = St,
- case khash:lookup(Dbs, DbName) of
+ ToRem = case khash:lookup(Dbs, DbName) of
{value, DDocIds} ->
- khash:fold(DDocIds, fun(_, Keys, _) ->
- khash:fold(Keys, fun(Key, _, _) ->
- [#entry{pid = Pid}] = ets:lookup(?CACHE, Key),
- ddoc_cache_refresher:stop(Pid),
- remove_key(St, Key)
- end, nil)
- end, nil),
- khash:del(Dbs, DbName),
- ?EVENT(evicted, DbName);
+ AccOut = khash:fold(DDocIds, fun(_, Keys, Acc1) ->
+ khash:to_list(Keys) ++ Acc1
+ end, []),
+ ?EVENT(evicted, DbName),
+ AccOut;
not_found ->
?EVENT(evict_noop, DbName),
- ok
+ []
end,
+ lists:foreach(fun({Key, Pid}) ->
+ remove_entry(St, Key, Pid)
+ end, ToRem),
+ khash:del(Dbs, DbName),
{noreply, St};
handle_cast({do_refresh, DbName, DDocIdList}, St) ->
@@ -223,9 +160,8 @@ handle_cast({do_refresh, DbName, DDocIdList}, St) ->
lists:foreach(fun(DDocId) ->
case khash:lookup(DDocIds, DDocId) of
{value, Keys} ->
- khash:fold(Keys, fun(Key, _, _) ->
- [#entry{pid = Pid}] = ets:lookup(?CACHE, Key),
- ddoc_cache_refresher:refresh(Pid)
+ khash:fold(Keys, fun(_, Pid, _) ->
+ ddoc_cache_entry:refresh(Pid)
end, nil);
not_found ->
ok
@@ -247,6 +183,18 @@ handle_info({'EXIT', Pid, _Reason}, #st{evictor = Pid} = St) ->
),
{noreply, St#st{evictor=Evictor}};
+handle_info({'EXIT', Pid, normal}, St) ->
+ % This clause handles when an entry starts
+ % up but encounters an error or uncacheable
+ % response from its recover call.
+ #st{
+ pids = Pids
+ } = St,
+ {value, Key} = khash:lookup(Pids, Pid),
+ khash:del(Pids, Pid),
+ remove_key(St, Key),
+ {noreply, St};
+
handle_info(Msg, St) ->
{stop, {invalid_info, Msg}, St}.
@@ -267,20 +215,46 @@ handle_db_event(_DbName, _Event, St) ->
{ok, St}.
-store_key(Dbs, Key) ->
+trim(_, _, 0) ->
+ full;
+
+trim(_St, CurSize, MaxSize) when CurSize < MaxSize ->
+ {ok, 0};
+
+trim(St, CurSize, MaxSize) when CurSize >= MaxSize ->
+ case ets:first(?LRU) of
+ '$end_of_table' ->
+ full;
+ {_Ts, Key, Pid} ->
+ remove_entry(St, Key, Pid),
+ {ok, 1}
+ end.
+
+
+remove_entry(St, Key, Pid) ->
+ #st{
+ pids = Pids
+ } = St,
+ unlink(Pid),
+ ddoc_cache_entry:shutdown(Pid),
+ khash:del(Pids, Pid),
+ remove_key(St, Key).
+
+
+store_key(Dbs, Key, Pid) ->
DbName = ddoc_cache_entry:dbname(Key),
DDocId = ddoc_cache_entry:ddocid(Key),
case khash:lookup(Dbs, DbName) of
{value, DDocIds} ->
case khash:lookup(DDocIds, DDocId) of
{value, Keys} ->
- khash:put(Keys, Key, []);
+ khash:put(Keys, Key, Pid);
not_found ->
- {ok, Keys} = khash:from_list([{Key, []}]),
+ {ok, Keys} = khash:from_list([{Key, Pid}]),
khash:put(DDocIds, DDocId, Keys)
end;
not_found ->
- {ok, Keys} = khash:from_list([{Key, []}]),
+ {ok, Keys} = khash:from_list([{Key, Pid}]),
{ok, DDocIds} = khash:from_list([{DDocId, Keys}]),
khash:put(Dbs, DbName, DDocIds)
end.
@@ -288,31 +262,18 @@ store_key(Dbs, Key) ->
remove_key(St, Key) ->
#st{
- atimes = ATimes
- } = St,
- {value, ATime} = khash:lookup(ATimes, Key),
- remove_key(St, Key, ATime).
-
-
-remove_key(St, Key, ATime) ->
- #st{
- atimes = ATimes
- } = St,
- true = ets:delete(?CACHE, Key),
- true = ets:delete(?LRU, {ATime, Key}),
- ok = khash:del(ATimes, Key).
-
-
-trim(St) ->
- #st{
- atimes = ATimes
+ dbs = Dbs
} = St,
- MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 5000)),
- case khash:size(ATimes) > MaxSize of
- true ->
- {ATime, Key} = ets:first(?LRU),
- remove_key(St, Key, ATime),
- trim(St);
- false ->
- ok
+ DbName = ddoc_cache_entry:dbname(Key),
+ DDocId = ddoc_cache_entry:ddocid(Key),
+ {value, DDocIds} = khash:lookup(Dbs, DbName),
+ {value, Keys} = khash:lookup(DDocIds, DDocId),
+ khash:del(Keys, Key),
+ case khash:size(Keys) of
+ 0 -> khash:del(DDocIds, DDocId);
+ _ -> ok
+ end,
+ case khash:size(DDocIds) of
+ 0 -> khash:del(Dbs, DbName);
+ _ -> ok
end.
diff --git a/src/ddoc_cache/src/ddoc_cache_opener.erl b/src/ddoc_cache/src/ddoc_cache_opener.erl
index 8368f82..7839bcb 100644
--- a/src/ddoc_cache/src/ddoc_cache_opener.erl
+++ b/src/ddoc_cache/src/ddoc_cache_opener.erl
@@ -14,81 +14,30 @@
-behaviour(gen_server).
-vsn(1).
--include_lib("couch/include/couch_db.hrl").
--include_lib("mem3/include/mem3.hrl").
-
-export([
start_link/0
]).
+
-export([
init/1,
terminate/2,
-
handle_call/3,
handle_cast/2,
handle_info/2,
-
code_change/3
]).
--export([
- open/1
-]).
-
--include("ddoc_cache.hrl").
-
-
--record(st, {
- db_ddocs
-}).
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-open(Key) ->
- try ets:lookup(?CACHE, Key) of
- [] ->
- couch_stats:increment_counter([ddoc_cache, miss]),
- Resp = gen_server:call(?MODULE, {open, Key}, infinity),
- ddoc_cache_entry:handle_resp(Resp);
- [#entry{val = Val}] ->
- couch_stats:increment_counter([ddoc_cache, hit]),
- ddoc_cache_lru:accessed(Key),
- {ok, Val}
- catch _:_ ->
- couch_stats:increment_counter([ddoc_cache, recovery]),
- ddoc_cache_entry:open(Key)
- end.
-
-
init(_) ->
- process_flag(trap_exit, true),
- {ok, #st{}}.
+ {ok, nil}.
terminate(_Reason, _St) ->
ok.
-handle_call({open, OpenerKey}, From, St) ->
- case ets:lookup(?CACHE, OpenerKey) of
- [] ->
- case ets:lookup(?OPENERS, OpenerKey) of
- [#opener{clients=Clients}=O] ->
- ets:insert(?OPENERS, O#opener{clients=[From | Clients]}),
- {noreply, St};
- [] ->
- Pid = ddoc_cache_entry:spawn_opener(OpenerKey),
- Opener = #opener{
- key = OpenerKey,
- pid = Pid,
- clients = [From]
- },
- ets:insert(?OPENERS, Opener),
- {noreply, St}
- end;
- [#entry{val = Val}] ->
- {reply, {ok, Val}, St}
- end;
handle_call(Msg, _From, St) ->
{stop, {invalid_call, Msg}, {invalid_call, Msg}, St}.
@@ -108,33 +57,9 @@ handle_cast(Msg, St) ->
{stop, {invalid_cast, Msg}, St}.
-handle_info({'EXIT', _Pid, {open_ok, OpenerKey, Resp}}, St) ->
- respond(OpenerKey, {open_ok, OpenerKey, Resp}),
- {noreply, St};
-
-handle_info({'EXIT', _Pid, {open_error, OpenerKey, Type, Reason, Stack}}, St) ->
- respond(OpenerKey, {open_error, OpenerKey, Type, Reason, Stack}),
- {noreply, St};
-
-handle_info({'EXIT', Pid, Reason}, St) ->
- Pattern = #opener{pid=Pid, _='_'},
- case ets:match_object(?OPENERS, Pattern) of
- [#opener{key=OpenerKey, clients=Clients}] ->
- [gen_server:reply(C, {error, Reason}) || C <- Clients],
- ets:delete(?OPENERS, OpenerKey),
- {noreply, St};
- [] ->
- {stop, {unknown_pid_died, {Pid, Reason}}, St}
- end;
-
handle_info(Msg, St) ->
{stop, {invalid_info, Msg}, St}.
+
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-
-
-respond(OpenerKey, Resp) ->
- [#opener{clients=Clients}] = ets:lookup(?OPENERS, OpenerKey),
- [gen_server:reply(C, Resp) || C <- Clients],
- ets:delete(?OPENERS, OpenerKey).
diff --git a/src/ddoc_cache/src/ddoc_cache_refresher.erl b/src/ddoc_cache/src/ddoc_cache_refresher.erl
deleted file mode 100644
index ee2d644..0000000
--- a/src/ddoc_cache/src/ddoc_cache_refresher.erl
+++ /dev/null
@@ -1,102 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(ddoc_cache_refresher).
-
-
--export([
- spawn_link/2,
- refresh/1,
- stop/1
-]).
-
-
--export([
- init/1
-]).
-
-
--include("ddoc_cache.hrl").
-
-
-spawn_link(Key, Interval) ->
- proc_lib:spawn_link(?MODULE, init, [{self(), Key, Interval}]).
-
-
-refresh(Pid) ->
- Pid ! refresh.
-
-
-stop(Pid) ->
- unlink(Pid),
- Pid ! stop.
-
-
-init({Parent, Key, Interval}) ->
- erlang:monitor(process, Parent),
- try
- loop(Key, Interval)
- catch T:R ->
- S = erlang:get_stacktrace(),
- exit({T, R, S})
- end.
-
-
-loop(Key, Interval) ->
- receive
- refresh ->
- do_refresh(Key, Interval);
- stop ->
- ok
- after Interval ->
- do_refresh(Key, Interval)
- end.
-
-
-do_refresh(Key, Interval) ->
- drain_refreshes(),
- {_Pid, Ref} = ddoc_cache_entry:spawn_refresher(Key),
- receive
- {'DOWN', Ref, _, _, Resp} ->
- case Resp of
- {open_ok, Key, {ok, Val}} ->
- maybe_update(Key, Val, Interval);
- _Else ->
- ddoc_cache_lru:remove(Key)
- end
- end.
-
-
-drain_refreshes() ->
- receive
- refresh ->
- drain_refreshes()
- after 0 ->
- ok
- end.
-
-
-maybe_update(Key, Val, Interval) ->
- case ets:lookup(?CACHE, Key) of
- [] ->
- ok;
- [#entry{val = Val}] ->
- ?EVENT(update_noop, Key),
- loop(Key, Interval);
- [#entry{pid = Pid}] when Pid == self() ->
- case ddoc_cache_lru:update(Key, Val) of
- ok ->
- loop(Key, Interval);
- evicted ->
- ok
- end
- end.
diff --git a/src/ddoc_cache/src/ddoc_cache_speed.erl b/src/ddoc_cache/src/ddoc_cache_speed.erl
index 38cd0b9..d172f21 100644
--- a/src/ddoc_cache/src/ddoc_cache_speed.erl
+++ b/src/ddoc_cache/src/ddoc_cache_speed.erl
@@ -6,7 +6,7 @@
]).
--define(RANGE, 1000).
+-include("ddoc_cache.hrl").
go(WorkerCount) when is_integer(WorkerCount), WorkerCount > 0 ->
@@ -24,7 +24,7 @@ spawn_workers(0) ->
spawn_workers(WorkerCount) ->
Self = self(),
WorkerDb = list_to_binary(integer_to_list(WorkerCount)),
- spawn(fun() ->
+ spawn_link(fun() ->
do_work(Self, WorkerDb, 0)
end),
spawn_workers(WorkerCount - 1).
@@ -48,7 +48,10 @@ report(Start, Count) ->
case timer:now_diff(Now, Start) of
N when N > 1000000 ->
{_, MQL} = process_info(whereis(ddoc_cache_lru), message_queue_len),
- io:format("~p ~p~n", [Count, MQL]),
+ ProcCount = erlang:system_info(process_count),
+ CacheSize = ets:info(?CACHE, size),
+ LRUSize = ets:info(?LRU, size),
+ io:format("~p ~p ~p ~p ~p~n", [Count, MQL, ProcCount, CacheSize, LRUSize]),
report(Now, 0);
_ ->
receive
diff --git a/src/ddoc_cache/src/ddoc_cache_tables.erl b/src/ddoc_cache/src/ddoc_cache_tables.erl
index 89aec6f..5856776 100644
--- a/src/ddoc_cache/src/ddoc_cache_tables.erl
+++ b/src/ddoc_cache/src/ddoc_cache_tables.erl
@@ -44,8 +44,7 @@ init(_) ->
{keypos, #entry.key}
] ++ BaseOpts,
ets:new(?CACHE, CacheOpts),
- ets:new(?LRU, [ordered_set] ++ BaseOpts),
- ets:new(?OPENERS, [set, {keypos, #opener.key}] ++ BaseOpts),
+ ets:new(?LRU, [ordered_set, {write_concurrency, true}] ++ BaseOpts),
{ok, nil}.
diff --git a/src/ddoc_cache/test/ddoc_cache_basic_test.erl b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
index c3b7760..227ac54 100644
--- a/src/ddoc_cache/test/ddoc_cache_basic_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_basic_test.erl
@@ -105,8 +105,9 @@ cache_ddoc_refresher_unchanged({DbName, _}) ->
dont_cache_not_found({DbName, _}) ->
+ DDocId = <<"_design/not_found">>,
ddoc_cache_tutil:clear(),
- Resp = ddoc_cache:open_doc(DbName, <<"_design/not_found">>),
+ Resp = ddoc_cache:open_doc(DbName, DDocId),
?assertEqual({not_found, missing}, Resp),
?assertEqual(0, ets:info(?CACHE, size)),
?assertEqual(0, ets:info(?LRU, size)).
diff --git a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
index 17e0770..91182ca 100644
--- a/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_coverage_test.erl
@@ -24,27 +24,16 @@ coverage_test_() ->
fun ddoc_cache_tutil:start_couch/0,
fun ddoc_cache_tutil:stop_couch/1,
[
- fun restart_opener/0,
fun restart_lru/0,
fun restart_tables/0,
- fun restart_evictor/0,
- fun lru_ignores_unknown_keys/0
+ fun restart_evictor/0
]
}.
-restart_opener() ->
- send_bad_messages(ddoc_cache_opener),
- wait_for_restart(ddoc_cache_opener, fun() ->
- whereis(ddoc_cache_opener) ! {'EXIT', a_pid, because}
- end),
- ?assertEqual(ok, ddoc_cache_opener:terminate(bang, bable)),
- ?assertEqual({ok, foo}, ddoc_cache_opener:code_change(1, foo, [])).
-
-
restart_lru() ->
send_bad_messages(ddoc_cache_lru),
- ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c, d})),
+ ?assertEqual(ok, ddoc_cache_lru:terminate(bang, {st, a, b, c})),
?assertEqual({ok, foo}, ddoc_cache_lru:code_change(1, foo, [])).
@@ -58,7 +47,7 @@ restart_evictor() ->
meck:new(ddoc_cache_ev, [passthrough]),
try
State = sys:get_state(ddoc_cache_lru),
- Evictor = element(5, State),
+ Evictor = element(4, State),
Ref = erlang:monitor(process, Evictor),
exit(Evictor, shutdown),
receive
@@ -68,22 +57,13 @@ restart_evictor() ->
end,
meck:wait(ddoc_cache_ev, event, [evictor_died, '_'], 1000),
NewState = sys:get_state(ddoc_cache_lru),
- NewEvictor = element(5, NewState),
+ NewEvictor = element(4, NewState),
?assertNotEqual(Evictor, NewEvictor)
after
meck:unload()
end.
-lru_ignores_unknown_keys() ->
- ?assertEqual(evicted, gen_server:call(ddoc_cache_lru, {update, foo, bar})),
- ?assertEqual(ok, gen_server:call(ddoc_cache_lru, {remove, foo})),
- Pid = whereis(ddoc_cache_lru),
- gen_server:cast(ddoc_cache_lru, {accessed, foo}),
- timer:sleep(200),
- ?assert(is_process_alive(Pid)).
-
-
send_bad_messages(Name) ->
wait_for_restart(Name, fun() ->
?assertEqual({invalid_call, foo}, gen_server:call(Name, foo))
diff --git a/src/ddoc_cache/test/ddoc_cache_opener_test.erl b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
index 5ea4b0f..21986c9 100644
--- a/src/ddoc_cache/test/ddoc_cache_opener_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_opener_test.erl
@@ -17,60 +17,60 @@
-include_lib("eunit/include/eunit.hrl").
-include("ddoc_cache_test.hrl").
-
-opener_test_() ->
- {
- setup,
- fun ddoc_cache_tutil:start_couch/0,
- fun ddoc_cache_tutil:stop_couch/1,
- {with, [
- fun check_multiple/1,
- fun handles_opened/1,
- fun handles_error/1
- ]}
- }.
-
-
-check_multiple({DbName, _}) ->
- ddoc_cache_tutil:clear(),
- % We're faking multiple concurrent readers by pausing the
- % ddoc_cache_opener process, sending it a few messages
- % and then resuming the process.
- Pid = whereis(ddoc_cache_opener),
- Key = {ddoc_cache_entry_ddocid, {DbName, ?FOOBAR}},
- erlang:suspend_process(Pid),
- lists:foreach(fun(_) ->
- Pid ! {'$gen_call', {self(), make_ref()}, {open, Key}}
- end, lists:seq(1, 10)),
- erlang:resume_process(Pid),
- lists:foreach(fun(_) ->
- receive
- {_, {open_ok, _, _}} -> ok
- end
- end, lists:seq(1, 10)).
-
-
-handles_opened({DbName, _}) ->
- ddoc_cache_tutil:clear(),
- {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
- [#entry{key = Key, val = Val}] = ets:tab2list(?CACHE),
- Resp = gen_server:call(ddoc_cache_opener, {open, Key}),
- ?assertEqual({ok, Val}, Resp).
-
-
-handles_error({DbName, _}) ->
- ddoc_cache_tutil:clear(),
- meck:new(ddoc_cache_entry, [passthrough]),
- meck:expect(ddoc_cache_entry, do_open, fun(_, _) ->
- couch_log:error("OHAI", []),
- erlang:error(borkity)
- end),
- try
- ?assertError(
- {ddoc_cache_error, _},
- ddoc_cache:open_doc(DbName, ?FOOBAR)
- )
- after
- meck:unload()
- end.
-
+%%
+%% opener_test_() ->
+%% {
+%% setup,
+%% fun ddoc_cache_tutil:start_couch/0,
+%% fun ddoc_cache_tutil:stop_couch/1,
+%% {with, [
+%% fun check_multiple/1,
+%% fun handles_opened/1,
+%% fun handles_error/1
+%% ]}
+%% }.
+%%
+%%
+%% check_multiple({DbName, _}) ->
+%% ddoc_cache_tutil:clear(),
+%% % We're faking multiple concurrent readers by pausing the
+%% % ddoc_cache_opener process, sending it a few messages
+%% % and then resuming the process.
+%% Pid = whereis(ddoc_cache_opener),
+%% Key = {ddoc_cache_entry_ddocid, {DbName, ?FOOBAR}},
+%% erlang:suspend_process(Pid),
+%% lists:foreach(fun(_) ->
+%% Pid ! {'$gen_call', {self(), make_ref()}, {open, Key}}
+%% end, lists:seq(1, 10)),
+%% erlang:resume_process(Pid),
+%% lists:foreach(fun(_) ->
+%% receive
+%% {_, {open_ok, _, _}} -> ok
+%% end
+%% end, lists:seq(1, 10)).
+%%
+%%
+%% handles_opened({DbName, _}) ->
+%% ddoc_cache_tutil:clear(),
+%% {ok, _} = ddoc_cache:open_doc(DbName, ?FOOBAR),
+%% [#entry{key = Key, val = Val}] = ets:tab2list(?CACHE),
+%% Resp = gen_server:call(ddoc_cache_opener, {open, Key}),
+%% ?assertEqual({ok, Val}, Resp).
+%%
+%%
+%% handles_error({DbName, _}) ->
+%% ddoc_cache_tutil:clear(),
+%% meck:new(ddoc_cache_entry, [passthrough]),
+%% meck:expect(ddoc_cache_entry, do_open, fun(_, _) ->
+%% couch_log:error("OHAI", []),
+%% erlang:error(borkity)
+%% end),
+%% try
+%% ?assertError(
+%% {ddoc_cache_error, _},
+%% ddoc_cache:open_doc(DbName, ?FOOBAR)
+%% )
+%% after
+%% meck:unload()
+%% end.
+%%
diff --git a/src/ddoc_cache/test/ddoc_cache_refresher_test.erl b/src/ddoc_cache/test/ddoc_cache_refresher_test.erl
deleted file mode 100644
index eb806e3..0000000
--- a/src/ddoc_cache/test/ddoc_cache_refresher_test.erl
+++ /dev/null
@@ -1,167 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(ddoc_cache_refresher_test).
-
-
--include_lib("eunit/include/eunit.hrl").
--include("ddoc_cache_test.hrl").
-
-
-key() ->
- {ddoc_cache_entry_custom, {<<"dbname_here">>, ?MODULE}}.
-
-
-setup() ->
- Ctx = test_util:start_couch(),
- ets:new(?CACHE, [public, named_table, set, {keypos, #entry.key}]),
- ets:insert(?CACHE, #entry{key = key(), val = bang}),
- meck:new(ddoc_cache_lru, [passthrough]),
- meck:new(ddoc_cache_entry, [passthrough]),
- Ctx.
-
-
-teardown(Ctx) ->
- meck:unload(),
- ets:delete(?CACHE),
- test_util:stop_couch(Ctx).
-
-
-refresher_test_() ->
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun handles_error/0,
- fun refresh_on_timeout/0,
- fun refresh_once/0,
- fun dies_on_missing_entry/0,
- fun dies_on_evicted/0
- ]
- }.
-
-
-handles_error() ->
- meck:expect(ddoc_cache_entry, spawn_refresher, fun(_) ->
- throw(foo)
- end),
- {Pid, Ref} = spawn_refresher(),
- ddoc_cache_refresher:refresh(Pid),
- receive
- {'DOWN', Ref, _, _, {throw, foo, _}} ->
- ok
- end.
-
-
-refresh_on_timeout() ->
- meck:expect(ddoc_cache_entry, spawn_refresher, fun(Key) ->
- Ref = erlang:make_ref(),
- self() ! {'DOWN', Ref, process, pid, {open_ok, Key, {ok, zot}}},
- {self(), Ref}
- end),
- meck:expect(ddoc_cache_lru, update, fun(_, zot) ->
- ok
- end),
- {Pid, _} = spawn_refresher(),
- ets:update_element(?CACHE, key(), {#entry.pid, Pid}),
- % This is the assertion that if we wait long enough
- % the update will be called.
- meck:wait(ddoc_cache_lru, update, ['_', '_'], 1000),
- ?assert(is_process_alive(Pid)),
- ddoc_cache_refresher:stop(Pid).
-
-
-refresh_once() ->
- Counter = spawn_counter(),
- meck:expect(ddoc_cache_entry, spawn_refresher, fun(Key) ->
- Ref = erlang:make_ref(),
- Count = get_count(Counter),
- self() ! {'DOWN', Ref, process, pid, {open_ok, Key, {ok, Count}}},
- {self(), Ref}
- end),
- meck:expect(ddoc_cache_lru, update, fun(_, 1) ->
- ok
- end),
- {Pid, _} = spawn_refresher(),
- ets:update_element(?CACHE, key(), {#entry.pid, Pid}),
- erlang:suspend_process(Pid),
- lists:foreach(fun(_) ->
- ddoc_cache_refresher:refresh(Pid)
- end, lists:seq(1, 100)),
- erlang:resume_process(Pid),
- % This is the assertion that if we wait long enough
- % the update will be called.
- meck:wait(ddoc_cache_lru, update, ['_', '_'], 1000),
- ?assert(is_process_alive(Pid)),
- ddoc_cache_refresher:stop(Pid).
-
-
-dies_on_missing_entry() ->
- meck:expect(ddoc_cache_entry, spawn_refresher, fun(Key) ->
- Ref = erlang:make_ref(),
- self() ! {'DOWN', Ref, process, pid, {open_ok, Key, {ok, zot}}},
- {self(), Ref}
- end),
- {Pid, _} = spawn_refresher(),
- ets:delete(?CACHE, key()),
- ddoc_cache_refresher:refresh(Pid),
- receive
- {'DOWN', _, _, Pid, normal} ->
- ok
- end.
-
-
-dies_on_evicted() ->
- meck:expect(ddoc_cache_entry, spawn_refresher, fun(Key) ->
- Ref = erlang:make_ref(),
- self() ! {'DOWN', Ref, process, pid, {open_ok, Key, {ok, zot}}},
- {self(), Ref}
- end),
- meck:expect(ddoc_cache_lru, update, fun(_, zot) ->
- evicted
- end),
- {Pid, _} = spawn_refresher(),
- ets:update_element(?CACHE, key(), {#entry.pid, Pid}),
- ddoc_cache_refresher:refresh(Pid),
- receive
- {'DOWN', _, _, Pid, normal} ->
- ok
- end.
-
-
-spawn_refresher() ->
- erlang:spawn_monitor(ddoc_cache_refresher, init, [{self(), key(), 100}]).
-
-
-spawn_counter() ->
- erlang:spawn_link(fun do_counting/0).
-
-
-get_count(Pid) ->
- Pid ! {get, self()},
- receive
- {count, Pid, N} ->
- N
- end.
-
-
-do_counting() ->
- do_counting(0).
-
-
-do_counting(N) ->
- receive
- {get, From} ->
- From ! {count, self(), N + 1},
- do_counting(N + 1)
- end.
\ No newline at end of file
diff --git a/src/ddoc_cache/test/ddoc_cache_remove_test.erl b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
index 91f8ac2..7596b99 100644
--- a/src/ddoc_cache/test/ddoc_cache_remove_test.erl
+++ b/src/ddoc_cache/test/ddoc_cache_remove_test.erl
@@ -95,7 +95,7 @@ remove_ddoc_rev({DbName, _}) ->
do_compact(Shard#shard.name)
end, mem3:local_shards(DbName)),
% Trigger a refresh rather than wait for the timeout
- ddoc_cache_refresher:refresh(Pid),
+ ddoc_cache_entry:refresh(Pid),
meck:wait(ddoc_cache_ev, event, [removed, Key], 1000),
?assertMatch(
{{not_found, missing}, _},
@@ -127,8 +127,8 @@ remove_ddoc_rev_only({DbName, _}) ->
do_compact(Shard#shard.name)
end, mem3:local_shards(DbName)),
% Trigger a refresh rather than wait for the timeout
- ddoc_cache_refresher:refresh(NoRevPid),
- ddoc_cache_refresher:refresh(RevPid),
+ ddoc_cache_entry:refresh(NoRevPid),
+ ddoc_cache_entry:refresh(RevPid),
meck:wait(ddoc_cache_ev, event, [update_noop, NoRevKey], 1000),
meck:wait(ddoc_cache_ev, event, [removed, RevKey], 1000),
?assertMatch({ok, _}, ddoc_cache:open_doc(DbName, ?VDU)),
diff --git a/src/ddoc_cache/test/ddoc_cache_tutil.erl b/src/ddoc_cache/test/ddoc_cache_tutil.erl
index 6782b9d..cdd372b 100644
--- a/src/ddoc_cache/test/ddoc_cache_tutil.erl
+++ b/src/ddoc_cache/test/ddoc_cache_tutil.erl
@@ -21,7 +21,7 @@
start_couch() ->
- purge_modules(),
+ %purge_modules(),
Ctx = test_util:start_couch(?CONFIG_CHAIN, [chttpd, ddoc_cache]),
TmpDb = ?tempdb(),
ok = fabric:create_db(TmpDb, [{q, "1"}, {n, "1"}]),
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.
[couchdb] 01/03: Tweaks for performance
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch optimize-ddoc-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2443c2f4b30682fef29a393b1aa50106febf0d7b
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Jun 28 10:51:29 2017 -0500
Tweaks for performance
---
src/ddoc_cache/src/ddoc_cache_lru.erl | 26 ++++++++++++++------------
1 file changed, 14 insertions(+), 12 deletions(-)
diff --git a/src/ddoc_cache/src/ddoc_cache_lru.erl b/src/ddoc_cache/src/ddoc_cache_lru.erl
index b986b07..5b10959 100644
--- a/src/ddoc_cache/src/ddoc_cache_lru.erl
+++ b/src/ddoc_cache/src/ddoc_cache_lru.erl
@@ -55,19 +55,19 @@ start_link() ->
insert(Key, Val) ->
- gen_server:call(?MODULE, {insert, Key, Val}).
+ gen_server:call(?MODULE, {insert, Key, Val}, infinity).
accessed(Key) ->
- gen_server:cast(?MODULE, {accessed, Key}).
+ gen_server:call(?MODULE, {accessed, Key}).
update(Key, Val) ->
- gen_server:call(?MODULE, {update, Key, Val}).
+ gen_server:call(?MODULE, {update, Key, Val}, infinity).
remove(Key) ->
- gen_server:call(?MODULE, {remove, Key}).
+ gen_server:call(?MODULE, {remove, Key}, infinity).
refresh(DbName, DDocIds) ->
@@ -107,13 +107,17 @@ handle_call({insert, Key, Val}, _From, St) ->
NewSt = St#st{time = NewTime},
Pid = ddoc_cache_refresher:spawn_link(Key, ?REFRESH_TIMEOUT),
true = ets:insert(?CACHE, #entry{key = Key, val = Val, pid = Pid}),
- true = ets:insert(?LRU, {NewTime, Key}),
+ true = ets:insert(?LRU, {{NewTime, Key}}),
ok = khash:put(ATimes, Key, NewTime),
store_key(Dbs, Key),
trim(NewSt),
?EVENT(inserted, {Key, Val}),
{reply, ok, NewSt};
+handle_call({accessed, Key}, _From, St) ->
+ {noreply, NewSt} = handle_cast({accessed, Key}, St),
+ {reply, ok, NewSt};
+
handle_call({update, Key, Val}, _From, St) ->
#st{
atimes = ATimes
@@ -170,10 +174,8 @@ handle_cast({accessed, Key}, St) ->
NewTime = Time + 1,
case khash:lookup(ATimes, Key) of
{value, OldTime} ->
- [#entry{pid = Pid}] = ets:lookup(?CACHE, Key),
- true = is_process_alive(Pid),
- true = ets:delete(?LRU, OldTime),
- true = ets:insert(?LRU, {NewTime, Key}),
+ true = ets:delete(?LRU, {OldTime, Key}),
+ true = ets:insert(?LRU, {{NewTime, Key}}),
ok = khash:put(ATimes, Key, NewTime),
?EVENT(accessed, Key);
not_found ->
@@ -297,7 +299,7 @@ remove_key(St, Key, ATime) ->
atimes = ATimes
} = St,
true = ets:delete(?CACHE, Key),
- true = ets:delete(?LRU, ATime),
+ true = ets:delete(?LRU, {ATime, Key}),
ok = khash:del(ATimes, Key).
@@ -305,10 +307,10 @@ trim(St) ->
#st{
atimes = ATimes
} = St,
- MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 1000)),
+ MaxSize = max(0, config:get_integer("ddoc_cache", "max_size", 5000)),
case khash:size(ATimes) > MaxSize of
true ->
- [{ATime, Key}] = ets:lookup(?LRU, ets:first(?LRU)),
+ {ATime, Key} = ets:first(?LRU),
remove_key(St, Key, ATime),
trim(St);
false ->
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.
[couchdb] 02/03: TMP: Simple benchmark script
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch optimize-ddoc-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3c2b4804abf9ef223aa7387c0f706c3867d123b5
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Jun 28 10:58:34 2017 -0500
TMP: Simple benchmark script
---
src/ddoc_cache/src/ddoc_cache_speed.erl | 61 +++++++++++++++++++++++++++++++++
1 file changed, 61 insertions(+)
diff --git a/src/ddoc_cache/src/ddoc_cache_speed.erl b/src/ddoc_cache/src/ddoc_cache_speed.erl
new file mode 100644
index 0000000..38cd0b9
--- /dev/null
+++ b/src/ddoc_cache/src/ddoc_cache_speed.erl
@@ -0,0 +1,61 @@
+-module(ddoc_cache_speed).
+
+-export([
+ go/1,
+ recover/1
+]).
+
+
+-define(RANGE, 1000).
+
+
+go(WorkerCount) when is_integer(WorkerCount), WorkerCount > 0 ->
+ spawn_workers(WorkerCount),
+ report().
+
+
+recover(DbName) ->
+ {ok, {stuff, DbName}}.
+
+
+spawn_workers(0) ->
+ ok;
+
+spawn_workers(WorkerCount) ->
+ Self = self(),
+ WorkerDb = list_to_binary(integer_to_list(WorkerCount)),
+ spawn(fun() ->
+ do_work(Self, WorkerDb, 0)
+ end),
+ spawn_workers(WorkerCount - 1).
+
+
+do_work(Parent, WorkerDb, Count) when Count >= 25 ->
+ Parent ! {done, Count},
+ do_work(Parent, WorkerDb, 0);
+
+do_work(Parent, WorkerDb, Count) ->
+ {ok, _} = ddoc_cache:open_custom(WorkerDb, ?MODULE),
+ do_work(Parent, WorkerDb, Count + 1).
+
+
+report() ->
+ report(os:timestamp(), 0).
+
+
+report(Start, Count) ->
+ Now = os:timestamp(),
+ case timer:now_diff(Now, Start) of
+ N when N > 1000000 ->
+ {_, MQL} = process_info(whereis(ddoc_cache_lru), message_queue_len),
+ io:format("~p ~p~n", [Count, MQL]),
+ report(Now, 0);
+ _ ->
+ receive
+ {done, Done} ->
+ report(Start, Count + Done)
+ after 100 ->
+ report(Start, Count)
+ end
+ end.
+
--
To stop receiving notification emails like this one, please contact
"commits@couchdb.apache.org" <co...@couchdb.apache.org>.