You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2014/08/28 14:17:48 UTC
[06/10] ddoc-cache commit: updated refs/heads/master to 4ffc6b0
Add support for keying the cache by revision
This patch splits up the code paths between ddoc_cache and
ddoc_cache_opener a bit more clearly, keeping the gen_server
implementation details "mostly" in one place. The overloaded
open function has been replaced with more explicit function
names. I've left the old open to support old code during the
upgrade process but it should be removed after the next release.
The opener lookup keys are a bit mismatched from the LRU keys so
I've renamed those variables to make it clear when we're talking
about the openers rather than the cache itself. It could be
argued that the revision oblivious doc lookup could reuse the
result of any pending revision specific lookup on the same id
but it seems a bit too optimistic here so I've left them as
separate opener keys.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/commit/41589453
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/tree/41589453
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/diff/41589453
Branch: refs/heads/master
Commit: 4158945359493733d0e7e94ca388146f145fcd53
Parents: 8ab9683
Author: Brian Mitchell <br...@cloudant.com>
Authored: Tue Oct 15 17:05:55 2013 -0400
Committer: Robert Newson <rn...@apache.org>
Committed: Tue Jul 29 11:56:44 2014 +0100
----------------------------------------------------------------------
src/ddoc_cache.erl | 86 ++++++++----------
src/ddoc_cache_opener.erl | 200 +++++++++++++++++++++++++++++------------
2 files changed, 181 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/blob/41589453/src/ddoc_cache.erl
----------------------------------------------------------------------
diff --git a/src/ddoc_cache.erl b/src/ddoc_cache.erl
index e862667..6c54c36 100644
--- a/src/ddoc_cache.erl
+++ b/src/ddoc_cache.erl
@@ -12,70 +12,60 @@
-module(ddoc_cache).
+-include_lib("couch/include/couch_db.hrl").
--export([
- start/0,
- stop/0,
-
- open/2,
- evict/2
-]).
+-export([start/0, stop/0]).
+% public API
+-export([open_doc/2, open_doc/3, open_validation_funs/1, evict/2]).
--define(CACHE, ddoc_cache_lru).
--define(OPENER, ddoc_cache_opener).
-
+% deprecated API
+-export([open/2]).
start() ->
application:start(ddoc_cache).
-
stop() ->
application:stop(ddoc_cache).
+open_doc(DbName, DocId) ->
+ Key = {DbName, DocId, '_'},
+ case ddoc_cache_opener:match_newest(Key) of
+ {ok, _} = Resp ->
+ Resp;
+ missing ->
+ ddoc_cache_opener:open_doc(DbName, DocId);
+ recover ->
+ ddoc_cache_opener:recover_doc(DbName, DocId)
+ end.
-open(DbName, validation_funs) ->
- open({DbName, validation_funs});
-open(DbName, <<"_design/", _/binary>>=DDocId) when is_binary(DbName) ->
- open({DbName, DDocId});
-open(DbName, DDocId) when is_binary(DDocId) ->
- open({DbName, <<"_design/", DDocId/binary>>}).
-
-
-open(Key) ->
- try ets_lru:lookup_d(?CACHE, Key) of
+open_doc(DbName, DocId, RevId) ->
+ Key = {DbName, DocId, RevId},
+ case ddoc_cache_opener:lookup(Key) of
{ok, _} = Resp ->
Resp;
- _ ->
- case gen_server:call(?OPENER, {open, Key}, infinity) of
- {open_ok, Resp} ->
- Resp;
- {open_error, throw, Error} ->
- throw(Error);
- {open_error, error, Error} ->
- erlang:error(Error);
- {open_error, exit, Error} ->
- exit(Error)
- end
- catch
- error:badarg ->
- recover(Key)
+ missing ->
+ ddoc_cache_opener:open_doc(DbName, DocId, RevId);
+ recover ->
+ ddoc_cache_opener:recover_doc(DbName, DocId, RevId)
end.
+open_validation_funs(DbName) ->
+ Key = {DbName, validation_funs},
+ case ddoc_cache_opener:lookup(Key) of
+ {ok, _} = Resp ->
+ Resp;
+ missing ->
+ ddoc_cache_opener:open_validation_funs(DbName);
+ recover ->
+ ddoc_cache_opener:recover_validation_funs(DbName)
+ end.
evict(ShardDbName, DDocIds) ->
DbName = mem3:dbname(ShardDbName),
- gen_server:cast(?OPENER, {evict, DbName, DDocIds}).
-
+ ddoc_cache_opener:evict_docs(DbName, DDocIds).
-recover({DbName, validation_funs}) ->
- {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
- Funs = lists:flatmap(fun(DDoc) ->
- case couch_doc:get_validate_doc_fun(DDoc) of
- nil -> [];
- Fun -> [Fun]
- end
- end, DDocs),
- {ok, Funs};
-recover({DbName, DDocId}) ->
- fabric:open_doc(DbName, DDocId, [ejson_body]).
+open(DbName, validation_funs) ->
+ open_validation_funs(DbName);
+open(DbName, DocId) ->
+ open_doc(DbName, DocId).
http://git-wip-us.apache.org/repos/asf/couchdb-ddoc-cache/blob/41589453/src/ddoc_cache_opener.erl
----------------------------------------------------------------------
diff --git a/src/ddoc_cache_opener.erl b/src/ddoc_cache_opener.erl
index b34359f..780f9dd 100644
--- a/src/ddoc_cache_opener.erl
+++ b/src/ddoc_cache_opener.erl
@@ -13,38 +13,49 @@
-module(ddoc_cache_opener).
-behaviour(gen_server).
+-include_lib("couch/include/couch_db.hrl").
-include_lib("mem3/include/mem3.hrl").
--export([
- start_link/0
-]).
+% worker API
+-export([start_link/0]).
+% public API
-export([
- open_ddoc/1
+ open_doc/2,
+ open_doc/3,
+ open_validation_funs/1,
+ evict_docs/2,
+ lookup/1,
+ match_newest/1,
+ recover_doc/2,
+ recover_doc/3,
+ recover_validation_funs/1
]).
+% couch_event listener callback
+-export([handle_db_event/3]).
+
+% gen_server behavior
-export([
init/1,
terminate/2,
-
handle_call/3,
handle_cast/2,
handle_info/2,
-
code_change/3
]).
+% sub-process spawn API
-export([
- handle_db_event/3
+ fetch_doc_data/1
]).
-
-define(CACHE, ddoc_cache_lru).
-define(OPENING, ddoc_cache_opening).
-type dbname() :: iodata().
-type docid() :: iodata().
--type revision() :: {integer(), binary()}.
+-type revision() :: {pos_integer(), <<_:128>>}.
-record(opener, {
key,
@@ -57,10 +68,78 @@
evictor
}).
-
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+-spec open_doc(dbname(), docid()) -> {ok, #doc{}}.
+open_doc(DbName, DocId) ->
+ Resp = gen_server:call(?MODULE, {open, {DbName, DocId}}, infinity),
+ handle_open_response(Resp).
+
+-spec open_doc(dbname(), docid(), revision()) -> {ok, #doc{}}.
+open_doc(DbName, DocId, RevId) ->
+ Resp = gen_server:call(?MODULE, {open, {DbName, DocId, RevId}}, infinity),
+ handle_open_response(Resp).
+
+-spec open_validation_funs(dbname()) -> {ok, [fun()]}.
+open_validation_funs(DbName) ->
+ Resp = gen_server:call(?MODULE, {open, {DbName, validation_funs}}, infinity),
+ handle_open_response(Resp).
+
+-spec evict_docs(dbname(), [docid()]) -> ok.
+evict_docs(DbName, DocIds) ->
+ gen_server:cast(?MODULE, {evict, DbName, DocIds}).
+
+lookup(Key) ->
+ try ets_lru:lookup_d(?CACHE, Key) of
+ {ok, _} = Resp ->
+ Resp;
+ _ ->
+ missing
+ catch
+ error:badarg ->
+ recover
+ end.
+
+match_newest(Key) ->
+ try ets_lru:match_object(?CACHE, Key, '_') of
+ [] ->
+ missing;
+ Docs ->
+ Sorted = lists:sort(fun (#doc{revs=L}, #doc{revs=R}) ->
+ L >= R
+ end, Docs),
+ {ok, hd(Sorted)}
+ catch
+ error:badarg ->
+ recover
+ end.
+
+recover_doc(DbName, DDocId) ->
+ fabric:open_doc(DbName, DDocId, []).
+
+recover_doc(DbName, DDocId, RevId) ->
+ {ok, [Resp]} = fabric:open_revs(DbName, DDocId, [RevId], []),
+ Resp.
+
+recover_validation_funs(DbName) ->
+ {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
+ Funs = lists:flatmap(fun(DDoc) ->
+ case couch_doc:get_validate_doc_fun(DDoc) of
+ nil -> [];
+ Fun -> [Fun]
+ end
+ end, DDocs),
+ {ok, Funs}.
+
+handle_db_event(ShardDbName, created, St) ->
+ gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+ {ok, St};
+handle_db_event(ShardDbName, deleted, St) ->
+ gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
+ {ok, St};
+handle_db_event(_DbName, _Event, St) ->
+ {ok, St}.
init(_) ->
process_flag(trap_exit, true),
@@ -72,7 +151,6 @@ init(_) ->
evictor = Evictor
}}.
-
terminate(_Reason, St) ->
case is_pid(St#st.evictor) of
true -> exit(St#st.evictor, kill);
@@ -80,15 +158,14 @@ terminate(_Reason, St) ->
end,
ok.
-
-handle_call({open, {_DbName, _DDocId}=Key}, From, St) ->
- case ets:lookup(?OPENING, Key) of
+handle_call({open, OpenerKey}, From, St) ->
+ case ets:lookup(?OPENING, OpenerKey) of
[#opener{clients=Clients}=O] ->
ets:insert(?OPENING, O#opener{clients=[From | Clients]}),
{noreply, St};
[] ->
- Pid = spawn_link(?MODULE, open_ddoc, [Key]),
- ets:insert(?OPENING, #opener{key=Key, pid=Pid, clients=[From]}),
+ Pid = spawn_link(?MODULE, fetch_doc_data, [OpenerKey]),
+ ets:insert(?OPENING, #opener{key=OpenerKey, pid=Pid, clients=[From]}),
{noreply, St}
end;
@@ -105,39 +182,41 @@ handle_cast({evict, DbName, DDocIds}, St) ->
{noreply, St};
handle_cast({do_evict, DbName}, St) ->
- DDocIds = ets_lru:match(?CACHE, {DbName, '$1'}, '_'),
+ DDocIds = lists:flatten(ets_lru:match(?CACHE, {DbName, '$1', '_'}, '_')),
handle_cast({do_evict, DbName, DDocIds}, St);
handle_cast({do_evict, DbName, DDocIds}, St) ->
ets_lru:remove(?CACHE, {DbName, validation_funs}),
lists:foreach(fun(DDocId) ->
- ets_lru:remove(?CACHE, {DbName, DDocId})
+ RevIds = ets_lru:match(?CACHE, {DbName, DDocId, '$1'}, '_'),
+ lists:foreach(fun([RevId]) ->
+ ets_lru:remove(?CACHE, {DbName, DDocId, RevId})
+ end, RevIds)
end, DDocIds),
{noreply, St};
handle_cast(Msg, St) ->
{stop, {invalid_cast, Msg}, St}.
-
handle_info({'EXIT', Pid, Reason}, #st{evictor=Pid}=St) ->
couch_log:error("ddoc_cache_opener evictor died ~w", [Reason]),
{ok, Evictor} = couch_event:link_listener(?MODULE, handle_db_event, nil, [all_dbs]),
{noreply, St#st{evictor=Evictor}};
-handle_info({'EXIT', _Pid, {open_ok, Key, Resp}}, St) ->
- respond(Key, {open_ok, Resp}),
+handle_info({'EXIT', _Pid, {open_ok, OpenerKey, Resp}}, St) ->
+ respond(OpenerKey, {open_ok, Resp}),
{noreply, St};
-handle_info({'EXIT', _Pid, {open_error, Key, Type, Error}}, St) ->
- respond(Key, {open_error, Type, Error}),
+handle_info({'EXIT', _Pid, {open_error, OpenerKey, Type, Error}}, St) ->
+ respond(OpenerKey, {open_error, Type, Error}),
{noreply, St};
handle_info({'EXIT', Pid, Reason}, St) ->
Pattern = #opener{pid=Pid, _='_'},
case ets:match_object(?OPENING, Pattern) of
- [#opener{key=Key, clients=Clients}] ->
+ [#opener{key=OpenerKey, clients=Clients}] ->
_ = [gen_server:reply(C, {error, Reason}) || C <- Clients],
- ets:delete(?OPENING, Key),
+ ets:delete(?OPENING, OpenerKey),
{noreply, St};
[] ->
{stop, {unknown_pid_died, {Pid, Reason}}, St}
@@ -146,46 +225,53 @@ handle_info({'EXIT', Pid, Reason}, St) ->
handle_info(Msg, St) ->
{stop, {invalid_info, Msg}, St}.
-
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-
-handle_db_event(ShardDbName, created, St) ->
- gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
- {ok, St};
-handle_db_event(ShardDbName, deleted, St) ->
- gen_server:cast(?MODULE, {evict, mem3:dbname(ShardDbName)}),
- {ok, St};
-handle_db_event(_DbName, _Event, St) ->
- {ok, St}.
-
-
--spec open_ddoc({dbname(), validation_funs | docid()}) -> no_return().
-open_ddoc({DbName, validation_funs}=Key) ->
- {ok, DDocs} = fabric:design_docs(mem3:dbname(DbName)),
- Funs = lists:flatmap(fun(DDoc) ->
- case couch_doc:get_validate_doc_fun(DDoc) of
- nil -> [];
- Fun -> [Fun]
- end
- end, DDocs),
- ok = ets_lru:insert(?CACHE, {DbName, validation_funs}, Funs),
- exit({open_ok, Key, {ok, Funs}});
-open_ddoc({DbName, DDocId}=Key) ->
- try fabric:open_doc(DbName, DDocId, [ejson_body]) of
+-spec fetch_doc_data({dbname(), validation_funs}) -> no_return();
+ ({dbname(), docid()}) -> no_return();
+ ({dbname(), docid(), revision()}) -> no_return().
+fetch_doc_data({DbName, validation_funs}) ->
+ OpenerKey = {DbName, validation_funs},
+ {ok, Funs} = recover_validation_funs(DbName),
+ ok = ets_lru:insert(?CACHE, OpenerKey, Funs),
+ exit({open_ok, OpenerKey, {ok, Funs}});
+fetch_doc_data({DbName, DocId}) ->
+ OpenerKey = {DbName, DocId},
+ try recover_doc(DbName, DocId) of
{ok, Doc} ->
- ok = ets_lru:insert(?CACHE, {DbName, DDocId}, Doc),
- exit({open_ok, Key, {ok, Doc}});
+ {RevCount, [RevHash| _]} = Doc#doc.revs,
+ RevId = {RevCount, RevHash},
+ ok = ets_lru:insert(?CACHE, {DbName, DocId, RevId}, Doc),
+ exit({open_ok, OpenerKey, {ok, Doc}});
Else ->
- exit({open_ok, Key, Else})
+ exit({open_ok, OpenerKey, Else})
catch
Type:Reason ->
- exit({open_error, Key, Type, Reason})
+ exit({open_error, OpenerKey, Type, Reason})
+ end;
+fetch_doc_data({DbName, DocId, RevId}) ->
+ OpenerKey = {DbName, DocId, RevId},
+ try recover_doc(DbName, DocId, RevId) of
+ {ok, Doc} ->
+ ok = ets_lru:insert(?CACHE, {DbName, DocId, RevId}, Doc),
+ exit({open_ok, OpenerKey, {ok, Doc}});
+ Else ->
+ exit({open_ok, OpenerKey, Else})
+ catch
+ Type:Reason ->
+ exit({open_error, OpenerKey, Type, Reason})
end.
+handle_open_response(Resp) ->
+ case Resp of
+ {open_ok, Value} -> Value;
+ {open_error, throw, Error} -> throw(Error);
+ {open_error, error, Error} -> erlang:error(Error);
+ {open_error, exit, Error} -> exit(Error)
+ end.
-respond(Key, Resp) ->
- [#opener{clients=Clients}] = ets:lookup(?OPENING, Key),
+respond(OpenerKey, Resp) ->
+ [#opener{clients=Clients}] = ets:lookup(?OPENING, OpenerKey),
_ = [gen_server:reply(C, Resp) || C <- Clients],
- ets:delete(?OPENING, Key).
+ ets:delete(?OPENING, OpenerKey).