You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/03/23 13:38:22 UTC
[1/2] mem3 commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to aae55e8 [Forced Update!]
Repository: couchdb-mem3
Updated Branches:
refs/heads/COUCHDB-3287-pluggable-storage-engines e2fa78fc6 -> aae55e889 (forced update)
Fix stale shards cache
There's a race condition in mem3_shards that can result in having shards
in the cache for a database that's been deleted. This results in a
confused cluster that thinks a database exists until you attempt to open
it.
The fix is to ignore any cache insert requests that come from an older
version of the dbs db than mem3_shards cache knows about.
Big thanks to @jdoane for the identification and original patch.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/aae55e88
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/aae55e88
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/aae55e88
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: aae55e88961030fc767bbb462116de524d7cc390
Parents: 58aadeb
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Feb 24 12:55:37 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Mar 23 08:38:14 2017 -0500
----------------------------------------------------------------------
src/mem3_shards.erl | 54 ++++++++++++++++++++++++++++++++++++++----------
1 file changed, 43 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/aae55e88/src/mem3_shards.erl
----------------------------------------------------------------------
diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl
index accede9..22eb6de 100644
--- a/src/mem3_shards.erl
+++ b/src/mem3_shards.erl
@@ -27,7 +27,8 @@
-record(st, {
max_size = 25000,
cur_size = 0,
- changes_pid
+ changes_pid,
+ update_seq
}).
-include_lib("mem3/include/mem3.hrl").
@@ -181,11 +182,12 @@ init([]) ->
ets:new(?ATIMES, [ordered_set, protected, named_table]),
ok = config:listen_for_changes(?MODULE, nil),
SizeList = config:get("mem3", "shard_cache_size", "25000"),
- {Pid, _} = spawn_monitor(fun() -> listen_for_changes(get_update_seq()) end),
+ UpdateSeq = get_update_seq(),
{ok, #st{
max_size = list_to_integer(SizeList),
cur_size = 0,
- changes_pid = Pid
+ changes_pid = start_changes_listener(UpdateSeq),
+ update_seq = UpdateSeq
}}.
handle_call({set_max_size, Size}, _From, St) ->
@@ -200,12 +202,23 @@ handle_cast({cache_hit, DbName}, St) ->
couch_stats:increment_counter([mem3, shard_cache, hit]),
cache_hit(DbName),
{noreply, St};
-handle_cast({cache_insert, DbName, Shards}, St) ->
+handle_cast({cache_insert, DbName, Shards, UpdateSeq}, St) ->
couch_stats:increment_counter([mem3, shard_cache, miss]),
- {noreply, cache_free(cache_insert(St, DbName, Shards))};
+ NewSt = case UpdateSeq < St#st.update_seq of
+ true -> St;
+ false -> cache_free(cache_insert(St, DbName, Shards))
+ end,
+ {noreply, NewSt};
handle_cast({cache_remove, DbName}, St) ->
couch_stats:increment_counter([mem3, shard_cache, eviction]),
{noreply, cache_remove(St, DbName)};
+handle_cast({cache_insert_change, DbName, Shards, UpdateSeq}, St) ->
+ Msg = {cache_insert, DbName, Shards, UpdateSeq},
+ {noreply, NewSt} = handle_cast(Msg, St),
+ {noreply, NewSt#st{update_seq = UpdateSeq}};
+handle_cast({cache_remove_change, DbName, UpdateSeq}, St) ->
+ {noreply, NewSt} = handle_cast({cache_remove, DbName}, St),
+ {noreply, NewSt#st{update_seq = UpdateSeq}};
handle_cast(_Msg, St) ->
{noreply, St}.
@@ -222,8 +235,9 @@ handle_info({'DOWN', _, _, Pid, Reason}, #st{changes_pid=Pid}=St) ->
erlang:send_after(5000, self(), {start_listener, Seq}),
{noreply, NewSt#st{changes_pid=undefined}};
handle_info({start_listener, Seq}, St) ->
- {NewPid, _} = spawn_monitor(fun() -> listen_for_changes(Seq) end),
- {noreply, St#st{changes_pid=NewPid}};
+ {noreply, St#st{
+ changes_pid = start_changes_listener(Seq)
+ }};
handle_info(restart_config_listener, State) ->
ok = config:listen_for_changes(?MODULE, nil),
{noreply, State};
@@ -239,6 +253,21 @@ code_change(_OldVsn, #st{}=St, _Extra) ->
%% internal functions
+start_changes_listener(SinceSeq) ->
+ Self = self(),
+ {Pid, _} = erlang:spawn_monitor(fun() ->
+ erlang:spawn_link(fun() ->
+ Ref = erlang:monitor(process, Self),
+ receive
+ {'DOWN', Ref, _, _, _} ->
+ ok
+ end,
+ exit(shutdown)
+ end),
+ listen_for_changes(SinceSeq)
+ end),
+ Pid.
+
fold_fun(#full_doc_info{}=FDI, Acc) ->
DI = couch_doc:to_doc_info(FDI),
fold_fun(DI, Acc);
@@ -278,10 +307,11 @@ changes_callback({stop, EndSeq}, _) ->
exit({seq, EndSeq});
changes_callback({change, {Change}, _}, _) ->
DbName = couch_util:get_value(<<"id">>, Change),
+ Seq = couch_util:get_value(<<"seq">>, Change),
case DbName of <<"_design/", _/binary>> -> ok; _Else ->
case mem3_util:is_deleted(Change) of
true ->
- gen_server:cast(?MODULE, {cache_remove, DbName});
+ gen_server:cast(?MODULE, {cache_remove_change, DbName, Seq});
false ->
case couch_util:get_value(doc, Change) of
{error, Reason} ->
@@ -289,13 +319,14 @@ changes_callback({change, {Change}, _}, _) ->
[DbName, Reason]);
{Doc} ->
Shards = mem3_util:build_ordered_shards(DbName, Doc),
- gen_server:cast(?MODULE, {cache_insert, DbName, Shards}),
+ Msg = {cache_insert_change, DbName, Shards, Seq},
+ gen_server:cast(?MODULE, Msg),
[create_if_missing(mem3:name(S), mem3:engine(S)) || S
<- Shards, mem3:node(S) =:= node()]
end
end
end,
- {ok, couch_util:get_value(<<"seq">>, Change)};
+ {ok, Seq};
changes_callback(timeout, _) ->
ok.
@@ -311,8 +342,9 @@ load_shards_from_disk(DbName) when is_binary(DbName) ->
load_shards_from_db(ShardDb, DbName) ->
case couch_db:open_doc(ShardDb, DbName, [ejson_body]) of
{ok, #doc{body = {Props}}} ->
+ Seq = couch_db:get_update_seq(ShardDb),
Shards = mem3_util:build_ordered_shards(DbName, Props),
- gen_server:cast(?MODULE, {cache_insert, DbName, Shards}),
+ gen_server:cast(?MODULE, {cache_insert, DbName, Shards, Seq}),
Shards;
{not_found, _} ->
erlang:error(database_does_not_exist, ?b2l(DbName))
[2/2] mem3 commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to aae55e8
Posted by da...@apache.org.
Store and use the storage engine property
This adds an optional key to database documents that lists the
configured storage engine. This allows mem3_shards to create the shard
with the appropriate storage engine when recovering from a network
split.
COUCHDB-3287
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/58aadeb6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/58aadeb6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/58aadeb6
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 58aadeb65f0f122467f9f8d1aa3278545f7b90d4
Parents: 6d00bae
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Apr 6 11:18:01 2016 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Mar 23 08:38:14 2017 -0500
----------------------------------------------------------------------
include/mem3.hrl | 6 ++++--
src/mem3.erl | 20 +++++++++++++++---
src/mem3_shards.erl | 50 +++++++++++++++++---------------------------
src/mem3_util.erl | 17 ++++++++++++---
test/mem3_util_test.erl | 16 +++++++-------
5 files changed, 62 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/58aadeb6/include/mem3.hrl
----------------------------------------------------------------------
diff --git a/include/mem3.hrl b/include/mem3.hrl
index d6ac0be..6579210 100644
--- a/include/mem3.hrl
+++ b/include/mem3.hrl
@@ -16,7 +16,8 @@
node :: node() | '_',
dbname :: binary(),
range :: [non_neg_integer() | '$1' | '$2'] | '_',
- ref :: reference() | 'undefined' | '_'
+ ref :: reference() | 'undefined' | '_',
+ opts :: list()
}).
%% Do not reference outside of mem3.
@@ -26,7 +27,8 @@
dbname :: binary(),
range :: [non_neg_integer() | '$1' | '$2'] | '_',
ref :: reference() | 'undefined' | '_',
- order :: non_neg_integer() | 'undefined' | '_'
+ order :: non_neg_integer() | 'undefined' | '_',
+ opts :: list()
}).
%% types
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/58aadeb6/src/mem3.erl
----------------------------------------------------------------------
diff --git a/src/mem3.erl b/src/mem3.erl
index e9c1473..5e218f7 100644
--- a/src/mem3.erl
+++ b/src/mem3.erl
@@ -23,7 +23,7 @@
-export([get_placement/1]).
%% For mem3 use only.
--export([name/1, node/1, range/1]).
+-export([name/1, node/1, range/1, engine/1]).
-include_lib("mem3/include/mem3.hrl").
-include_lib("couch/include/couch_db.hrl").
@@ -99,7 +99,8 @@ shards_int(DbName, Options) ->
name = ShardDbName,
dbname = ShardDbName,
range = [0, (2 bsl 31)-1],
- order = undefined}];
+ order = undefined,
+ opts = []}];
ShardDbName ->
%% shard_db is treated as a single sharded db to support calls to db_info
%% and view_all_docs
@@ -107,7 +108,8 @@ shards_int(DbName, Options) ->
node = node(),
name = ShardDbName,
dbname = ShardDbName,
- range = [0, (2 bsl 31)-1]}];
+ range = [0, (2 bsl 31)-1],
+ opts = []}];
_ ->
mem3_shards:for_db(DbName, Options)
end.
@@ -307,3 +309,15 @@ name(#shard{name=Name}) ->
Name;
name(#ordered_shard{name=Name}) ->
Name.
+
+engine(#shard{opts=Opts}) ->
+ engine(Opts);
+engine(#ordered_shard{opts=Opts}) ->
+ engine(Opts);
+engine(Opts) when is_list(Opts) ->
+ case couch_util:get_value(engine, Opts) of
+ Engine when is_binary(Engine) ->
+ [{engine, Engine}];
+ _ ->
+ []
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/58aadeb6/src/mem3_shards.erl
----------------------------------------------------------------------
diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl
index a638db8..accede9 100644
--- a/src/mem3_shards.erl
+++ b/src/mem3_shards.erl
@@ -65,19 +65,14 @@ for_docid(DbName, DocId) ->
for_docid(DbName, DocId, Options) ->
HashKey = mem3_util:hash(DocId),
ShardHead = #shard{
- name = '_',
- node = '_',
dbname = DbName,
- range = ['$1','$2'],
- ref = '_'
+ range = ['$1', '$2'],
+ _ = '_'
},
OrderedShardHead = #ordered_shard{
- name = '_',
- node = '_',
dbname = DbName,
- range = ['$1','$2'],
- ref = '_',
- order = '_'
+ range = ['$1', '$2'],
+ _ = '_'
},
Conditions = [{'=<', '$1', HashKey}, {'=<', HashKey, '$2'}],
ShardSpec = {ShardHead, Conditions, ['$_']},
@@ -103,18 +98,13 @@ for_shard_name(ShardName, Options) ->
DbName = mem3:dbname(ShardName),
ShardHead = #shard{
name = ShardName,
- node = '_',
dbname = DbName,
- range = '_',
- ref = '_'
+ _ = '_'
},
OrderedShardHead = #ordered_shard{
name = ShardName,
- node = '_',
dbname = DbName,
- range = '_',
- ref = '_',
- order = '_'
+ _ = '_'
},
ShardSpec = {ShardHead, [], ['$_']},
OrderedShardSpec = {OrderedShardHead, [], ['$_']},
@@ -300,7 +290,7 @@ changes_callback({change, {Change}, _}, _) ->
{Doc} ->
Shards = mem3_util:build_ordered_shards(DbName, Doc),
gen_server:cast(?MODULE, {cache_insert, DbName, Shards}),
- [create_if_missing(mem3:name(S)) || S
+ [create_if_missing(mem3:name(S), mem3:engine(S)) || S
<- Shards, mem3:node(S) =:= node()]
end
end
@@ -337,20 +327,18 @@ in_range(Shard, HashKey) ->
[B, E] = mem3:range(Shard),
B =< HashKey andalso HashKey =< E.
-create_if_missing(Name) ->
- DbDir = config:get("couchdb", "database_dir"),
- Filename = filename:join(DbDir, ?b2l(Name) ++ ".couch"),
- case filelib:is_regular(Filename) of
- true ->
- ok;
- false ->
- case couch_server:create(Name, [?ADMIN_CTX]) of
- {ok, Db} ->
- couch_db:close(Db);
- Error ->
- couch_log:error("~p tried to create ~s, got ~p",
- [?MODULE, Name, Error])
- end
+create_if_missing(Name, Options) ->
+ case couch_server:exists(Name) of
+ true ->
+ ok;
+ false ->
+ case couch_server:create(Name, [?ADMIN_CTX] ++ Options) of
+ {ok, Db} ->
+ couch_db:close(Db);
+ Error ->
+ couch_log:error("~p tried to create ~s, got ~p",
+ [?MODULE, Name, Error])
+ end
end.
cache_insert(#st{cur_size=Cur}=St, DbName, Shards) ->
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/58aadeb6/src/mem3_util.erl
----------------------------------------------------------------------
diff --git a/src/mem3_util.erl b/src/mem3_util.erl
index 2cd444d..4e1b5fd 100644
--- a/src/mem3_util.erl
+++ b/src/mem3_util.erl
@@ -153,6 +153,10 @@ build_ordered_shards(DbName, DocProps) ->
build_shards_by_node(DbName, DocProps) ->
{ByNode} = couch_util:get_value(<<"by_node">>, DocProps, {[]}),
Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""),
+ EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of
+ Engine when is_binary(Engine) -> [{engine, Engine}];
+ _ -> []
+ end,
lists:flatmap(fun({Node, Ranges}) ->
lists:map(fun(Range) ->
[B,E] = string:tokens(?b2l(Range), "-"),
@@ -161,7 +165,8 @@ build_shards_by_node(DbName, DocProps) ->
name_shard(#shard{
dbname = DbName,
node = to_atom(Node),
- range = [Beg, End]
+ range = [Beg, End],
+ opts = EngineOpt
}, Suffix)
end, Ranges)
end, ByNode).
@@ -169,6 +174,10 @@ build_shards_by_node(DbName, DocProps) ->
build_shards_by_range(DbName, DocProps) ->
{ByRange} = couch_util:get_value(<<"by_range">>, DocProps, {[]}),
Suffix = couch_util:get_value(<<"shard_suffix">>, DocProps, ""),
+ EngineOpt = case couch_util:get_value(<<"engine">>, DocProps) of
+ Engine when is_binary(Engine) -> [{engine, Engine}];
+ _ -> []
+ end,
lists:flatmap(fun({Range, Nodes}) ->
lists:map(fun({Node, Order}) ->
[B,E] = string:tokens(?b2l(Range), "-"),
@@ -178,7 +187,8 @@ build_shards_by_range(DbName, DocProps) ->
dbname = DbName,
node = to_atom(Node),
range = [Beg, End],
- order = Order
+ order = Order,
+ opts = EngineOpt
}, Suffix)
end, lists:zip(Nodes, lists:seq(1, length(Nodes))))
end, ByRange).
@@ -247,7 +257,8 @@ downcast(#ordered_shard{}=S) ->
node = S#ordered_shard.node,
dbname = S#ordered_shard.dbname,
range = S#ordered_shard.range,
- ref = S#ordered_shard.ref
+ ref = S#ordered_shard.ref,
+ opts = S#ordered_shard.opts
};
downcast(Shards) when is_list(Shards) ->
[downcast(Shard) || Shard <- Shards].
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/58aadeb6/test/mem3_util_test.erl
----------------------------------------------------------------------
diff --git a/test/mem3_util_test.erl b/test/mem3_util_test.erl
index 340a58a..42bc5c7 100644
--- a/test/mem3_util_test.erl
+++ b/test/mem3_util_test.erl
@@ -85,35 +85,35 @@ build_shards_test() ->
[{shard,<<"shards/00000000-1fffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[0,536870911],
- undefined},
+ undefined,[]},
{shard,<<"shards/20000000-3fffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[536870912,1073741823],
- undefined},
+ undefined,[]},
{shard,<<"shards/40000000-5fffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[1073741824,1610612735],
- undefined},
+ undefined,[]},
{shard,<<"shards/60000000-7fffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[1610612736,2147483647],
- undefined},
+ undefined,[]},
{shard,<<"shards/80000000-9fffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[2147483648,2684354559],
- undefined},
+ undefined,[]},
{shard,<<"shards/a0000000-bfffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[2684354560,3221225471],
- undefined},
+ undefined,[]},
{shard,<<"shards/c0000000-dfffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[3221225472,3758096383],
- undefined},
+ undefined,[]},
{shard,<<"shards/e0000000-ffffffff/testdb1">>,
'bigcouch@node.local',<<"testdb1">>,
[3758096384,4294967295],
- undefined}],
+ undefined,[]}],
?assertEqual(ExpectedShards1, Shards1),
ok.