You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2016/05/09 21:29:21 UTC
[4/4] mem3 commit: updated refs/heads/master to 0b70afb
Use ets:select/2 to retrieve shards by name
The result of mem3_shards:for_db/1 on databases with high q values can
be very large, resulting in suboptimal performance for high-volume
callers.
mem3_sync_event_listener is only interested in a small subset of the
result of mem3_shards:for_db/1; moving this filter in to an ets:select/2
call improves performance significantly.
COUCHDB-2984
Project: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/commit/130efcd6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/tree/130efcd6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-mem3/diff/130efcd6
Branch: refs/heads/master
Commit: 130efcd6b0b6f9fa6403e131586dfbf003643074
Parents: d5e0a4a
Author: Benjamin Anderson <b...@banjiewen.net>
Authored: Sat Apr 9 23:08:39 2016 -0700
Committer: Benjamin Anderson <b...@banjiewen.net>
Committed: Mon May 9 14:20:56 2016 -0700
----------------------------------------------------------------------
src/mem3_shards.erl | 49 +++++++++++++++++++++++++++++++++++
src/mem3_sync_event_listener.erl | 15 +++++++----
2 files changed, 59 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/130efcd6/src/mem3_shards.erl
----------------------------------------------------------------------
diff --git a/src/mem3_shards.erl b/src/mem3_shards.erl
index 14aed34..224673f 100644
--- a/src/mem3_shards.erl
+++ b/src/mem3_shards.erl
@@ -21,6 +21,7 @@
-export([start_link/0]).
-export([for_db/1, for_db/2, for_docid/2, for_docid/3, get/3, local/1, fold/2]).
+-export([for_shard_name/1]).
-export([set_max_size/1]).
-record(st, {
@@ -94,6 +95,42 @@ for_docid(DbName, DocId, Options) ->
false -> mem3_util:downcast(Shards)
end.
+for_shard_name(ShardName) ->
+ for_shard_name(ShardName, []).
+
+for_shard_name(ShardName, Options) ->
+ DbName = mem3:dbname(ShardName),
+ ShardHead = #shard{
+ name = ShardName,
+ node = '_',
+ dbname = DbName,
+ range = '_',
+ ref = '_'
+ },
+ OrderedShardHead = #ordered_shard{
+ name = ShardName,
+ node = '_',
+ dbname = DbName,
+ range = '_',
+ ref = '_',
+ order = '_'
+ },
+ ShardSpec = {ShardHead, [], ['$_']},
+ OrderedShardSpec = {OrderedShardHead, [], ['$_']},
+ Shards = try ets:select(?SHARDS, [ShardSpec, OrderedShardSpec]) of
+ [] ->
+ filter_shards_by_name(ShardName, load_shards_from_disk(DbName));
+ Else ->
+ gen_server:cast(?MODULE, {cache_hit, DbName}),
+ Else
+ catch error:badarg ->
+ filter_shards_by_name(ShardName, load_shards_from_disk(DbName))
+ end,
+ case lists:member(ordered, Options) of
+ true -> Shards;
+ false -> mem3_util:downcast(Shards)
+ end.
+
get(DbName, Node, Range) ->
Res = lists:foldl(fun(#shard{node=N, range=R}=S, Acc) ->
case {N, R} of
@@ -360,3 +397,15 @@ cache_clear(St) ->
true = ets:delete_all_objects(?SHARDS),
true = ets:delete_all_objects(?ATIMES),
St#st{cur_size=0}.
+
+filter_shards_by_name(Name, Shards) ->
+ filter_shards_by_name(Name, [], Shards).
+
+filter_shards_by_name(_, Matches, []) ->
+ Matches;
+filter_shards_by_name(Name, Matches, [#ordered_shard{name=Name}=S|Ss]) ->
+ filter_shards_by_name(Name, [S|Matches], Ss);
+filter_shards_by_name(Name, Matches, [#shard{name=Name}=S|Ss]) ->
+ filter_shards_by_name(Name, [S|Matches], Ss);
+filter_shards_by_name(Name, Matches, [_|Ss]) ->
+ filter_shards_by_name(Name, Matches, Ss).
http://git-wip-us.apache.org/repos/asf/couchdb-mem3/blob/130efcd6/src/mem3_sync_event_listener.erl
----------------------------------------------------------------------
diff --git a/src/mem3_sync_event_listener.erl b/src/mem3_sync_event_listener.erl
index d74c21f..ca058db 100644
--- a/src/mem3_sync_event_listener.erl
+++ b/src/mem3_sync_event_listener.erl
@@ -207,13 +207,18 @@ maybe_push_shards(St) ->
end.
push_shard(ShardName) ->
- try mem3:shards(mem3:dbname(ShardName)) of
+ try mem3_shards:for_shard_name(ShardName) of
Shards ->
- Targets = [S || #shard{node=N, name=Name} = S <- Shards,
- N =/= node(), Name =:= ShardName],
Live = nodes(),
- [mem3_sync:push(ShardName,N) || #shard{node=N} <- Targets,
- lists:member(N, Live)]
+ lists:foreach(
+ fun(#shard{node=N}) ->
+ case lists:member(N, Live) of
+ true -> mem3_sync:push(ShardName, N);
+ false -> ok
+ end
+ end,
+ Shards
+ )
catch error:database_does_not_exist ->
ok
end.