You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2017/05/19 15:10:27 UTC
[couchdb] branch master updated: Revert couch_lru to use gb_trees
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/master by this push:
new fb6faec Revert couch_lru to use gb_trees
fb6faec is described below
commit fb6faec9b91b6593c3c10e35728a2f9cfc623a43
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Wed May 17 17:30:20 2017 -0400
Revert couch_lru to use gb_trees
Recently couch_lru was changed to use ets tables.
During eprof profiling it showed improved performance however recently in a
larger test with more concurrent updates and 5000 max dbs open it showed a
significant degradation compared to the previous (gb_tree-based) version.
GH Issue #528
---
src/couch/src/couch_lru.erl | 62 +++++++++------------
src/couch/test/couch_lru_tests.erl | 109 -------------------------------------
2 files changed, 27 insertions(+), 144 deletions(-)
diff --git a/src/couch/src/couch_lru.erl b/src/couch/src/couch_lru.erl
index b79286e..b58a623 100644
--- a/src/couch/src/couch_lru.erl
+++ b/src/couch/src/couch_lru.erl
@@ -16,39 +16,33 @@
-include_lib("couch/include/couch_db.hrl").
new() ->
- Updates = ets:new(couch_lru_updates, [ordered_set]),
- Dbs = ets:new(couch_lru_dbs, [set]),
- {0, Updates, Dbs}.
-
-insert(DbName, {Count, Updates, Dbs}) ->
- update(DbName, {Count, Updates, Dbs}).
-
-update(DbName, {Count, Updates, Dbs}) ->
- case ets:lookup(Dbs, DbName) of
- [] ->
- true = ets:insert(Dbs, {DbName, Count});
- [{DbName, OldCount}] ->
- true = ets:update_element(Dbs, DbName, {2, Count}),
- true = ets:delete(Updates, {OldCount, DbName})
- end,
- true = ets:insert(Updates, {{Count, DbName}}),
- {Count + 1, Updates, Dbs}.
-
-
-close({Count, Updates, Dbs}) ->
- case close_int(ets:next(Updates, {-1, <<>>}), Updates, Dbs) of
- true ->
- {true, {Count, Updates, Dbs}};
- false ->
- false
+ {gb_trees:empty(), dict:new()}.
+
+insert(DbName, {Tree0, Dict0}) ->
+ Lru = erlang:now(),
+ {gb_trees:insert(Lru, DbName, Tree0), dict:store(DbName, Lru, Dict0)}.
+
+update(DbName, {Tree0, Dict0}) ->
+ case dict:find(DbName, Dict0) of
+ {ok, Old} ->
+ New = erlang:now(),
+ Tree = gb_trees:insert(New, DbName, gb_trees:delete(Old, Tree0)),
+ Dict = dict:store(DbName, New, Dict0),
+ {Tree, Dict};
+ error ->
+ % We closed this database before processing the update. Ignore
+ {Tree0, Dict0}
end.
+%% Attempt to close the oldest idle database.
+close({Tree, _} = Cache) ->
+ close_int(gb_trees:next(gb_trees:iterator(Tree)), Cache).
%% internals
-close_int('$end_of_table', _Updates, _Dbs) ->
+close_int(none, _) ->
false;
-close_int({_Count, DbName} = Key, Updates, Dbs) ->
+close_int({Lru, DbName, Iter}, {Tree, Dict} = Cache) ->
case ets:update_element(couch_dbs, DbName, {#db.fd_monitor, locked}) of
true ->
[#db{main_pid = Pid} = Db] = ets:lookup(couch_dbs, DbName),
@@ -56,16 +50,14 @@ close_int({_Count, DbName} = Key, Updates, Dbs) ->
true = ets:delete(couch_dbs, DbName),
true = ets:delete(couch_dbs_pid_to_name, Pid),
exit(Pid, kill),
- true = ets:delete(Updates, Key),
- true = ets:delete(Dbs, DbName),
- true;
+ {true, {gb_trees:delete(Lru, Tree), dict:erase(DbName, Dict)}};
false ->
true = ets:update_element(couch_dbs, DbName, {#db.fd_monitor, nil}),
couch_stats:increment_counter([couchdb, couch_server, lru_skip]),
- close_int(ets:next(Updates, Key), Updates, Dbs)
+ close_int(gb_trees:next(Iter), update(DbName, Cache))
end;
false ->
- true = ets:delete(Updates, Key),
- true = ets:delete(Dbs, DbName),
- close_int(ets:next(Updates, Key), Updates, Dbs)
- end.
+ NewTree = gb_trees:delete(Lru, Tree),
+ NewIter = gb_trees:iterator(NewTree),
+ close_int(gb_trees:next(NewIter), {NewTree, dict:erase(DbName, Dict)})
+end.
diff --git a/src/couch/test/couch_lru_tests.erl b/src/couch/test/couch_lru_tests.erl
deleted file mode 100644
index 1559835..0000000
--- a/src/couch/test/couch_lru_tests.erl
+++ /dev/null
@@ -1,109 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_lru_tests).
-
--include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
-
-
-setup() ->
- ok = meck:new(couch_db, [passthrough]),
- ets:new(couch_dbs, [set, public, named_table, {keypos, #db.name}]),
- ets:new(couch_dbs_pid_to_name, [set, public, named_table]),
- couch_lru:new().
-
-teardown(_) ->
- ets:delete(couch_dbs),
- ets:delete(couch_dbs_pid_to_name),
- (catch meck:unload(couch_db)).
-
-new_test_() ->
- {setup,
- fun() -> couch_lru:new() end,
- fun(Lru) ->
- ?_assertMatch({0, _, _}, Lru)
- end
- }.
-
-insert_test_() ->
- {setup,
- fun() -> couch_lru:new() end,
- fun(Lru) ->
- Key = <<"test">>,
- {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
- [
- ?_assertEqual(1, ets_size(Dbs)),
- ?_assert(ets:member(Dbs, Key)),
- ?_assertEqual(1, ets_size(Updates)),
- ?_assert(ets:member(Updates, {0, Key}))
- ]
- end
- }.
-
-insert_same_test_() ->
- {setup,
- fun() -> couch_lru:new() end,
- fun(Lru) ->
- Key = <<"test">>,
- Lru1 = {1, Updates, Dbs} = couch_lru:insert(Key, Lru),
- {2, Updates, Dbs} = couch_lru:insert(Key, Lru1),
- [
- ?_assertEqual(1, ets_size(Dbs)),
- ?_assert(ets:member(Dbs, Key)),
- ?_assertEqual(1, ets_size(Updates)),
- ?_assert(ets:member(Updates, {1, Key}))
- ]
- end
- }.
-
-update_test_() ->
- {setup,
- fun() -> couch_lru:new() end,
- fun(Lru) ->
- Key = <<"test">>,
- Lru1 = {1, Updates, Dbs} = couch_lru:update(Key, Lru),
- {2, Updates, Dbs} = couch_lru:update(Key, Lru1),
- [
- ?_assertEqual(1, ets_size(Dbs)),
- ?_assert(ets:member(Dbs, Key)),
- ?_assertEqual(1, ets_size(Updates)),
- ?_assert(ets:member(Updates, {1, Key}))
- ]
- end
- }.
-
-close_test_() ->
- {setup,
- fun setup/0,
- fun teardown/1,
- fun(Lru) ->
- ok = meck:expect(couch_db, is_idle, 1, true),
- {ok, Lru1} = add_record(Lru, <<"test1">>, c:pid(0, 1001, 0)),
- {ok, Lru2} = add_record(Lru1, <<"test2">>, c:pid(0, 2001, 0)),
- {true, {2, Updates, Dbs}} = couch_lru:close(Lru2),
- [
- ?_assertEqual(1, ets_size(Dbs)),
- ?_assert(ets:member(Dbs, <<"test2">>)),
- ?_assertEqual(1, ets_size(Updates)),
- ?_assert(ets:member(Updates, {1, <<"test2">>}))
- ]
- end
- }.
-
-add_record(Lru, Key, Pid) ->
- true = ets:insert(couch_dbs, #db{name = Key, main_pid = Pid}),
- true = ets:insert(couch_dbs_pid_to_name, {Pid, Key}),
- {ok, couch_lru:insert(Key, Lru)}.
-
-ets_size(Ets) ->
- proplists:get_value(size, ets:info(Ets)).
--
To stop receiving notification emails like this one, please contact
['"commits@couchdb.apache.org" <co...@couchdb.apache.org>'].