You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2019/11/11 23:42:27 UTC
[couchdb] branch expiring-cache updated (76e1341 -> f91f9b3)
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a change to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
discard 76e1341 Improve efficiency and observability of expiration
discard e20c671 WIP
add 3f322a5 Remove compiler warning
add ed1c3d7 Merge pull request #2274 from cloudant/fix-warning
add 797fe08 Remove old clause which is no longer used
add c3ef462 Merge pull request #2275 from cloudant/remove-ints-client-remains
add 5334997 Chunkify local docs
add 3ded0e5 Add a special error for an invalid legacy local doc revsion
add 987efb3 add test to prove we can view swap
add 8d5c107 Use "\xFF/metadataVersion" key for checking metadata
add 8bb0718 Abandon a view job if the db or ddoc is deleted
add 583d7fe Pass contexts to fabric2_db functions
add 8d28d85 Merge pull request #2279 from cloudant/refactor-user-ctx-handling
add 3db0ba7 Ensure we can create partitioned design docs with FDB
new 3a40ac3 WIP
new f8b4b77 Improve efficiency and observability of expiration
new 3bddfb9 Rename last_expiration to last_removal
new 2ef0f55 Add couch_expiring_cache module
new f91f9b3 Temporarily fix broken function
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (76e1341)
\
N -- N -- N refs/heads/expiring-cache (f91f9b3)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/chttpd/src/chttpd.erl | 3 -
src/chttpd/src/chttpd_db.erl | 105 +++++++----------
src/couch/src/couch_httpd.erl | 3 -
.../src/couch_expiring_cache.erl | 13 +++
.../src/couch_expiring_cache_server.erl | 14 +--
src/couch_jobs/src/couch_jobs.hrl | 5 +-
src/couch_mrview/src/couch_mrview.erl | 14 +--
src/couch_views/src/couch_views_indexer.erl | 23 +++-
src/fabric/include/fabric2.hrl | 14 +--
src/fabric/src/fabric2_db.erl | 14 ++-
src/fabric/src/fabric2_fdb.erl | 93 +++++++++++----
src/fabric/test/fabric2_doc_crud_tests.erl | 125 +++++++++++++++++++++
..._tests.erl => fabric2_local_doc_fold_tests.erl} | 85 +++++++-------
src/mem3/src/mem3_reshard_index.erl | 10 +-
test/elixir/test/basics_test.exs | 2 +
test/elixir/test/map_test.exs | 67 +++++++++++
16 files changed, 414 insertions(+), 176 deletions(-)
create mode 100644 src/couch_expiring_cache/src/couch_expiring_cache.erl
copy src/fabric/test/{fabric2_doc_fold_tests.erl => fabric2_local_doc_fold_tests.erl} (73%)
[couchdb] 03/05: Rename last_expiration to last_removal
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3bddfb94b0243a051e548a912ce04c7885a366f5
Author: Jay Doane <ja...@apache.org>
AuthorDate: Sun Nov 10 23:52:50 2019 -0800
Rename last_expiration to last_removal
---
.../src/couch_expiring_cache_server.erl | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
index f67e7be..82a90aa 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -42,7 +42,7 @@ init(_) ->
{ok, #{
timer_ref => Ref,
lag => 0,
- last_expiration => 0,
+ last_removal => 0,
min_ts => 0}}.
@@ -58,15 +58,15 @@ handle_cast(Msg, St) ->
{stop, {bad_cast, Msg}, St}.
-handle_info(remove_expired, St) ->
+handle_info(remove_expired, St = #{min_ts := MinTS0}) ->
Now = erlang:system_time(second),
- MinTS = remove_expired(Now),
+ MinTS = max(MinTS0, remove_expired(Now)),
Ref = schedule_remove_expired(),
{noreply, St#{
- timer_ref => Ref,
- lag => Now - MinTS,
- last_expiration => Now,
- min_ts => MinTS}};
+ timer_ref := Ref,
+ lag := Now - MinTS,
+ last_removal := Now,
+ min_ts := MinTS}};
handle_info(Msg, St) ->
{stop, {bad_info, Msg}, St}.
[couchdb] 05/05: Temporarily fix broken function
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f91f9b35f79c882a3fb7b0c5b04b2cc8a04d8bdd
Author: Jay Doane <ja...@apache.org>
AuthorDate: Mon Nov 11 15:37:49 2019 -0800
Temporarily fix broken function
---
src/mem3/src/mem3_reshard_index.erl | 10 +---------
1 file changed, 1 insertion(+), 9 deletions(-)
diff --git a/src/mem3/src/mem3_reshard_index.erl b/src/mem3/src/mem3_reshard_index.erl
index d4cb7ca..fb4648b 100644
--- a/src/mem3/src/mem3_reshard_index.erl
+++ b/src/mem3/src/mem3_reshard_index.erl
@@ -100,15 +100,7 @@ mrview_indices(DbName, Doc) ->
dreyfus_indices(DbName, Doc) ->
- try
- Indices = dreyfus_index:design_doc_to_indexes(Doc),
- [{dreyfus, DbName, Index} || Index <- Indices]
- catch
- Tag:Err ->
- Msg = "~p couldn't get dreyfus indices ~p ~p ~p:~p",
- couch_log:error(Msg, [?MODULE, DbName, Doc, Tag, Err]),
- []
- end.
+ [].
hastings_indices(DbName, Doc) ->
[couchdb] 01/05: WIP
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3a40ac30a426fb24e7bc48050021633841057057
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Oct 22 00:07:29 2019 -0700
WIP
---
src/couch_expiring_cache/README.md | 3 +
src/couch_expiring_cache/rebar.config | 14 ++
.../src/couch_expiring_cache.app.src | 30 +++++
.../src/couch_expiring_cache_app.erl | 26 ++++
.../src/couch_expiring_cache_fdb.erl | 144 +++++++++++++++++++++
.../src/couch_expiring_cache_server.erl | 98 ++++++++++++++
.../src/couch_expiring_cache_sup.erl | 46 +++++++
7 files changed, 361 insertions(+)
diff --git a/src/couch_expiring_cache/README.md b/src/couch_expiring_cache/README.md
new file mode 100644
index 0000000..34cbc09
--- /dev/null
+++ b/src/couch_expiring_cache/README.md
@@ -0,0 +1,3 @@
+# Couch Expiring Cache
+
+Key value cache with expiring entries.
diff --git a/src/couch_expiring_cache/rebar.config b/src/couch_expiring_cache/rebar.config
new file mode 100644
index 0000000..362c878
--- /dev/null
+++ b/src/couch_expiring_cache/rebar.config
@@ -0,0 +1,14 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{cover_enabled, true}.
+{cover_print_enabled, true}.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache.app.src b/src/couch_expiring_cache/src/couch_expiring_cache.app.src
new file mode 100644
index 0000000..cf9443a
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache.app.src
@@ -0,0 +1,30 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application, couch_expiring_cache, [
+ {description, "CouchDB Expiring Cache"},
+ {vsn, git},
+ {mod, {couch_expiring_cache_app, []}},
+ {registered, [
+ couch_expiring_cache_sup,
+ couch_expiring_cache_server
+ ]},
+ {applications, [
+ kernel,
+ stdlib,
+ erlfdb,
+ config,
+ couch_log,
+ couch_stats,
+ fabric
+ ]}
+]}.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_app.erl b/src/couch_expiring_cache/src/couch_expiring_cache_app.erl
new file mode 100644
index 0000000..7a4a63b
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_app.erl
@@ -0,0 +1,26 @@
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_app).
+
+
+-behaviour(application).
+
+
+-export([
+ start/2,
+ stop/1
+]).
+
+
+start(_Type, []) ->
+ couch_expiring_cache_sup:start_link().
+
+
+stop([]) ->
+ ok.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
new file mode 100644
index 0000000..f432813
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -0,0 +1,144 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_fdb).
+
+-export([
+ get_range/2,
+ cache_prefix/1,
+ layer_prefix/1,
+ list_keys/1,
+ list_exp/0,
+ remove_exp/3,
+ %% delete_all/1,
+ insert/5,
+ lookup/2,
+ remove/2
+]).
+
+
+-define(DEFAULT_LIMIT, 100000).
+
+-define(XC, 53). % coordinate with fabric2.hrl
+-define(PK, 1).
+-define(EXP, 2).
+
+
+% Data model
+% see: https://forums.foundationdb.org/t/designing-key-value-expiration-in-fdb/156
+%
+% (?XC, ?PK, Name, Key) := (Val, StaleTS, ExpireTS)
+% (?XC, ?EXP, ExpireTS, Name, Key) := ()
+
+
+list_keys(Name) ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ list_keys_int(Name, Callback, Tx)
+ end).
+
+list_keys_int(Name, Callback, Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?PK, Name}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ {Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Key, Acc)
+ end, [], []).
+
+
+list_exp() ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?EXP}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ Unpacked = {_ExpiresTS, _Name, _Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Unpacked, Acc)
+ end, [], [])
+ end).
+
+
+get_range(EndTS, Limit) when Limit > 0 ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?EXP}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ Unpacked = {_ExpiresTS, _Name, _Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Unpacked, Acc)
+ end, [], [{end_key, EndTS}, {limit, Limit}])
+ end).
+
+
+remove_exp(ExpiresTS, Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
+ ok = erlfdb:clear(Tx, PK),
+ ok = erlfdb:clear(Tx, XK)
+ end).
+
+
+insert(Name, Key, Val, StaleTS, ExpiresTS) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ PV = erlfdb_tuple:pack({Val, StaleTS, ExpiresTS}),
+ ok = erlfdb:set(Tx, PK, PV),
+
+ XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
+ XV = erlfdb_tuple:pack({}),
+ ok = erlfdb:set(Tx, XK, XV)
+ end).
+
+
+lookup(Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ case erlfdb:wait(erlfdb:get(Tx, PK)) of
+ not_found ->
+ not_found;
+ Bin when is_binary(Bin) ->
+ {Val, StaleTS, ExpiresTS} = erlfdb_tuple:unpack(Bin),
+ Now = erlang:system_time(second),
+ if
+ Now < StaleTS -> {fresh, Val};
+ Now < ExpiresTS -> {stale, Val};
+ true -> expired
+ end
+ end
+ end).
+
+
+remove(Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ erlfdb:clear(Tx, PK)
+ end).
+
+
+layer_prefix(Tx) ->
+ fabric2_fdb:get_dir(Tx).
+
+
+cache_prefix(Tx) ->
+ erlfdb_tuple:pack({?XC}, layer_prefix(Tx)).
+
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
new file mode 100644
index 0000000..0072c38
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -0,0 +1,98 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_server).
+
+-behaviour(gen_server).
+
+-export([
+ start_link/0
+]).
+
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+-define(PERIOD_DEFAULT, 10).
+-define(MAX_JITTER_DEFAULT, 1).
+
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+ Ref = schedule_remove_expired(),
+ {ok, #{timer_ref => Ref}}.
+
+
+terminate(_, _) ->
+ ok.
+
+
+handle_call(Msg, _From, St) ->
+ {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+ {stop, {bad_cast, Msg}, St}.
+
+
+handle_info(remove_expired, St) ->
+ ok = remove_expired(),
+ Ref = schedule_remove_expired(),
+ {noreply, St#{timer_ref => Ref}};
+
+handle_info(Msg, St) ->
+ {stop, {bad_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+remove_expired() ->
+ Now = erlang:system_time(second),
+ Limit = 10,
+ Expired = couch_expiring_cache_fdb:get_range(Now, Limit),
+ case Expired of
+ [] ->
+ ok;
+ _ ->
+ lists:foreach(fun({TS, Name, Key} = Exp) ->
+ couch_log:info("~p remove_expired ~p", [?MODULE, Exp]),
+ couch_expiring_cache_fdb:remove_exp(TS, Name, Key)
+ end, Expired)
+ end.
+
+
+schedule_remove_expired() ->
+ Timeout = get_period_sec(),
+ MaxJitter = max(Timeout div 2, get_max_jitter_sec()),
+ Wait = Timeout + rand:uniform(max(1, MaxJitter)),
+ erlang:send_after(Wait * 1000, self(), remove_expired).
+
+
+get_period_sec() ->
+ config:get_integer("couch_expiring_cache", "period_sec",
+ ?PERIOD_DEFAULT).
+
+
+get_max_jitter_sec() ->
+ config:get_integer("couch_expiring_cache", "max_jitter_sec",
+ ?MAX_JITTER_DEFAULT).
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl b/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl
new file mode 100644
index 0000000..22b7b4d
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl
@@ -0,0 +1,46 @@
+%
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_sup).
+
+
+-behaviour(supervisor).
+
+
+-export([
+ start_link/0
+]).
+
+-export([
+ init/1
+]).
+
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+
+init([]) ->
+ Flags = #{
+ strategy => one_for_one,
+ intensity => 3,
+ period => 10
+ },
+ Children = [
+ #{
+ id => couch_expiring_cache_server,
+ restart => permanent,
+ start => {couch_expiring_cache_server, start_link, []}
+ }
+ ],
+ {ok, {Flags, Children}}.
[couchdb] 04/05: Add couch_expiring_cache module
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2ef0f5596db626853803d360e99829fadc687db8
Author: Jay Doane <ja...@apache.org>
AuthorDate: Mon Nov 11 15:32:17 2019 -0800
Add couch_expiring_cache module
---
src/couch_expiring_cache/src/couch_expiring_cache.erl | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache.erl b/src/couch_expiring_cache/src/couch_expiring_cache.erl
new file mode 100644
index 0000000..62b6367
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache.erl
@@ -0,0 +1,13 @@
+-module(couch_expiring_cache).
+
+-export([
+ lookup/2,
+ insert/5
+]).
+
+
+insert(Name, Key, Value, StaleTS, ExpiresTS) ->
+ couch_expiring_cache_fdb:insert(Name, Key, Value, StaleTS, ExpiresTS).
+
+lookup(Name, Key) ->
+ couch_expiring_cache_fdb:lookup(Name, Key).
[couchdb] 02/05: Improve efficiency and observability of expiration
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f8b4b775db2dba2f3856ac5ec11646c83d278256
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Oct 29 00:03:19 2019 -0700
Improve efficiency and observability of expiration
---
.../src/couch_expiring_cache_fdb.erl | 55 ++++++++++++++------
.../src/couch_expiring_cache_server.erl | 58 ++++++++++++----------
2 files changed, 72 insertions(+), 41 deletions(-)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
index f432813..283a92b 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -13,6 +13,7 @@
-module(couch_expiring_cache_fdb).
-export([
+ clear_expired_range/2,
get_range/2,
cache_prefix/1,
layer_prefix/1,
@@ -22,11 +23,11 @@
%% delete_all/1,
insert/5,
lookup/2,
- remove/2
+ remove_primary/2
]).
--define(DEFAULT_LIMIT, 100000).
+-define(DEFAULT_LIMIT, 100000). % How to enforce?
-define(XC, 53). % coordinate with fabric2.hrl
-define(PK, 1).
@@ -78,12 +79,32 @@ get_range(EndTS, Limit) when Limit > 0 ->
remove_exp(ExpiresTS, Name, Key) ->
fabric2_fdb:transactional(fun(Tx) ->
- Prefix = layer_prefix(Tx),
+ clear_expired(Tx, ExpiresTS, Name, Key, layer_prefix(Tx))
+ end).
+
- PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
- XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
- ok = erlfdb:clear(Tx, PK),
- ok = erlfdb:clear(Tx, XK)
+clear_expired(Tx, ExpiresTS, Name, Key, Prefix) ->
+ PK = primary_key(Name, Key, Prefix),
+ XK = expiry_key(ExpiresTS, Name, Key, Prefix),
+ ok = erlfdb:clear(Tx, PK),
+ ok = erlfdb:clear(Tx, XK).
+
+
+clear_expired_range(EndTS, Limit) when Limit > 0 ->
+ Callback = fun
+ (TS, 0) when TS > 0 -> TS;
+ (TS, Acc) -> min(TS, Acc)
+ end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ LayerPrefix = layer_prefix(Tx),
+ ExpiresPrefix = erlfdb_tuple:pack({?XC, ?EXP}, LayerPrefix),
+ fabric2_fdb:fold_range({tx, Tx}, ExpiresPrefix, fun({K, _V}, Acc) ->
+ Unpacked = erlfdb_tuple:unpack(K, ExpiresPrefix),
+ couch_log:info("~p clearing ~p", [?MODULE, Unpacked]),
+ {ExpiresTS, Name, Key} = Unpacked,
+ clear_expired(Tx, ExpiresTS, Name, Key, LayerPrefix),
+ Callback(ExpiresTS, Acc)
+ end, 0, [{end_key, EndTS}, {limit, Limit}])
end).
@@ -91,11 +112,11 @@ insert(Name, Key, Val, StaleTS, ExpiresTS) ->
fabric2_fdb:transactional(fun(Tx) ->
Prefix = layer_prefix(Tx),
- PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ PK = primary_key(Name, Key, Prefix),
PV = erlfdb_tuple:pack({Val, StaleTS, ExpiresTS}),
ok = erlfdb:set(Tx, PK, PV),
- XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
+ XK = expiry_key(ExpiresTS, Name, Key, Prefix),
XV = erlfdb_tuple:pack({}),
ok = erlfdb:set(Tx, XK, XV)
end).
@@ -105,7 +126,7 @@ lookup(Name, Key) ->
fabric2_fdb:transactional(fun(Tx) ->
Prefix = layer_prefix(Tx),
- PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ PK = primary_key(Name, Key, Prefix),
case erlfdb:wait(erlfdb:get(Tx, PK)) of
not_found ->
not_found;
@@ -121,15 +142,21 @@ lookup(Name, Key) ->
end).
-remove(Name, Key) ->
+remove_primary(Name, Key) ->
fabric2_fdb:transactional(fun(Tx) ->
- Prefix = layer_prefix(Tx),
-
- PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ PK = primary_key(Name, Key, layer_prefix(Tx)),
erlfdb:clear(Tx, PK)
end).
+primary_key(Name, Key, Prefix) ->
+ erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix).
+
+
+expiry_key(ExpiresTS, Name, Key, Prefix) ->
+ erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix).
+
+
layer_prefix(Tx) ->
fabric2_fdb:get_dir(Tx).
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
index 0072c38..f67e7be 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -28,8 +28,9 @@
]).
--define(PERIOD_DEFAULT, 10).
--define(MAX_JITTER_DEFAULT, 1).
+-define(DEFAULT_BATCH, 1000).
+-define(DEFAULT_PERIOD_MS, 5000).
+-define(DEFAULT_MAX_JITTER_MS, 1000).
start_link() ->
@@ -38,7 +39,11 @@ start_link() ->
init(_) ->
Ref = schedule_remove_expired(),
- {ok, #{timer_ref => Ref}}.
+ {ok, #{
+ timer_ref => Ref,
+ lag => 0,
+ last_expiration => 0,
+ min_ts => 0}}.
terminate(_, _) ->
@@ -54,9 +59,14 @@ handle_cast(Msg, St) ->
handle_info(remove_expired, St) ->
- ok = remove_expired(),
+ Now = erlang:system_time(second),
+ MinTS = remove_expired(Now),
Ref = schedule_remove_expired(),
- {noreply, St#{timer_ref => Ref}};
+ {noreply, St#{
+ timer_ref => Ref,
+ lag => Now - MinTS,
+ last_expiration => Now,
+ min_ts => MinTS}};
handle_info(Msg, St) ->
{stop, {bad_info, Msg}, St}.
@@ -66,33 +76,27 @@ code_change(_OldVsn, St, _Extra) ->
{ok, St}.
-remove_expired() ->
- Now = erlang:system_time(second),
- Limit = 10,
- Expired = couch_expiring_cache_fdb:get_range(Now, Limit),
- case Expired of
- [] ->
- ok;
- _ ->
- lists:foreach(fun({TS, Name, Key} = Exp) ->
- couch_log:info("~p remove_expired ~p", [?MODULE, Exp]),
- couch_expiring_cache_fdb:remove_exp(TS, Name, Key)
- end, Expired)
- end.
+remove_expired(EndTS) ->
+ couch_expiring_cache_fdb:clear_expired_range(EndTS, batch_size()).
schedule_remove_expired() ->
- Timeout = get_period_sec(),
- MaxJitter = max(Timeout div 2, get_max_jitter_sec()),
+ Timeout = period_ms(),
+ MaxJitter = max(Timeout div 2, max_jitter_ms()),
Wait = Timeout + rand:uniform(max(1, MaxJitter)),
- erlang:send_after(Wait * 1000, self(), remove_expired).
+ erlang:send_after(Wait, self(), remove_expired).
+
+
+period_ms() ->
+ config:get_integer("couch_expiring_cache", "period_ms",
+ ?DEFAULT_PERIOD_MS).
-get_period_sec() ->
- config:get_integer("couch_expiring_cache", "period_sec",
- ?PERIOD_DEFAULT).
+max_jitter_ms() ->
+ config:get_integer("couch_expiring_cache", "max_jitter_ms",
+ ?DEFAULT_MAX_JITTER_MS).
-get_max_jitter_sec() ->
- config:get_integer("couch_expiring_cache", "max_jitter_sec",
- ?MAX_JITTER_DEFAULT).
+batch_size() ->
+ config:get_integer("couch_expiring_cache", "batch_size",
+ ?DEFAULT_BATCH).