You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2019/11/11 23:42:28 UTC
[couchdb] 01/05: WIP
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3a40ac30a426fb24e7bc48050021633841057057
Author: Jay Doane <ja...@apache.org>
AuthorDate: Tue Oct 22 00:07:29 2019 -0700
WIP
---
src/couch_expiring_cache/README.md | 3 +
src/couch_expiring_cache/rebar.config | 14 ++
.../src/couch_expiring_cache.app.src | 30 +++++
.../src/couch_expiring_cache_app.erl | 26 ++++
.../src/couch_expiring_cache_fdb.erl | 144 +++++++++++++++++++++
.../src/couch_expiring_cache_server.erl | 98 ++++++++++++++
.../src/couch_expiring_cache_sup.erl | 46 +++++++
7 files changed, 361 insertions(+)
diff --git a/src/couch_expiring_cache/README.md b/src/couch_expiring_cache/README.md
new file mode 100644
index 0000000..34cbc09
--- /dev/null
+++ b/src/couch_expiring_cache/README.md
@@ -0,0 +1,3 @@
+# Couch Expiring Cache
+
+Key value cache with expiring entries.
diff --git a/src/couch_expiring_cache/rebar.config b/src/couch_expiring_cache/rebar.config
new file mode 100644
index 0000000..362c878
--- /dev/null
+++ b/src/couch_expiring_cache/rebar.config
@@ -0,0 +1,14 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{cover_enabled, true}.
+{cover_print_enabled, true}.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache.app.src b/src/couch_expiring_cache/src/couch_expiring_cache.app.src
new file mode 100644
index 0000000..cf9443a
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache.app.src
@@ -0,0 +1,30 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+{application, couch_expiring_cache, [
+ {description, "CouchDB Expiring Cache"},
+ {vsn, git},
+ {mod, {couch_expiring_cache_app, []}},
+ {registered, [
+ couch_expiring_cache_sup,
+ couch_expiring_cache_server
+ ]},
+ {applications, [
+ kernel,
+ stdlib,
+ erlfdb,
+ config,
+ couch_log,
+ couch_stats,
+ fabric
+ ]}
+]}.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_app.erl b/src/couch_expiring_cache/src/couch_expiring_cache_app.erl
new file mode 100644
index 0000000..7a4a63b
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_app.erl
@@ -0,0 +1,26 @@
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_app).
+
+
+-behaviour(application).
+
+
+-export([
+ start/2,
+ stop/1
+]).
+
+
+start(_Type, []) ->
+ couch_expiring_cache_sup:start_link().
+
+
+stop([]) ->
+ ok.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
new file mode 100644
index 0000000..f432813
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -0,0 +1,144 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_fdb).
+
+-export([
+ get_range/2,
+ cache_prefix/1,
+ layer_prefix/1,
+ list_keys/1,
+ list_exp/0,
+ remove_exp/3,
+ %% delete_all/1,
+ insert/5,
+ lookup/2,
+ remove/2
+]).
+
+
+-define(DEFAULT_LIMIT, 100000).
+
+-define(XC, 53). % coordinate with fabric2.hrl
+-define(PK, 1).
+-define(EXP, 2).
+
+
+% Data model
+% see: https://forums.foundationdb.org/t/designing-key-value-expiration-in-fdb/156
+%
+% (?XC, ?PK, Name, Key) := (Val, StaleTS, ExpireTS)
+% (?XC, ?EXP, ExpireTS, Name, Key) := ()
+
+
+list_keys(Name) ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ list_keys_int(Name, Callback, Tx)
+ end).
+
+list_keys_int(Name, Callback, Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?PK, Name}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ {Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Key, Acc)
+ end, [], []).
+
+
+list_exp() ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?EXP}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ Unpacked = {_ExpiresTS, _Name, _Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Unpacked, Acc)
+ end, [], [])
+ end).
+
+
+get_range(EndTS, Limit) when Limit > 0 ->
+ Callback = fun(Key, Acc) -> [Key | Acc] end,
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = erlfdb_tuple:pack({?XC, ?EXP}, layer_prefix(Tx)),
+ fabric2_fdb:fold_range({tx, Tx}, Prefix, fun({K, _V}, Acc) ->
+ Unpacked = {_ExpiresTS, _Name, _Key} = erlfdb_tuple:unpack(K, Prefix),
+ Callback(Unpacked, Acc)
+ end, [], [{end_key, EndTS}, {limit, Limit}])
+ end).
+
+
+remove_exp(ExpiresTS, Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
+ ok = erlfdb:clear(Tx, PK),
+ ok = erlfdb:clear(Tx, XK)
+ end).
+
+
+insert(Name, Key, Val, StaleTS, ExpiresTS) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ PV = erlfdb_tuple:pack({Val, StaleTS, ExpiresTS}),
+ ok = erlfdb:set(Tx, PK, PV),
+
+ XK = erlfdb_tuple:pack({?XC, ?EXP, ExpiresTS, Name, Key}, Prefix),
+ XV = erlfdb_tuple:pack({}),
+ ok = erlfdb:set(Tx, XK, XV)
+ end).
+
+
+lookup(Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ case erlfdb:wait(erlfdb:get(Tx, PK)) of
+ not_found ->
+ not_found;
+ Bin when is_binary(Bin) ->
+ {Val, StaleTS, ExpiresTS} = erlfdb_tuple:unpack(Bin),
+ Now = erlang:system_time(second),
+ if
+ Now < StaleTS -> {fresh, Val};
+ Now < ExpiresTS -> {stale, Val};
+ true -> expired
+ end
+ end
+ end).
+
+
+remove(Name, Key) ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ Prefix = layer_prefix(Tx),
+
+ PK = erlfdb_tuple:pack({?XC, ?PK, Name, Key}, Prefix),
+ erlfdb:clear(Tx, PK)
+ end).
+
+
+layer_prefix(Tx) ->
+ fabric2_fdb:get_dir(Tx).
+
+
+cache_prefix(Tx) ->
+ erlfdb_tuple:pack({?XC}, layer_prefix(Tx)).
+
+
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+-endif.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
new file mode 100644
index 0000000..0072c38
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -0,0 +1,98 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_server).
+
+-behaviour(gen_server).
+
+-export([
+ start_link/0
+]).
+
+-export([
+ init/1,
+ terminate/2,
+ handle_call/3,
+ handle_cast/2,
+ handle_info/2,
+ code_change/3
+]).
+
+
+-define(PERIOD_DEFAULT, 10).
+-define(MAX_JITTER_DEFAULT, 1).
+
+
+start_link() ->
+ gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+
+init(_) ->
+ Ref = schedule_remove_expired(),
+ {ok, #{timer_ref => Ref}}.
+
+
+terminate(_, _) ->
+ ok.
+
+
+handle_call(Msg, _From, St) ->
+ {stop, {bad_call, Msg}, {bad_call, Msg}, St}.
+
+
+handle_cast(Msg, St) ->
+ {stop, {bad_cast, Msg}, St}.
+
+
+handle_info(remove_expired, St) ->
+ ok = remove_expired(),
+ Ref = schedule_remove_expired(),
+ {noreply, St#{timer_ref => Ref}};
+
+handle_info(Msg, St) ->
+ {stop, {bad_info, Msg}, St}.
+
+
+code_change(_OldVsn, St, _Extra) ->
+ {ok, St}.
+
+
+remove_expired() ->
+ Now = erlang:system_time(second),
+ Limit = 10,
+ Expired = couch_expiring_cache_fdb:get_range(Now, Limit),
+ case Expired of
+ [] ->
+ ok;
+ _ ->
+ lists:foreach(fun({TS, Name, Key} = Exp) ->
+ couch_log:info("~p remove_expired ~p", [?MODULE, Exp]),
+ couch_expiring_cache_fdb:remove_exp(TS, Name, Key)
+ end, Expired)
+ end.
+
+
+schedule_remove_expired() ->
+ Timeout = get_period_sec(),
+ MaxJitter = max(Timeout div 2, get_max_jitter_sec()),
+ Wait = Timeout + rand:uniform(max(1, MaxJitter)),
+ erlang:send_after(Wait * 1000, self(), remove_expired).
+
+
+get_period_sec() ->
+ config:get_integer("couch_expiring_cache", "period_sec",
+ ?PERIOD_DEFAULT).
+
+
+get_max_jitter_sec() ->
+ config:get_integer("couch_expiring_cache", "max_jitter_sec",
+ ?MAX_JITTER_DEFAULT).
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl b/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl
new file mode 100644
index 0000000..22b7b4d
--- /dev/null
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_sup.erl
@@ -0,0 +1,46 @@
+%
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_expiring_cache_sup).
+
+
+-behaviour(supervisor).
+
+
+-export([
+ start_link/0
+]).
+
+-export([
+ init/1
+]).
+
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+
+init([]) ->
+ Flags = #{
+ strategy => one_for_one,
+ intensity => 3,
+ period => 10
+ },
+ Children = [
+ #{
+ id => couch_expiring_cache_server,
+ restart => permanent,
+ start => {couch_expiring_cache_server, start_link, []}
+ }
+ ],
+ {ok, {Flags, Children}}.