[couchdb] branch expiring-cache updated: Documentation & cleanup

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jaydoane pushed a commit to branch expiring-cache
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/expiring-cache by this push:
     new 43894b1  Documentation & cleanup
43894b1 is described below

commit 43894b11f31792d0f028182eb7202104998d3c7b
Author: Jay Doane <ja...@apache.org>
AuthorDate: Thu Nov 21 15:23:30 2019 -0800

    Documentation & cleanup
---
 src/couch_expiring_cache/README.md                 | 19 +++++++++-
 .../include/couch_expiring_cache.hrl               | 14 ++++++++
 .../src/couch_expiring_cache.erl                   | 16 +++++++--
 .../src/couch_expiring_cache_fdb.erl               | 12 +++++--
 .../src/couch_expiring_cache_server.erl            | 42 ++++++++++++----------
 .../test/couch_expiring_cache_tests.erl            |  6 ++--
 6 files changed, 81 insertions(+), 28 deletions(-)

diff --git a/src/couch_expiring_cache/README.md b/src/couch_expiring_cache/README.md
index 34cbc09..049b787 100644
--- a/src/couch_expiring_cache/README.md
+++ b/src/couch_expiring_cache/README.md
@@ -1,3 +1,20 @@
 # Couch Expiring Cache
 
-Key value cache with expiring entries.
+Key value cache with entries which are automatically deleted after
+they expire.
+
+This is a description of some of the modules:
+
+* `couch_expiring_cache`: The API module, it contains functions for
+  inserting and looking up cache entries, which are simply
+  pass-throughs to `couch_expiring_cache_fdb`.
+
+* `couch_expiring_cache_fdb`: The module which interacts with FDB, in
+  addition to insertion and lookup functions, it also contains a
+  function to clear an expired range.
+
+* `couch_expiring_cache_server`: A gen_server, this module
+  periodically removes expired cache entries using configurable
+  parameters for period, jitter, and batch size. It also fetches the
+  layer prefix on startup, and caches it in an ets table for
+  performance.
diff --git a/src/couch_expiring_cache/include/couch_expiring_cache.hrl b/src/couch_expiring_cache/include/couch_expiring_cache.hrl
index fe771c5..0d24025 100644
--- a/src/couch_expiring_cache/include/couch_expiring_cache.hrl
+++ b/src/couch_expiring_cache/include/couch_expiring_cache.hrl
@@ -1 +1,15 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
 -define(TIME_UNIT, millisecond).
+
+-type millisecond() :: non_neg_integer().
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache.erl b/src/couch_expiring_cache/src/couch_expiring_cache.erl
index 61f4c20..3e6a48a 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache.erl
@@ -1,3 +1,15 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+%   http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
 -module(couch_expiring_cache).
 
 -export([
@@ -6,11 +18,11 @@
 ]).
 
 
--type millisecond() :: non_neg_integer().
+-include_lib("couch_expiring_cache/include/couch_expiring_cache.hrl").
 
 
 -spec insert(Name :: binary(), Key :: binary(), Value :: binary(),
-        StaleTS :: millisecond(), ExpiresTS :: millisecond()) -> ok.
+    StaleTS :: millisecond(), ExpiresTS :: millisecond()) -> ok.
 insert(Name, Key, Value, StaleTS, ExpiresTS) ->
     couch_expiring_cache_fdb:insert(Name, Key, Value, StaleTS, ExpiresTS).
 
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
index febf103..a9aac82 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -34,6 +34,8 @@
 % (?XC, ?EXP, ExpireTS, Name, Key) := ()
 
 
+-spec insert(Name :: binary(), Key :: binary(), Value :: binary(),
+    StaleTS :: millisecond(), ExpiresTS :: millisecond()) -> ok.
 insert(Name, Key, Val, StaleTS, ExpiresTS) ->
     LayerPrefix = couch_expiring_cache_server:layer_prefix(),
     PK = primary_key(Name, Key, LayerPrefix),
@@ -46,6 +48,8 @@ insert(Name, Key, Val, StaleTS, ExpiresTS) ->
     end).
 
 
+-spec lookup(Name :: binary(), Key :: binary()) ->
+    not_found | {fresh, Val :: binary()} | {stale, Val :: binary()} | expired.
 lookup(Name, Key) ->
     LayerPrefix = couch_expiring_cache_server:layer_prefix(),
     PK = primary_key(Name, Key, LayerPrefix),
@@ -65,17 +69,19 @@ lookup(Name, Key) ->
     end).
 
 
+-spec clear_expired_range(EndTS :: millisecond(), Limit :: non_neg_integer()) ->
+    OldestTS :: integer().
 clear_expired_range(EndTS, Limit) when Limit > 0 ->
+    LayerPrefix = couch_expiring_cache_server:layer_prefix(),
+    ExpiresPrefix = erlfdb_tuple:pack({?XC, ?EXP}, LayerPrefix),
     Callback = fun
         (TS, 0) when TS > 0 -> TS;
         (TS, Acc) -> min(TS, Acc)
     end,
-    LayerPrefix = couch_expiring_cache_server:layer_prefix(),
-    ExpiresPrefix = erlfdb_tuple:pack({?XC, ?EXP}, LayerPrefix),
     fabric2_fdb:transactional(fun(Tx) ->
         fabric2_fdb:fold_range({tx, Tx}, ExpiresPrefix, fun({K, _V}, Acc) ->
             Unpacked = erlfdb_tuple:unpack(K, ExpiresPrefix),
-            couch_log:info("~p clearing ~p", [?MODULE, Unpacked]),
+            couch_log:debug("~p clearing ~p", [?MODULE, Unpacked]),
             {ExpiresTS, Name, Key} = Unpacked,
             clear_expired(Tx, ExpiresTS, Name, Key, LayerPrefix),
             Callback(ExpiresTS, Acc)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
index e14e863..b2ab221 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -46,11 +46,12 @@ start_link() ->
 
 init(_) ->
     ?MODULE = ets:new(?MODULE, [named_table, public, {read_concurrency, true}]),
-    true = set_layer_prefix(),
+    true = ets_insert_layer_prefix(),
     {ok, #{
         timer_ref => schedule_remove_expired(),
-        last_removal => 0,
         oldest_ts => 0,
+        elapsed => 0,
+        largest_elapsed => 0,
         lag => 0}}.
 
 
@@ -66,15 +67,24 @@ handle_cast(Msg, St) ->
     {stop, {bad_cast, Msg}, St}.
 
 
-handle_info(remove_expired, St = #{oldest_ts := OldestTS0}) ->
-    Now = erlang:system_time(?TIME_UNIT),
-    OldestTS = max(OldestTS0, remove_expired(Now)),
-    Ref = schedule_remove_expired(),
+handle_info(remove_expired, St) ->
+    #{
+        oldest_ts := OldestTS0,
+        largest_elapsed := LargestElapsed
+    } = St,
+    BatchSize = batch_size(),
+
+    NowTS = erlang:system_time(?TIME_UNIT),
+    OldestTS = max(OldestTS0,
+        couch_expiring_cache_fdb:clear_expired_range(NowTS, BatchSize)),
+    Elapsed = erlang:system_time(?TIME_UNIT) - NowTS,
+
     {noreply, St#{
-        timer_ref := Ref,
-        last_removal := Now,
+        timer_ref := schedule_remove_expired(),
         oldest_ts := OldestTS,
-        lag := Now - OldestTS}};
+        elapsed := Elapsed,
+        largest_elapsed := max(Elapsed, LargestElapsed),
+        lag := NowTS - OldestTS}};
 
 handle_info(Msg, St) ->
     {stop, {bad_info, Msg}, St}.
@@ -92,15 +102,11 @@ layer_prefix() ->
 %% Private
 
 
-set_layer_prefix() ->
-    fabric2_fdb:transactional(fun(Tx) ->
-        LayerPrefix = fabric2_fdb:get_dir(Tx),
-        true = ets:insert(?MODULE, {layer_prefix, LayerPrefix})
-    end).
-
-
-remove_expired(EndTS) ->
-    couch_expiring_cache_fdb:clear_expired_range(EndTS, batch_size()).
+ets_insert_layer_prefix() ->
+    Prefix = fabric2_fdb:transactional(fun(Tx) ->
+        fabric2_fdb:get_dir(Tx)
+    end),
+    true = ets:insert(?MODULE, {layer_prefix, Prefix}).
 
 
 schedule_remove_expired() ->
diff --git a/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl b/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
index 95748e2..808d5cd 100644
--- a/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
+++ b/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
@@ -42,8 +42,7 @@ setup_couch() ->
 
 
 teardown_couch(Ctx) ->
-    test_util:stop_couch(Ctx),
-    meck:unload().
+    test_util:stop_couch(Ctx).
 
 
 setup() ->
@@ -54,8 +53,7 @@ setup() ->
 
 
 teardown(#{}) ->
-    application:stop(couch_expiring_cache),
-    meck:unload().
+    application:stop(couch_expiring_cache).
 
 
 simple_lifecycle(#{}) ->