You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ja...@apache.org on 2020/04/16 05:09:38 UTC
[couchdb] branch prototype/fdb-layer updated (27cbad7 -> 4098c12)
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a change to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from 27cbad7 report changes stats intermittently (#2777)
new 0eb1a73 Refactor expiring cache FDB interface
new 4098c12 Clean up old expiry key on update insert
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../src/couch_expiring_cache_fdb.erl | 87 ++++++++++++++++------
.../src/couch_expiring_cache_server.erl | 2 +-
.../test/couch_expiring_cache_tests.erl | 19 ++++-
src/fabric/include/fabric2.hrl | 1 +
4 files changed, 82 insertions(+), 27 deletions(-)
[couchdb] 01/02: Refactor expiring cache FDB interface
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0eb1a73af424aa6403c420f8f6ebb3fe4d760f62
Author: Jay Doane <ja...@apache.org>
AuthorDate: Wed Apr 15 22:01:13 2020 -0700
Refactor expiring cache FDB interface
- Rename `clear_expired_range` to `clear_range_to`
- Move `EXPIRING_CACHE` layer prefix into fabric2.hrl
- Move primary key setting to just after key & value calculations
- Factor out `get_val/2` to lookup a key from FDB and unpack the value
- Factor out `prefixes/2`
- Factor out `fold_range/5`
---
.../src/couch_expiring_cache_fdb.erl | 60 +++++++++++++---------
.../src/couch_expiring_cache_server.erl | 2 +-
src/fabric/include/fabric2.hrl | 1 +
3 files changed, 38 insertions(+), 25 deletions(-)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
index fa8508e..dc33766 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -16,15 +16,15 @@
insert/6,
lookup/3,
clear_all/1,
- clear_expired_range/3
+ clear_range_to/3
]).
--define(EXPIRING_CACHE, 53). % coordinate with fabric2.hrl
-define(PK, 1).
-define(EXP, 2).
+-include_lib("fabric/include/fabric2.hrl").
-include_lib("couch_expiring_cache/include/couch_expiring_cache.hrl").
@@ -41,9 +41,9 @@ insert(#{jtx := true} = JTx, Name, Key, Val, StaleTS, ExpiresTS) ->
#{tx := Tx, layer_prefix := LayerPrefix} = couch_jobs_fdb:get_jtx(JTx),
PK = primary_key(Name, Key, LayerPrefix),
PV = erlfdb_tuple:pack({Val, StaleTS, ExpiresTS}),
+ ok = erlfdb:set(Tx, PK, PV),
XK = expiry_key(ExpiresTS, Name, Key, LayerPrefix),
XV = erlfdb_tuple:pack({}),
- ok = erlfdb:set(Tx, PK, PV),
ok = erlfdb:set(Tx, XK, XV).
@@ -52,11 +52,10 @@ insert(#{jtx := true} = JTx, Name, Key, Val, StaleTS, ExpiresTS) ->
lookup(#{jtx := true} = JTx, Name, Key) ->
#{tx := Tx, layer_prefix := LayerPrefix} = couch_jobs_fdb:get_jtx(JTx),
PK = primary_key(Name, Key, LayerPrefix),
- case erlfdb:wait(erlfdb:get(Tx, PK)) of
+ case get_val(Tx, PK) of
not_found ->
not_found;
- Bin when is_binary(Bin) ->
- {Val, StaleTS, ExpiresTS} = erlfdb_tuple:unpack(Bin),
+ {Val, StaleTS, ExpiresTS} ->
Now = erlang:system_time(?TIME_UNIT),
if
Now < StaleTS -> {fresh, Val};
@@ -76,32 +75,30 @@ clear_all(Name) ->
end).
--spec clear_expired_range(Name :: binary(), EndTS :: ?TIME_UNIT,
+-spec clear_range_to(Name :: binary(), EndTS :: ?TIME_UNIT,
Limit :: non_neg_integer()) ->
OldestTS :: ?TIME_UNIT.
-clear_expired_range(Name, EndTS, Limit) when Limit > 0 ->
- fabric2_fdb:transactional(fun(Tx) ->
- LayerPrefix = fabric2_fdb:get_dir(Tx),
- ExpiresPrefix = erlfdb_tuple:pack(
- {?EXPIRING_CACHE, Name, ?EXP}, LayerPrefix),
- fabric2_fdb:fold_range({tx, Tx}, ExpiresPrefix, fun({K, _V}, Acc) ->
- Unpacked = erlfdb_tuple:unpack(K, ExpiresPrefix),
- couch_log:debug("~p clearing ~p", [?MODULE, Unpacked]),
- {ExpiresTS, Key} = Unpacked,
- clear_expired(Tx, ExpiresTS, Name, Key, LayerPrefix),
+clear_range_to(Name, EndTS, Limit) when Limit > 0 ->
+ fold_range(Name, EndTS, Limit,
+ fun(Tx, PK, XK, _Key, ExpiresTS, Acc) ->
+ ok = erlfdb:clear(Tx, PK),
+ ok = erlfdb:clear(Tx, XK),
oldest_ts(ExpiresTS, Acc)
- end, 0, [{end_key, EndTS}, {limit, Limit}])
- end).
+ end, 0).
%% Private
-clear_expired(Tx, ExpiresTS, Name, Key, Prefix) ->
- PK = primary_key(Name, Key, Prefix),
- XK = expiry_key(ExpiresTS, Name, Key, Prefix),
- ok = erlfdb:clear(Tx, PK),
- ok = erlfdb:clear(Tx, XK).
+fold_range(Name, EndTS, Limit, Fun, Acc0) when Limit > 0 ->
+ fabric2_fdb:transactional(fun(Tx) ->
+ {LayerPrefix, ExpiresPrefix} = prefixes(Tx, Name),
+ fabric2_fdb:fold_range({tx, Tx}, ExpiresPrefix, fun({XK, _XV}, Acc) ->
+ {ExpiresTS, Key} = erlfdb_tuple:unpack(XK, ExpiresPrefix),
+ PK = primary_key(Name, Key, LayerPrefix),
+ Fun(Tx, PK, XK, Key, ExpiresTS, Acc)
+ end, Acc0, [{end_key, EndTS}, {limit, Limit}])
+ end).
oldest_ts(TS, 0) -> TS; % handle initial Acc = 0 case
@@ -114,3 +111,18 @@ primary_key(Name, Key, Prefix) ->
expiry_key(ExpiresTS, Name, Key, Prefix) ->
erlfdb_tuple:pack({?EXPIRING_CACHE, Name, ?EXP, ExpiresTS, Key}, Prefix).
+
+
+prefixes(Tx, Name) ->
+ Layer = fabric2_fdb:get_dir(Tx),
+ Expires = erlfdb_tuple:pack({?EXPIRING_CACHE, Name, ?EXP}, Layer),
+ {Layer, Expires}.
+
+
+get_val(Tx, PK) ->
+ case erlfdb:wait(erlfdb:get(Tx, PK)) of
+ not_found ->
+ not_found;
+ Bin when is_binary(Bin) ->
+ erlfdb_tuple:unpack(Bin)
+ end.
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
index 99d3864..eb74e6e 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_server.erl
@@ -82,7 +82,7 @@ handle_info(remove_expired, St) ->
NowTS = erlang:system_time(?TIME_UNIT),
OldestTS = max(OldestTS0,
- couch_expiring_cache_fdb:clear_expired_range(Name, NowTS, BatchSize)),
+ couch_expiring_cache_fdb:clear_range_to(Name, NowTS, BatchSize)),
Elapsed = erlang:system_time(?TIME_UNIT) - NowTS,
{noreply, St#{
diff --git a/src/fabric/include/fabric2.hrl b/src/fabric/include/fabric2.hrl
index 2e588f8..bf3e2aa 100644
--- a/src/fabric/include/fabric2.hrl
+++ b/src/fabric/include/fabric2.hrl
@@ -24,6 +24,7 @@
-define(DB_HCA, 2).
-define(DELETED_DBS, 3).
-define(DBS, 15).
+-define(EXPIRING_CACHE, 53).
-define(TX_IDS, 255).
% Database Level: (LayerPrefix, ?DBS, DbPrefix, X, ...)
[couchdb] 02/02: Clean up old expiry key on update insert
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jaydoane pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4098c12abeffdf89988663ec9d17544f6509256f
Author: Jay Doane <ja...@apache.org>
AuthorDate: Wed Apr 15 22:01:33 2020 -0700
Clean up old expiry key on update insert
When an existing key is inserted with different timestamps, the primary
key is the same but the primary value is different from the existing one.
Currently, this results in a new expiry key being inserted, but the old
one is not deleted and lingers until it is removed by the inexorable
advance of time via the `remove_expired` server messages.
This checks whether there's already primary key for the inserted key,
and if so, cleans up the existing expiry key before proceeding with the
insert.
---
.../src/couch_expiring_cache_fdb.erl | 27 ++++++++++++++++++++++
.../test/couch_expiring_cache_tests.erl | 19 +++++++++++++--
2 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
index dc33766..7c4ad8f 100644
--- a/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
+++ b/src/couch_expiring_cache/src/couch_expiring_cache_fdb.erl
@@ -13,6 +13,7 @@
-module(couch_expiring_cache_fdb).
-export([
+ get_range_to/3,
insert/6,
lookup/3,
clear_all/1,
@@ -40,6 +41,16 @@
insert(#{jtx := true} = JTx, Name, Key, Val, StaleTS, ExpiresTS) ->
#{tx := Tx, layer_prefix := LayerPrefix} = couch_jobs_fdb:get_jtx(JTx),
PK = primary_key(Name, Key, LayerPrefix),
+ case get_val(Tx, PK) of
+ not_found ->
+ ok;
+ {_OldVal, _OldStaleTS, OldExpiresTS} ->
+ % Clean up current expiry key for this primary key. No
+ % need to clean up the existing primary key since it will
+ % be overwritten below.
+ OldXK = expiry_key(OldExpiresTS, Name, Key, LayerPrefix),
+ ok = erlfdb:clear(Tx, OldXK)
+ end,
PV = erlfdb_tuple:pack({Val, StaleTS, ExpiresTS}),
ok = erlfdb:set(Tx, PK, PV),
XK = expiry_key(ExpiresTS, Name, Key, LayerPrefix),
@@ -87,6 +98,22 @@ clear_range_to(Name, EndTS, Limit) when Limit > 0 ->
end, 0).
+-spec get_range_to(Name :: binary(), EndTS :: ?TIME_UNIT,
+ Limit :: non_neg_integer()) ->
+ [{Key :: binary(), Val :: binary()}].
+get_range_to(Name, EndTS, Limit) when Limit > 0 ->
+ fold_range(Name, EndTS, Limit,
+ fun(Tx, PK, _XK, Key, _ExpiresTS, Acc) ->
+ case get_val(Tx, PK) of
+ not_found ->
+ couch_log:error("~p:entry missing Key: ~p", [?MODULE, Key]),
+ Acc;
+ Val ->
+ [{Key, Val} | Acc]
+ end
+ end, []).
+
+
%% Private
diff --git a/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl b/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
index aeb1df6..2e06fcc 100644
--- a/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
+++ b/src/couch_expiring_cache/test/couch_expiring_cache_tests.erl
@@ -66,7 +66,7 @@ teardown(#{pid := Pid}) ->
simple_lifecycle(_) ->
- ?_test(begin
+ {timeout, 10, ?_test(begin
Now = erlang:system_time(?TIME_UNIT),
StaleTS = Now + 100,
ExpiresTS = Now + 200,
@@ -76,14 +76,29 @@ simple_lifecycle(_) ->
?assertEqual(ok, couch_expiring_cache_fdb:clear_all(Name)),
?assertEqual(not_found, couch_expiring_cache:lookup(Name, Key)),
+ ?assertEqual([], entries(Name)),
?assertEqual(ok,
couch_expiring_cache:insert(Name, Key, Val, StaleTS, ExpiresTS)),
?assertEqual({fresh, Val}, couch_expiring_cache:lookup(Name, Key)),
ok = wait_lookup(Name, Key, {stale, Val}),
+
+ % Refresh the existing key with updated timestamps
+ ?assertEqual(ok,
+ couch_expiring_cache:insert(Name, Key, Val,
+ StaleTS + 100, ExpiresTS + 100)),
+ ?assertEqual({fresh, Val}, couch_expiring_cache:lookup(Name, Key)),
+ ?assertEqual(1, length(entries(Name))),
+ ok = wait_lookup(Name, Key, {stale, Val}),
ok = wait_lookup(Name, Key, expired),
ok = wait_lookup(Name, Key, not_found),
+ ?assertEqual([], entries(Name)),
?assertEqual(not_found, couch_expiring_cache:lookup(Name, Key))
- end).
+ end)}.
+
+
+entries(Name) ->
+ FarFuture = erlang:system_time(?TIME_UNIT) * 2,
+ couch_expiring_cache_fdb:get_range_to(Name, FarFuture, _Limit=100).
wait_lookup(Name, Key, Expect) ->