You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/03/25 20:06:45 UTC
[couchdb] branch prototype/fdb-layer-view-cleanup created (now
2bd6462)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 2bd6462 Optionally cleanup stale indices automatically
This branch includes the following new commits:
new aa69628 Fix index updater configuration keys
new 9989d5e Update to use `fabric2_db:get_design_docs/1`
new f6d5419 Extend fabric2_index callbacks for index cleanup
new a07f219 Implement couch_views:cleanup_indices/2
new 2a0c328 Implement _view_cleanup for FoundationDB
new 2bd6462 Optionally cleanup stale indices automatically
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 05/06: Implement _view_cleanup for FoundationDB
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2a0c328e3fb670d882e1e5e0937b6c7dd8ab92d5
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:52:26 2020 -0500
Implement _view_cleanup for FoundationDB
---
src/chttpd/src/chttpd_db.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index dea992c..7f43984 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -272,7 +272,7 @@ handle_compact_req(Req, _Db) ->
send_method_not_allowed(Req, "POST").
handle_view_cleanup_req(Req, Db) ->
- ok = fabric:cleanup_index_files_all_nodes(Db),
+ ok = fabric2_index:cleanup(Db),
send_json(Req, 202, {[{ok, true}]}).
[couchdb] 01/06: Fix index updater configuration keys
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit aa69628f251941a05ce3a9d81f0e61f35d406b30
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:04:07 2020 -0500
Fix index updater configuration keys
The defaults commented out in `default.ini` did not match the names used
in `fabric2_index.erl`.
---
rel/overlay/etc/default.ini | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 749cdd2..3c36da2 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -228,14 +228,14 @@ port = 6984
;fdb_directory = couchdb
;
; Enable or disable index auto-updater
-;index_autoupdater_enabled = true
+;index_updater_enabled = true
;
; How long to wait from the first db update event until index building is
; triggered.
-;index_autoupdater_delay_msec = 60000
+;index_updater_delay_msec = 60000
;
; How often to check if databases may need their indices updated.
-;index_autoupdater_resolution_msec = 10000
+;index_updater_resolution_msec = 10000
; [rexi]
; buffer_count = 2000
[couchdb] 06/06: Optionally cleanup stale indices automatically
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2bd64628493942d5c21292054a5bf26aa50126ad
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:00:45 2020 -0500
Optionally cleanup stale indices automatically
---
rel/overlay/etc/default.ini | 3 +++
src/fabric/src/fabric2_index.erl | 22 +++++++++++++++++-----
2 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 3c36da2..29583d1 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -236,6 +236,9 @@ port = 6984
;
; How often to check if databases may need their indices updated.
;index_updater_resolution_msec = 10000
+;
+; Enable or disable automatic stale index removal in the auto-updater
+;index_updater_remove_old_indices = false
; [rexi]
; buffer_count = 2000
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index b197cb8..c6410d6 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -60,10 +60,8 @@ db_updated(DbName) when is_binary(DbName) ->
cleanup(Db) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
- DDocs = fabric2_db:get_design_docs(TxDb),
- lists:foreach(fun(Mod) ->
- Mod:cleanup_indices(TxDb, DDocs)
- end, registrations())
+ DDocs = fabric2_db:get_design_docs(Db),
+ cleanup_indices(TxDb, DDocs)
end)
catch
error:database_does_not_exist ->
@@ -185,6 +183,12 @@ build_indices(Db, DDocs) ->
end, registrations()).
+cleanup_indices(Db, DDocs) ->
+ lists:foreach(fun(Mod) ->
+ Mod:cleanup_indices(TxDb, DDocs)
+ end, registrations()).
+
+
registrations() ->
application:get_env(fabric, indices, []).
@@ -195,7 +199,11 @@ process_db(DbName) when is_binary(DbName) ->
DDocs1 = fabric2_db:get_design_docs(TxDb),
DDocs2 = lists:filter(fun should_update/1, DDocs1),
DDocs3 = shuffle(DDocs2),
- build_indices(TxDb, DDocs3)
+ build_indices(TxDb, DDocs3),
+ case auto_cleanup() of
+ true -> cleanup_indices(TxDb, DDocs1);
+ false -> ok
+ end
end).
@@ -225,3 +233,7 @@ delay_msec() ->
resolution_msec() ->
config:get_integer("fabric", "index_updater_resolution_msec",
?DEFAULT_RESOLUTION_MSEC).
+
+
+auto_cleanup() ->
+ config:get_boolean("fabric", "index_updater_remove_old_indices", false).
[couchdb] 02/06: Update to use `fabric2_db:get_design_docs/1`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9989d5ee9a954ee80333dc7a7917cc4f7b49ffdc
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:33:15 2020 -0500
Update to use `fabric2_db:get_design_docs/1`
---
src/fabric/src/fabric2_index.erl | 18 +-----------------
1 file changed, 1 insertion(+), 17 deletions(-)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 9382105..12863fb 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -171,29 +171,13 @@ registrations() ->
process_db(DbName) when is_binary(DbName) ->
{ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
fabric2_fdb:transactional(Db, fun(TxDb) ->
- DDocs1 = get_design_docs(TxDb),
+ DDocs1 = fabric2_db:get_design_docs(TxDb),
DDocs2 = lists:filter(fun should_update/1, DDocs1),
DDocs3 = shuffle(DDocs2),
build_indices(TxDb, DDocs3)
end).
-get_design_docs(Db) ->
- Callback = fun
- ({meta, _}, Acc) -> {ok, Acc};
- (complete, Acc) -> {ok, Acc};
- ({row, Row}, Acc) -> {ok, [get_doc(Db, Row) | Acc]}
- end,
- {ok, DDocs} = fabric2_db:fold_design_docs(Db, Callback, [], []),
- DDocs.
-
-
-get_doc(Db, Row) ->
- {_, DocId} = lists:keyfind(id, 1, Row),
- {ok, #doc{deleted = false} = Doc} = fabric2_db:open_doc(Db, DocId, []),
- Doc.
-
-
should_update(#doc{body = {Props}}) ->
couch_util:get_value(<<"autoupdate">>, Props, true).
[couchdb] 03/06: Extend fabric2_index callbacks for index cleanup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f6d5419451b93c50b2bdd3221b3c67b56468282e
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:50:45 2020 -0500
Extend fabric2_index callbacks for index cleanup
Each registered index type can now get a signal on when to clean up
their indexes.
---
src/fabric/src/fabric2_index.erl | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 12863fb..b197cb8 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -38,6 +38,9 @@
-callback build_indices(Db :: map(), DDocs :: list(#doc{})) ->
[{ok, JobId::binary()} | {error, any()}].
+-callback cleanup_indices(Db :: map(), DDocs :: list(#doc{})) ->
+ [ok | {error, any()}].
+
-define(SHARDS, 32).
-define(DEFAULT_DELAY_MSEC, 60000).
@@ -54,6 +57,24 @@ db_updated(DbName) when is_binary(DbName) ->
ets:insert_new(Table, {DbName, now_msec()}).
+cleanup(Db) ->
+ try
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ DDocs = fabric2_db:get_design_docs(TxDb),
+ lists:foreach(fun(Mod) ->
+ Mod:cleanup_indices(TxDb, DDocs)
+ end, registrations())
+ end)
+ catch
+ error:database_does_not_exist ->
+ ok;
+ Tag:Reason ->
+ Stack = erlang:get_stacktrace(),
+ LogMsg = "~p failed to cleanup indices for `~s` ~p:~p ~p",
+ couch_log:error(LogMsg, [?MODULE, Db, Tag, Reason, Stack])
+ end.
+
+
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
[couchdb] 04/06: Implement couch_views:cleanup_indices/2
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit a07f219272102abb89ea308ea79ce02c412da6c8
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:51:47 2020 -0500
Implement couch_views:cleanup_indices/2
---
src/couch_views/src/couch_views.erl | 17 ++++++++++++++
src/couch_views/src/couch_views_fdb.erl | 40 ++++++++++++++++++++++++++++++++-
2 files changed, 56 insertions(+), 1 deletion(-)
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 2268052..5f7e767 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -21,6 +21,7 @@
% fabric2_index behavior
build_indices/2,
+ cleanup_indices/2,
get_info/2
]).
@@ -75,6 +76,22 @@ build_indices(#{} = Db, DDocs) when is_list(DDocs) ->
end, DDocs).
+cleanup_indices(#{} = Db, DDocs) when is_list(DDocs) ->
+ ActiveSigs = lists:filtermap(fun(DDoc) ->
+ try couch_views_util:ddoc_to_mrst(DbName, DDoc) of
+ {ok, #mrst{sig = Sig}} ->
+ {true, Sig}
+ catch _:_ ->
+ false
+ end
+ end, DDocs),
+ ExistingSigs = couch_views_fdb:list_signatures(Db),
+ StaleSigs = ExistingSigs -- ActiveSigs,
+ lists:foreach(fun(Sig) ->
+ couch_views_fdb:clear_index(Db, Sig)
+ end, StaleSigs).
+
+
get_info(Db, DDoc) ->
DbName = fabric2_db:name(Db),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index a0224b2..64d6f66 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -21,7 +21,10 @@
fold_map_idx/6,
- write_doc/4
+ write_doc/4,
+
+ list_signatures/1,
+ clear_index/2
]).
-ifdef(TEST).
@@ -151,6 +154,41 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
end, lists:zip3(ViewIds, Results, KVSizes)).
+list_signatures(Db) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ ViewSeqRange = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ}
+ RangePrefix = erlfdb_tuple:pack(ViewSeqRange, DbPrefix),
+ fabric2_fdb:fold_range(Db, RangePrefix, fun({Key, _Val}, Acc) ->
+ {Sig} = erlfdb_tuple:unpack(Key, RangePrefix),
+ [Sig | Acc]
+ end, [], []).
+
+
+clear_index(Db, Signature) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = Db,
+
+ % Clear index info keys
+ Keys = [
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Signature},
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_ROW_COUNT, Signature},
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Signature}
+ ],
+ lists:foreach(fun(Key) ->
+ FDBKey = erlfdb_tuple:pack(Key, DbPrefix),
+ erlfdb:clear(Tx, FDBKey)
+ end, Keys),
+
+ % Clear index data
+ RangeTuple = {?DB_VIEWS, ?VIEW_DATA, Signature}
+ RangePrefix = erlfdb_tuple:pack(RangeTuple, DbPrefix),
+ erlfdb:clear_range_startswith(Tx, RangePrefix).
+
+
% For each row in a map view we store the the key/value
% in FoundationDB:
%