You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/03/25 20:06:45 UTC

[couchdb] branch prototype/fdb-layer-view-cleanup created (now 2bd6462)

This is an automated email from the ASF dual-hosted git repository.

davisp pushed a change to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


      at 2bd6462  Optionally cleanup stale indices automatically

This branch includes the following new commits:

     new aa69628  Fix index updater configuration keys
     new 9989d5e  Update to use `fabric2_db:get_design_docs/1`
     new f6d5419  Extend fabric2_index callbacks for index cleanup
     new a07f219  Implement couch_views:cleanup_indices/2
     new 2a0c328  Implement _view_cleanup for FoundationDB
     new 2bd6462  Optionally cleanup stale indices automatically

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[couchdb] 05/06: Implement _view_cleanup for FoundationDB

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 2a0c328e3fb670d882e1e5e0937b6c7dd8ab92d5
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:52:26 2020 -0500

    Implement _view_cleanup for FoundationDB
---
 src/chttpd/src/chttpd_db.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index dea992c..7f43984 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -272,7 +272,7 @@ handle_compact_req(Req, _Db) ->
     send_method_not_allowed(Req, "POST").
 
 handle_view_cleanup_req(Req, Db) ->
-    ok = fabric:cleanup_index_files_all_nodes(Db),
+    ok = fabric2_index:cleanup(Db),
     send_json(Req, 202, {[{ok, true}]}).
 
 


[couchdb] 01/06: Fix index updater configuration keys

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit aa69628f251941a05ce3a9d81f0e61f35d406b30
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:04:07 2020 -0500

    Fix index updater configuration keys
    
    The defaults commented out in `default.ini` did not match the names used
    in `fabric2_index.erl`.
---
 rel/overlay/etc/default.ini | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 749cdd2..3c36da2 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -228,14 +228,14 @@ port = 6984
 ;fdb_directory = couchdb
 ;
 ; Enable or disable index auto-updater
-;index_autoupdater_enabled = true
+;index_updater_enabled = true
 ;
 ; How long to wait from the first db update event until index building is
 ; triggered.
-;index_autoupdater_delay_msec = 60000
+;index_updater_delay_msec = 60000
 ;
 ; How often to check if databases may need their indices updated.
-;index_autoupdater_resolution_msec = 10000
+;index_updater_resolution_msec = 10000
 
 ; [rexi]
 ; buffer_count = 2000


[couchdb] 06/06: Optionally cleanup stale indices automatically

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 2bd64628493942d5c21292054a5bf26aa50126ad
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:00:45 2020 -0500

    Optionally cleanup stale indices automatically
---
 rel/overlay/etc/default.ini      |  3 +++
 src/fabric/src/fabric2_index.erl | 22 +++++++++++++++++-----
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 3c36da2..29583d1 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -236,6 +236,9 @@ port = 6984
 ;
 ; How often to check if databases may need their indices updated.
 ;index_updater_resolution_msec = 10000
+;
+; Enable or disable automatic stale index removal in the auto-updater
+;index_updater_remove_old_indices = false
 
 ; [rexi]
 ; buffer_count = 2000
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index b197cb8..c6410d6 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -60,10 +60,8 @@ db_updated(DbName) when is_binary(DbName) ->
 cleanup(Db) ->
     try
         fabric2_fdb:transactional(Db, fun(TxDb) ->
-            DDocs = fabric2_db:get_design_docs(TxDb),
-            lists:foreach(fun(Mod) ->
-                Mod:cleanup_indices(TxDb, DDocs)
-            end, registrations())
+            DDocs = fabric2_db:get_design_docs(Db),
+            cleanup_indices(TxDb, DDocs)
         end)
     catch
         error:database_does_not_exist ->
@@ -185,6 +183,12 @@ build_indices(Db, DDocs) ->
     end, registrations()).
 
 
+cleanup_indices(Db, DDocs) ->
+    lists:foreach(fun(Mod) ->
+        Mod:cleanup_indices(TxDb, DDocs)
+    end, registrations()).
+
+
 registrations() ->
     application:get_env(fabric, indices, []).
 
@@ -195,7 +199,11 @@ process_db(DbName) when is_binary(DbName) ->
         DDocs1 = fabric2_db:get_design_docs(TxDb),
         DDocs2 = lists:filter(fun should_update/1, DDocs1),
         DDocs3 = shuffle(DDocs2),
-        build_indices(TxDb, DDocs3)
+        build_indices(TxDb, DDocs3),
+        case auto_cleanup() of
+            true -> cleanup_indices(TxDb, DDocs1);
+            false -> ok
+        end
     end).
 
 
@@ -225,3 +233,7 @@ delay_msec() ->
 resolution_msec() ->
     config:get_integer("fabric", "index_updater_resolution_msec",
         ?DEFAULT_RESOLUTION_MSEC).
+
+
+auto_cleanup() ->
+    config:get_boolean("fabric", "index_updater_remove_old_indices", false).


[couchdb] 02/06: Update to use `fabric2_db:get_design_docs/1`

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 9989d5ee9a954ee80333dc7a7917cc4f7b49ffdc
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:33:15 2020 -0500

    Update to use `fabric2_db:get_design_docs/1`
---
 src/fabric/src/fabric2_index.erl | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 9382105..12863fb 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -171,29 +171,13 @@ registrations() ->
 process_db(DbName) when is_binary(DbName) ->
     {ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
     fabric2_fdb:transactional(Db, fun(TxDb) ->
-        DDocs1 = get_design_docs(TxDb),
+        DDocs1 = fabric2_db:get_design_docs(TxDb),
         DDocs2 = lists:filter(fun should_update/1, DDocs1),
         DDocs3 = shuffle(DDocs2),
         build_indices(TxDb, DDocs3)
     end).
 
 
-get_design_docs(Db) ->
-    Callback = fun
-        ({meta, _}, Acc) ->  {ok, Acc};
-        (complete, Acc) ->  {ok, Acc};
-        ({row, Row}, Acc) -> {ok, [get_doc(Db, Row) | Acc]}
-    end,
-    {ok, DDocs} = fabric2_db:fold_design_docs(Db, Callback, [], []),
-    DDocs.
-
-
-get_doc(Db, Row) ->
-    {_, DocId} = lists:keyfind(id, 1, Row),
-    {ok, #doc{deleted = false} = Doc} = fabric2_db:open_doc(Db, DocId, []),
-    Doc.
-
-
 should_update(#doc{body = {Props}}) ->
     couch_util:get_value(<<"autoupdate">>, Props, true).
 


[couchdb] 03/06: Extend fabric2_index callbacks for index cleanup

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit f6d5419451b93c50b2bdd3221b3c67b56468282e
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:50:45 2020 -0500

    Extend fabric2_index callbacks for index cleanup
    
    Each registered index type can now get a signal on when to clean up
    their indexes.
---
 src/fabric/src/fabric2_index.erl | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 12863fb..b197cb8 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -38,6 +38,9 @@
 -callback build_indices(Db :: map(), DDocs :: list(#doc{})) ->
     [{ok, JobId::binary()} | {error, any()}].
 
+-callback cleanup_indices(Db :: map(), DDocs :: list(#doc{})) ->
+    [ok | {error, any()}].
+
 
 -define(SHARDS, 32).
 -define(DEFAULT_DELAY_MSEC, 60000).
@@ -54,6 +57,24 @@ db_updated(DbName) when is_binary(DbName) ->
     ets:insert_new(Table, {DbName, now_msec()}).
 
 
+cleanup(Db) ->
+    try
+        fabric2_fdb:transactional(Db, fun(TxDb) ->
+            DDocs = fabric2_db:get_design_docs(TxDb),
+            lists:foreach(fun(Mod) ->
+                Mod:cleanup_indices(TxDb, DDocs)
+            end, registrations())
+        end)
+    catch
+        error:database_does_not_exist ->
+            ok;
+        Tag:Reason ->
+            Stack = erlang:get_stacktrace(),
+            LogMsg = "~p failed to cleanup indices for `~s` ~p:~p ~p",
+            couch_log:error(LogMsg, [?MODULE, Db, Tag, Reason, Stack])
+    end.
+
+
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 


[couchdb] 04/06: Implement couch_views:cleanup_indices/2

Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

davisp pushed a commit to branch prototype/fdb-layer-view-cleanup
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit a07f219272102abb89ea308ea79ce02c412da6c8
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:51:47 2020 -0500

    Implement couch_views:cleanup_indices/2
---
 src/couch_views/src/couch_views.erl     | 17 ++++++++++++++
 src/couch_views/src/couch_views_fdb.erl | 40 ++++++++++++++++++++++++++++++++-
 2 files changed, 56 insertions(+), 1 deletion(-)

diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 2268052..5f7e767 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -21,6 +21,7 @@
 
     % fabric2_index behavior
     build_indices/2,
+    cleanup_indices/2,
     get_info/2
 ]).
 
@@ -75,6 +76,22 @@ build_indices(#{} = Db, DDocs) when is_list(DDocs) ->
     end, DDocs).
 
 
+cleanup_indices(#{} = Db, DDocs) when is_list(DDocs) ->
+    ActiveSigs = lists:filtermap(fun(DDoc) ->
+        try couch_views_util:ddoc_to_mrst(DbName, DDoc) of
+            {ok, #mrst{sig = Sig}} ->
+                {true, Sig}
+        catch _:_ ->
+            false
+        end
+    end, DDocs),
+    ExistingSigs = couch_views_fdb:list_signatures(Db),
+    StaleSigs = ExistingSigs -- ActiveSigs,
+    lists:foreach(fun(Sig) ->
+        couch_views_fdb:clear_index(Db, Sig)
+    end, StaleSigs).
+
+
 get_info(Db, DDoc) ->
     DbName = fabric2_db:name(Db),
     {ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index a0224b2..64d6f66 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -21,7 +21,10 @@
 
     fold_map_idx/6,
 
-    write_doc/4
+    write_doc/4,
+
+    list_signatures/1,
+    clear_index/2
 ]).
 
 -ifdef(TEST).
@@ -151,6 +154,41 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
     end, lists:zip3(ViewIds, Results, KVSizes)).
 
 
+list_signatures(Db) ->
+    #{
+        db_prefix := DbPrefix
+    } = Db,
+    ViewSeqRange = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ}
+    RangePrefix = erlfdb_tuple:pack(ViewSeqRange, DbPrefix),
+    fabric2_fdb:fold_range(Db, RangePrefix, fun({Key, _Val}, Acc) ->
+        {Sig} = erlfdb_tuple:unpack(Key, RangePrefix),
+        [Sig | Acc]
+    end, [], []).
+
+
+clear_index(Db, Signature) ->
+    #{
+        tx := Tx,
+        db_prefix := DbPrefix
+    } = Db,
+
+    % Clear index info keys
+    Keys = [
+        {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Signature},
+        {?DB_VIEWS, ?VIEW_INFO, ?VIEW_ROW_COUNT, Signature},
+        {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Signature}
+    ],
+    lists:foreach(fun(Key) ->
+        FDBKey = erlfdb_tuple:pack(Key, DbPrefix),
+        erlfdb:clear(Tx, FDBKey)
+    end, Keys),
+
+    % Clear index data
+    RangeTuple = {?DB_VIEWS, ?VIEW_DATA, Signature}
+    RangePrefix = erlfdb_tuple:pack(RangeTuple, DbPrefix),
+    erlfdb:clear_range_startswith(Tx, RangePrefix).
+
+
 % For each row in a map view we store the the key/value
 % in FoundationDB:
 %