You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2020/04/10 21:30:59 UTC
[couchdb] branch prototype/fdb-layer updated (d4bc3a5 -> 7575152)
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a change to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
from d4bc3a5 Fix flaky fabric2_index test
new 742c64e Fix index updater configuration keys
new 247b809 Rename variables to indicate transaction state
new 3c0a017 Move process_db/1 to match the logical progression
new 3e1c822 Update to use `fabric2_db:get_design_docs/1`
new 7bc9148 Extend fabric2_index callbacks for index cleanup
new e0d0391 Implement couch_views:cleanup_indices/2
new 4275a49 Implement _view_cleanup for FoundationDB
new 7aeb54b Optionally cleanup stale indices automatically
new 2e5a556 Remove jobs on index cleanup
new 30fdef7 Remove failed view jobs
new 7575152 Implement couch_views_cleanup_test.erl
The 11 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
rel/overlay/etc/default.ini | 9 +-
src/chttpd/src/chttpd_db.erl | 2 +-
src/chttpd/src/chttpd_httpd_handlers.erl | 2 +-
src/couch_views/src/couch_views.erl | 19 +
src/couch_views/src/couch_views_fdb.erl | 40 ++-
src/couch_views/src/couch_views_jobs.erl | 35 +-
src/couch_views/test/couch_views_cleanup_test.erl | 411 ++++++++++++++++++++++
src/couch_views/test/couch_views_indexer_test.erl | 2 +-
src/couch_views/test/couch_views_map_test.erl | 2 +-
src/fabric/src/fabric2_index.erl | 73 ++--
10 files changed, 553 insertions(+), 42 deletions(-)
create mode 100644 src/couch_views/test/couch_views_cleanup_test.erl
[couchdb] 05/11: Extend fabric2_index callbacks for index cleanup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7bc9148b75b1396b91b6cdccca2c8e87b791e0f0
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:50:45 2020 -0500
Extend fabric2_index callbacks for index cleanup
Each registered index type can now get a signal on when to clean up
their indexes.
---
src/fabric/src/fabric2_index.erl | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 9a6607e..7f9d519 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -19,6 +19,7 @@
-export([
register_index/1,
db_updated/1,
+ cleanup/1,
start_link/0
]).
@@ -38,6 +39,9 @@
-callback build_indices(Db :: map(), DDocs :: list(#doc{})) ->
[{ok, JobId::binary()} | {error, any()}].
+-callback cleanup_indices(Db :: map(), DDocs :: list(#doc{})) ->
+ [ok | {error, any()}].
+
-define(SHARDS, 32).
-define(DEFAULT_DELAY_MSEC, 60000).
@@ -54,6 +58,25 @@ db_updated(DbName) when is_binary(DbName) ->
ets:insert_new(Table, {DbName, now_msec()}).
+cleanup(Db) ->
+ try
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ DDocs = fabric2_db:get_design_docs(TxDb),
+ lists:foreach(fun(Mod) ->
+ Mod:cleanup_indices(TxDb, DDocs)
+ end, registrations())
+ end)
+ catch
+ error:database_does_not_exist ->
+ ok;
+ Tag:Reason ->
+ Stack = erlang:get_stacktrace(),
+ DbName = fabric2_db:name(Db),
+ LogMsg = "~p failed to cleanup indices for `~s` ~p:~p ~p",
+ couch_log:error(LogMsg, [?MODULE, DbName, Tag, Reason, Stack])
+ end.
+
+
start_link() ->
gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
[couchdb] 06/11: Implement couch_views:cleanup_indices/2
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e0d0391ff8a639101814fae8e74f73b2403561fd
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:51:47 2020 -0500
Implement couch_views:cleanup_indices/2
---
src/couch_views/src/couch_views.erl | 18 +++++++++++++++
src/couch_views/src/couch_views_fdb.erl | 40 ++++++++++++++++++++++++++++++++-
2 files changed, 57 insertions(+), 1 deletion(-)
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index 2acba00..cc18364 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -21,6 +21,7 @@
% fabric2_index behavior
build_indices/2,
+ cleanup_indices/2,
get_info/2
]).
@@ -76,6 +77,23 @@ build_indices(#{} = Db, DDocs) when is_list(DDocs) ->
end, DDocs).
+cleanup_indices(#{} = Db, DDocs) when is_list(DDocs) ->
+ DbName = fabric2_db:name(Db),
+ ActiveSigs = lists:filtermap(fun(DDoc) ->
+ try couch_views_util:ddoc_to_mrst(DbName, DDoc) of
+ {ok, #mrst{sig = Sig}} ->
+ {true, Sig}
+ catch _:_ ->
+ false
+ end
+ end, DDocs),
+ ExistingSigs = couch_views_fdb:list_signatures(Db),
+ StaleSigs = ExistingSigs -- ActiveSigs,
+ lists:foreach(fun(Sig) ->
+ couch_views_fdb:clear_index(Db, Sig)
+ end, StaleSigs).
+
+
get_info(Db, DDoc) ->
DbName = fabric2_db:name(Db),
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
diff --git a/src/couch_views/src/couch_views_fdb.erl b/src/couch_views/src/couch_views_fdb.erl
index 3b008d4..2181e53 100644
--- a/src/couch_views/src/couch_views_fdb.erl
+++ b/src/couch_views/src/couch_views_fdb.erl
@@ -27,7 +27,10 @@
fold_map_idx/6,
- write_doc/4
+ write_doc/4,
+
+ list_signatures/1,
+ clear_index/2
]).
-ifdef(TEST).
@@ -211,6 +214,41 @@ write_doc(TxDb, Sig, ViewIds, Doc) ->
end, lists:zip3(ViewIds, Results, KVSizes)).
+list_signatures(Db) ->
+ #{
+ db_prefix := DbPrefix
+ } = Db,
+ ViewSeqRange = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ},
+ RangePrefix = erlfdb_tuple:pack(ViewSeqRange, DbPrefix),
+ fabric2_fdb:fold_range(Db, RangePrefix, fun({Key, _Val}, Acc) ->
+ {Sig} = erlfdb_tuple:unpack(Key, RangePrefix),
+ [Sig | Acc]
+ end, [], []).
+
+
+clear_index(Db, Signature) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = Db,
+
+ % Clear index info keys
+ Keys = [
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Signature},
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_ROW_COUNT, Signature},
+ {?DB_VIEWS, ?VIEW_INFO, ?VIEW_KV_SIZE, Signature}
+ ],
+ lists:foreach(fun(Key) ->
+ FDBKey = erlfdb_tuple:pack(Key, DbPrefix),
+ erlfdb:clear(Tx, FDBKey)
+ end, Keys),
+
+ % Clear index data
+ RangeTuple = {?DB_VIEWS, ?VIEW_DATA, Signature},
+ RangePrefix = erlfdb_tuple:pack(RangeTuple, DbPrefix),
+ erlfdb:clear_range_startswith(Tx, RangePrefix).
+
+
% For each row in a map view we store the the key/value
% in FoundationDB:
%
[couchdb] 02/11: Rename variables to indicate transaction state
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 247b80997fc2b1777606746c6b4ec4eaa48ad352
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 16:04:06 2020 -0500
Rename variables to indicate transaction state
Usually we indicate the transaction status of a Db handle by naming it
`TxDb`. This updates fabric2_index:build_indices/2 to match that
pattern.
Co-Authored-By: Nick Vatamaniuc <va...@apache.org>
---
src/fabric/src/fabric2_index.erl | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 9382105..098d6ed 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -155,12 +155,12 @@ process_updates_iter([Db | Rest], Cont) ->
process_updates_iter(Rest, Cont).
-build_indices(_Db, []) ->
+build_indices(_TxDb, []) ->
[];
-build_indices(Db, DDocs) ->
+build_indices(TxDb, DDocs) ->
lists:flatmap(fun(Mod) ->
- Mod:build_indices(Db, DDocs)
+ Mod:build_indices(TxDb, DDocs)
end, registrations()).
[couchdb] 04/11: Update to use `fabric2_db:get_design_docs/1`
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3e1c822794990cab06002c4d8a7a6ac22069d9a4
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:33:15 2020 -0500
Update to use `fabric2_db:get_design_docs/1`
---
src/fabric/src/fabric2_index.erl | 18 +-----------------
1 file changed, 1 insertion(+), 17 deletions(-)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 8d0affe..9a6607e 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -158,7 +158,7 @@ process_updates_iter([Db | Rest], Cont) ->
process_db(DbName) when is_binary(DbName) ->
{ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
fabric2_fdb:transactional(Db, fun(TxDb) ->
- DDocs1 = get_design_docs(TxDb),
+ DDocs1 = fabric2_db:get_design_docs(TxDb),
DDocs2 = lists:filter(fun should_update/1, DDocs1),
DDocs3 = shuffle(DDocs2),
build_indices(TxDb, DDocs3)
@@ -178,22 +178,6 @@ registrations() ->
application:get_env(fabric, indices, []).
-get_design_docs(Db) ->
- Callback = fun
- ({meta, _}, Acc) -> {ok, Acc};
- (complete, Acc) -> {ok, Acc};
- ({row, Row}, Acc) -> {ok, [get_doc(Db, Row) | Acc]}
- end,
- {ok, DDocs} = fabric2_db:fold_design_docs(Db, Callback, [], []),
- DDocs.
-
-
-get_doc(Db, Row) ->
- {_, DocId} = lists:keyfind(id, 1, Row),
- {ok, #doc{deleted = false} = Doc} = fabric2_db:open_doc(Db, DocId, []),
- Doc.
-
-
should_update(#doc{body = {Props}}) ->
couch_util:get_value(<<"autoupdate">>, Props, true).
[couchdb] 09/11: Remove jobs on index cleanup
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2e5a5566be88036ae6bb8d1f420bd6028f7c6253
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Thu Apr 9 12:57:51 2020 -0500
Remove jobs on index cleanup
---
src/couch_views/src/couch_views.erl | 1 +
src/couch_views/src/couch_views_jobs.erl | 9 ++++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/src/couch_views/src/couch_views.erl b/src/couch_views/src/couch_views.erl
index cc18364..9d518eb 100644
--- a/src/couch_views/src/couch_views.erl
+++ b/src/couch_views/src/couch_views.erl
@@ -90,6 +90,7 @@ cleanup_indices(#{} = Db, DDocs) when is_list(DDocs) ->
ExistingSigs = couch_views_fdb:list_signatures(Db),
StaleSigs = ExistingSigs -- ActiveSigs,
lists:foreach(fun(Sig) ->
+ couch_views_jobs:remove(Db, Sig),
couch_views_fdb:clear_index(Db, Sig)
end, StaleSigs).
diff --git a/src/couch_views/src/couch_views_jobs.erl b/src/couch_views/src/couch_views_jobs.erl
index b97e7ce..76cc563 100644
--- a/src/couch_views/src/couch_views_jobs.erl
+++ b/src/couch_views/src/couch_views_jobs.erl
@@ -15,7 +15,8 @@
-export([
set_timeout/0,
build_view/3,
- build_view_async/2
+ build_view_async/2,
+ remove/2
]).
-ifdef(TEST).
@@ -60,6 +61,12 @@ build_view_async(TxDb0, Mrst) ->
{ok, JobId}.
+remove(TxDb, Sig) ->
+ DbName = fabric2_db:name(TxDb),
+ JobId = job_id(DbName, Sig),
+ couch_jobs:remove(TxDb, ?INDEX_JOB_TYPE, JobId).
+
+
ensure_correct_tx(#{tx := undefined} = TxDb) ->
TxDb;
[couchdb] 10/11: Remove failed view jobs
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 30fdef77571c67c945d70fb54c07157c4643f828
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Fri Apr 10 15:07:22 2020 -0500
Remove failed view jobs
If a client notices that a job has failed we restart it. If a job failed
for a different design document id then we resubmit the build request.
---
src/couch_views/src/couch_views_jobs.erl | 26 +++++++++++++++++------
src/couch_views/test/couch_views_indexer_test.erl | 2 +-
src/couch_views/test/couch_views_map_test.erl | 2 +-
3 files changed, 22 insertions(+), 8 deletions(-)
diff --git a/src/couch_views/src/couch_views_jobs.erl b/src/couch_views/src/couch_views_jobs.erl
index 76cc563..d0de44e 100644
--- a/src/couch_views/src/couch_views_jobs.erl
+++ b/src/couch_views/src/couch_views_jobs.erl
@@ -35,7 +35,7 @@ set_timeout() ->
build_view(TxDb, Mrst, UpdateSeq) ->
{ok, JobId} = build_view_async(TxDb, Mrst),
- case wait_for_job(JobId, UpdateSeq) of
+ case wait_for_job(JobId, Mrst#mrst.idx_name, UpdateSeq) of
ok -> ok;
retry -> build_view(TxDb, Mrst, UpdateSeq)
end.
@@ -77,10 +77,10 @@ ensure_correct_tx(#{tx := Tx} = TxDb) ->
end.
-wait_for_job(JobId, UpdateSeq) ->
+wait_for_job(JobId, DDocId, UpdateSeq) ->
case couch_jobs:subscribe(?INDEX_JOB_TYPE, JobId) of
{ok, Subscription, _State, _Data} ->
- wait_for_job(JobId, Subscription, UpdateSeq);
+ wait_for_job(JobId, Subscription, DDocId, UpdateSeq);
{ok, finished, Data} ->
case Data of
#{<<"view_seq">> := ViewSeq} when ViewSeq >= UpdateSeq ->
@@ -91,21 +91,35 @@ wait_for_job(JobId, UpdateSeq) ->
end.
-wait_for_job(JobId, Subscription, UpdateSeq) ->
+wait_for_job(JobId, Subscription, DDocId, UpdateSeq) ->
case wait(Subscription) of
+ {not_found, not_found} ->
+ erlang:error(index_not_found);
{error, Error} ->
erlang:error(Error);
+ {finished, #{<<"error">> := <<"ddoc_deleted">>} = Data} ->
+ case maps:get(<<"ddoc_id">>, Data) of
+ DDocId ->
+ couch_jobs:remove(undefined, ?INDEX_JOB_TYPE, JobId),
+ erlang:error({ddoc_deleted, maps:get(<<"reason">>, Data)});
+ _OtherDocId ->
+ % A different design doc wiht the same signature
+ % was deleted. Resubmit this job which will overwrite
+ % the ddoc_id in the job.
+ retry
+ end;
{finished, #{<<"error">> := Error, <<"reason">> := Reason}} ->
+ couch_jobs:remove(undefined, ?INDEX_JOB_TYPE, JobId),
erlang:error({binary_to_existing_atom(Error, latin1), Reason});
{finished, #{<<"view_seq">> := ViewSeq}} when ViewSeq >= UpdateSeq ->
ok;
{finished, _} ->
- wait_for_job(JobId, UpdateSeq);
+ wait_for_job(JobId, DDocId, UpdateSeq);
{_State, #{<<"view_seq">> := ViewSeq}} when ViewSeq >= UpdateSeq ->
couch_jobs:unsubscribe(Subscription),
ok;
{_, _} ->
- wait_for_job(JobId, Subscription, UpdateSeq)
+ wait_for_job(JobId, Subscription, DDocId, UpdateSeq)
end.
diff --git a/src/couch_views/test/couch_views_indexer_test.erl b/src/couch_views/test/couch_views_indexer_test.erl
index 8ddb64b..54f787d 100644
--- a/src/couch_views/test/couch_views_indexer_test.erl
+++ b/src/couch_views/test/couch_views_indexer_test.erl
@@ -375,7 +375,7 @@ index_autoupdater_callback(Db) ->
?assertMatch([{ok, <<_/binary>>}], Result),
[{ok, JobId}] = Result,
- ?assertEqual(ok, couch_views_jobs:wait_for_job(JobId, DbSeq)).
+ ?assertEqual(ok, couch_views_jobs:wait_for_job(JobId, DDoc#doc.id, DbSeq)).
index_budget_is_changing(Db) ->
ok = meck:new(couch_rate, [passthrough]),
diff --git a/src/couch_views/test/couch_views_map_test.erl b/src/couch_views/test/couch_views_map_test.erl
index 7d1e94b..2b679f0 100644
--- a/src/couch_views/test/couch_views_map_test.erl
+++ b/src/couch_views/test/couch_views_map_test.erl
@@ -409,7 +409,7 @@ should_map_update_is_lazy() ->
{ok, Mrst} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
JobId = couch_views_jobs:job_id(Db, Mrst),
UpdateSeq = fabric2_db:get_update_seq(Db),
- ok = couch_views_jobs:wait_for_job(JobId, UpdateSeq),
+ ok = couch_views_jobs:wait_for_job(JobId, DDoc#doc.id, UpdateSeq),
Args2 = #{
start_key => 8,
[couchdb] 07/11: Implement _view_cleanup for FoundationDB
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4275a496dbfcec36ff0777b1cf350fffcc7756b9
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 14:52:26 2020 -0500
Implement _view_cleanup for FoundationDB
---
src/chttpd/src/chttpd_db.erl | 2 +-
src/chttpd/src/chttpd_httpd_handlers.erl | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index 730cf3e..8dd0c93 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -265,7 +265,7 @@ handle_compact_req(Req, _Db) ->
send_method_not_allowed(Req, "POST").
handle_view_cleanup_req(Req, Db) ->
- ok = fabric:cleanup_index_files_all_nodes(Db),
+ ok = fabric2_index:cleanup(Db),
send_json(Req, 202, {[{ok, true}]}).
diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl
index 3fd56c3..79ec3db 100644
--- a/src/chttpd/src/chttpd_httpd_handlers.erl
+++ b/src/chttpd/src/chttpd_httpd_handlers.erl
@@ -40,7 +40,7 @@ url_handler(<<"_session">>) -> fun chttpd_auth:handle_session_req/1;
url_handler(<<"_up">>) -> fun chttpd_misc:handle_up_req/1;
url_handler(_) -> no_match.
-db_handler(<<"_view_cleanup">>) -> fun ?MODULE:not_implemented/2;
+db_handler(<<"_view_cleanup">>) -> fun chttpd_db:handle_view_cleanup_req/2;
db_handler(<<"_compact">>) -> fun chttpd_db:handle_compact_req/2;
db_handler(<<"_design">>) -> fun chttpd_db:handle_design_req/2;
db_handler(<<"_partition">>) -> fun chttpd_db:handle_partition_req/2;
[couchdb] 03/11: Move process_db/1 to match the logical progression
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3c0a017b63b31b1e8d62707c0b909d8f24def1ba
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 16:07:43 2020 -0500
Move process_db/1 to match the logical progression
Functions are easier to read and process if they're defined in the order
that they are referenced.
Co-Authored-By: Nick Vatamaniuc <va...@apache.org>
---
src/fabric/src/fabric2_index.erl | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 098d6ed..8d0affe 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -155,6 +155,16 @@ process_updates_iter([Db | Rest], Cont) ->
process_updates_iter(Rest, Cont).
+process_db(DbName) when is_binary(DbName) ->
+ {ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ DDocs1 = get_design_docs(TxDb),
+ DDocs2 = lists:filter(fun should_update/1, DDocs1),
+ DDocs3 = shuffle(DDocs2),
+ build_indices(TxDb, DDocs3)
+ end).
+
+
build_indices(_TxDb, []) ->
[];
@@ -168,16 +178,6 @@ registrations() ->
application:get_env(fabric, indices, []).
-process_db(DbName) when is_binary(DbName) ->
- {ok, Db} = fabric2_db:open(DbName, [?ADMIN_CTX]),
- fabric2_fdb:transactional(Db, fun(TxDb) ->
- DDocs1 = get_design_docs(TxDb),
- DDocs2 = lists:filter(fun should_update/1, DDocs1),
- DDocs3 = shuffle(DDocs2),
- build_indices(TxDb, DDocs3)
- end).
-
-
get_design_docs(Db) ->
Callback = fun
({meta, _}, Acc) -> {ok, Acc};
[couchdb] 01/11: Fix index updater configuration keys
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 742c64e6a7d9a60844005a04ff9c0c22e7a6721a
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:04:07 2020 -0500
Fix index updater configuration keys
The defaults commented out in `default.ini` did not match the names used
in `fabric2_index.erl`.
---
rel/overlay/etc/default.ini | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index d2a2c72..376089a 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -228,14 +228,14 @@ port = 6984
;fdb_directory = couchdb
;
; Enable or disable index auto-updater
-;index_autoupdater_enabled = true
+;index_updater_enabled = true
;
; How long to wait from the first db update event until index building is
; triggered.
-;index_autoupdater_delay_msec = 60000
+;index_updater_delay_msec = 60000
;
; How often to check if databases may need their indices updated.
-;index_autoupdater_resolution_msec = 10000
+;index_updater_resolution_msec = 10000
; [rexi]
; buffer_count = 2000
[couchdb] 08/11: Optionally cleanup stale indices automatically
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 7aeb54bf6012c234c5806db6f438427c9cb53c4a
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:00:45 2020 -0500
Optionally cleanup stale indices automatically
---
rel/overlay/etc/default.ini | 3 +++
src/fabric/src/fabric2_index.erl | 22 +++++++++++++++++-----
2 files changed, 20 insertions(+), 5 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 376089a..e10a5a0 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -236,6 +236,9 @@ port = 6984
;
; How often to check if databases may need their indices updated.
;index_updater_resolution_msec = 10000
+;
+; Enable or disable automatic stale index removal in the auto-updater
+;index_updater_remove_old_indices = false
; [rexi]
; buffer_count = 2000
diff --git a/src/fabric/src/fabric2_index.erl b/src/fabric/src/fabric2_index.erl
index 7f9d519..25c31a8 100644
--- a/src/fabric/src/fabric2_index.erl
+++ b/src/fabric/src/fabric2_index.erl
@@ -61,10 +61,8 @@ db_updated(DbName) when is_binary(DbName) ->
cleanup(Db) ->
try
fabric2_fdb:transactional(Db, fun(TxDb) ->
- DDocs = fabric2_db:get_design_docs(TxDb),
- lists:foreach(fun(Mod) ->
- Mod:cleanup_indices(TxDb, DDocs)
- end, registrations())
+ DDocs = fabric2_db:get_design_docs(Db),
+ cleanup_indices(TxDb, DDocs)
end)
catch
error:database_does_not_exist ->
@@ -184,7 +182,11 @@ process_db(DbName) when is_binary(DbName) ->
DDocs1 = fabric2_db:get_design_docs(TxDb),
DDocs2 = lists:filter(fun should_update/1, DDocs1),
DDocs3 = shuffle(DDocs2),
- build_indices(TxDb, DDocs3)
+ build_indices(TxDb, DDocs3),
+ case auto_cleanup() of
+ true -> cleanup_indices(TxDb, DDocs1);
+ false -> ok
+ end
end).
@@ -197,6 +199,12 @@ build_indices(TxDb, DDocs) ->
end, registrations()).
+cleanup_indices(TxDb, DDocs) ->
+ lists:foreach(fun(Mod) ->
+ Mod:cleanup_indices(TxDb, DDocs)
+ end, registrations()).
+
+
registrations() ->
application:get_env(fabric, indices, []).
@@ -227,3 +235,7 @@ delay_msec() ->
resolution_msec() ->
config:get_integer("fabric", "index_updater_resolution_msec",
?DEFAULT_RESOLUTION_MSEC).
+
+
+auto_cleanup() ->
+ config:get_boolean("fabric", "index_updater_remove_old_indices", false).
[couchdb] 11/11: Implement couch_views_cleanup_test.erl
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
davisp pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 757515231ff3d2e0228a078a1771837eff0b66d0
Author: Paul J. Davis <pa...@gmail.com>
AuthorDate: Wed Mar 25 15:16:22 2020 -0500
Implement couch_views_cleanup_test.erl
Add tests for view cleanup.
---
src/couch_views/test/couch_views_cleanup_test.erl | 411 ++++++++++++++++++++++
1 file changed, 411 insertions(+)
diff --git a/src/couch_views/test/couch_views_cleanup_test.erl b/src/couch_views/test/couch_views_cleanup_test.erl
new file mode 100644
index 0000000..b5e081a
--- /dev/null
+++ b/src/couch_views/test/couch_views_cleanup_test.erl
@@ -0,0 +1,411 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_views_cleanup_test).
+
+
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch_views/include/couch_views.hrl").
+-include_lib("couch_mrview/include/couch_mrview.hrl").
+-include_lib("fabric/include/fabric2.hrl").
+-include_lib("fabric/test/fabric2_test.hrl").
+
+
+clean_old_indices_test_() ->
+ {
+ "Test cleanup of stale indices",
+ {
+ setup,
+ fun setup_all/0,
+ fun cleanup_all/1,
+ {
+ foreach,
+ fun setup/0,
+ fun cleanup/1,
+ [
+ ?TDEF_FE(empty_db),
+ ?TDEF_FE(db_with_no_ddocs),
+ ?TDEF_FE(db_with_ddoc),
+ ?TDEF_FE(db_with_many_ddocs),
+ ?TDEF_FE(after_ddoc_deletion),
+ ?TDEF_FE(all_ddocs_deleted),
+ ?TDEF_FE(after_ddoc_recreated),
+ ?TDEF_FE(refcounted_sigs),
+ ?TDEF_FE(removes_old_jobs),
+ ?TDEF_FE(after_job_accepted_initial_build),
+ ?TDEF_FE(after_job_accepted_rebuild),
+ ?TDEF_FE(during_index_initial_build),
+ ?TDEF_FE(during_index_rebuild)
+ ]
+ }
+ }
+ }.
+
+
+setup_all() ->
+ test_util:start_couch([
+ fabric,
+ couch_jobs,
+ couch_js,
+ couch_views
+ ]).
+
+
+cleanup_all(Ctx) ->
+ test_util:stop_couch(Ctx).
+
+
+setup() ->
+ Opts = [{user_ctx, ?ADMIN_USER}],
+ {ok, Db} = fabric2_db:create(?tempdb(), Opts),
+ Db.
+
+
+cleanup(Db) ->
+ meck:unload(),
+ ok = fabric2_db:delete(fabric2_db:name(Db), []).
+
+
+empty_db(Db) ->
+ ?assertEqual(ok, fabric2_index:cleanup(Db)).
+
+
+db_with_no_ddocs(Db) ->
+ create_docs(Db, 10),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)).
+
+
+db_with_ddoc(Db) ->
+ create_docs(Db, 10),
+ DDoc = create_ddoc(Db, <<"foo">>),
+ ?assertEqual(10, length(run_query(Db, DDoc))),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+ ?assertEqual(10, length(run_query(Db, DDoc))).
+
+
+db_with_many_ddocs(Db) ->
+ create_docs(Db, 10),
+ DDocs = create_ddocs(Db, 5),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)).
+
+
+after_ddoc_deletion(Db) ->
+ create_docs(Db, 10),
+ DDocs = create_ddocs(Db, 2),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs),
+ [ToDel | RestDDocs] = DDocs,
+ delete_doc(Db, ToDel),
+ % Not yet cleaned up
+ ?assertEqual(true, view_has_data(Db, ToDel)),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+ ?assertError({ddoc_deleted, _}, run_query(Db, ToDel)),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, RestDDocs).
+
+
+all_ddocs_deleted(Db) ->
+ create_docs(Db, 10),
+ DDocs = create_ddocs(Db, 5),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs),
+ lists:foreach(fun(DDoc) ->
+ delete_doc(Db, DDoc)
+ end, DDocs),
+ % Not yet cleaned up
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(true, view_has_data(Db, DDoc))
+ end, DDocs),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+ lists:foreach(fun(DDoc) ->
+ ?assertError({ddoc_deleted, _}, run_query(Db, DDoc))
+ end, DDocs).
+
+
+after_ddoc_recreated(Db) ->
+ create_docs(Db, 10),
+ DDocs = create_ddocs(Db, 3),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs),
+ [ToDel | RestDDocs] = DDocs,
+ Deleted = delete_doc(Db, ToDel),
+ % Not yet cleaned up
+ ?assertEqual(true, view_has_data(Db, ToDel)),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+ ?assertError({ddoc_deleted, _}, run_query(Db, ToDel)),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, RestDDocs),
+ recreate_doc(Db, Deleted),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+ lists:foreach(fun(DDoc) ->
+ ?assertEqual(10, length(run_query(Db, DDoc)))
+ end, DDocs).
+
+
+refcounted_sigs(Db) ->
+ create_docs(Db, 10),
+ DDoc1 = create_ddoc(Db, <<"1">>),
+ DDoc2 = create_doc(Db, <<"_design/2">>, DDoc1#doc.body),
+ ?assertEqual(10, length(run_query(Db, DDoc1))),
+ ?assertEqual(10, length(run_query(Db, DDoc2))),
+
+ ?assertEqual(true, view_has_data(Db, DDoc1)),
+ ?assertEqual(true, view_has_data(Db, DDoc2)),
+
+ delete_doc(Db, DDoc1),
+ ok = fabric2_index:cleanup(Db),
+
+ ?assertEqual(true, view_has_data(Db, DDoc1)),
+ ?assertEqual(true, view_has_data(Db, DDoc2)),
+
+ delete_doc(Db, DDoc2),
+ ok = fabric2_index:cleanup(Db),
+
+ ?assertEqual(false, view_has_data(Db, DDoc1)),
+ ?assertEqual(false, view_has_data(Db, DDoc2)).
+
+
+removes_old_jobs(Db) ->
+ create_docs(Db, 10),
+ DDoc = create_ddoc(Db, <<"foo">>),
+
+ ?assertEqual(10, length(run_query(Db, DDoc))),
+ ?assertEqual(true, view_has_data(Db, DDoc)),
+ ?assertEqual(true, job_exists(Db, DDoc)),
+
+ delete_doc(Db, DDoc),
+ ?assertEqual(ok, fabric2_index:cleanup(Db)),
+
+ ?assertEqual(false, view_has_data(Db, DDoc)),
+ ?assertEqual(false, job_exists(Db, DDoc)).
+
+
+after_job_accepted_initial_build(Db) ->
+ cleanup_during_initial_build(Db, fun meck_intercept_job_accept/2).
+
+
+after_job_accepted_rebuild(Db) ->
+ cleanup_during_rebuild(Db, fun meck_intercept_job_accept/2).
+
+
+during_index_initial_build(Db) ->
+ cleanup_during_initial_build(Db, fun meck_intercept_job_update/2).
+
+
+during_index_rebuild(Db) ->
+ cleanup_during_rebuild(Db, fun meck_intercept_job_update/2).
+
+
+cleanup_during_initial_build(Db, InterruptFun) ->
+ InterruptFun(fabric2_db:name(Db), self()),
+
+ create_docs(Db, 10),
+ DDoc = create_ddoc(Db, <<"foo">>),
+
+ {_, Ref1} = spawn_monitor(fun() -> run_query(Db, DDoc) end),
+
+ receive {JobPid, triggered} -> ok end,
+ delete_doc(Db, DDoc),
+ ok = fabric2_index:cleanup(Db),
+ JobPid ! continue,
+
+ receive {'DOWN', Ref1, _, _, _} -> ok end,
+
+ ok = fabric2_index:cleanup(Db),
+ ?assertError({ddoc_deleted, _}, run_query(Db, DDoc)),
+
+ ?assertEqual(false, view_has_data(Db, DDoc)),
+ ?assertEqual(false, job_exists(Db, DDoc)).
+
+
+cleanup_during_rebuild(Db, InterruptFun) ->
+ create_docs(Db, 10),
+ DDoc = create_ddoc(Db, <<"foo">>),
+ ?assertEqual(10, length(run_query(Db, DDoc))),
+
+ InterruptFun(fabric2_db:name(Db), self()),
+
+ create_docs(Db, 10, 10),
+
+ {_, Ref1} = spawn_monitor(fun() -> run_query(Db, DDoc) end),
+
+ receive {JobPid, triggered} -> ok end,
+ delete_doc(Db, DDoc),
+ ok = fabric2_index:cleanup(Db),
+ JobPid ! continue,
+
+ receive {'DOWN', Ref1, _, _, _} -> ok end,
+
+ ok = fabric2_index:cleanup(Db),
+ ?assertError({ddoc_deleted, _}, run_query(Db, DDoc)),
+
+ ?assertEqual(false, view_has_data(Db, DDoc)),
+ ?assertEqual(false, job_exists(Db, DDoc)).
+
+
+
+run_query(Db, DDocId) when is_binary(DDocId) ->
+ {ok, DDoc} = fabric2_db:open_doc(Db, <<"_design/", DDocId/binary>>),
+ run_query(Db, DDoc);
+
+run_query(Db, DDoc) ->
+ Fun = fun default_cb/2,
+ {ok, Result} = couch_views:query(Db, DDoc, <<"bar">>, Fun, [], #{}),
+ Result.
+
+
+default_cb(complete, Acc) ->
+ {ok, lists:reverse(Acc)};
+default_cb({final, Info}, []) ->
+ {ok, [Info]};
+default_cb({final, _}, Acc) ->
+ {ok, Acc};
+default_cb({meta, _}, Acc) ->
+ {ok, Acc};
+default_cb(ok, ddoc_updated) ->
+ {ok, ddoc_updated};
+default_cb(Row, Acc) ->
+ {ok, [Row | Acc]}.
+
+
+view_has_data(Db, DDoc) ->
+ DbName = fabric2_db:name(Db),
+ {ok, #mrst{sig = Sig}} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+ fabric2_fdb:transactional(Db, fun(TxDb) ->
+ #{
+ tx := Tx,
+ db_prefix := DbPrefix
+ } = TxDb,
+ SigKeyTuple = {?DB_VIEWS, ?VIEW_INFO, ?VIEW_UPDATE_SEQ, Sig},
+ SigKey = erlfdb_tuple:pack(SigKeyTuple, DbPrefix),
+ SigVal = erlfdb:wait(erlfdb:get(Tx, SigKey)),
+
+ RangeKeyTuple = {?DB_VIEWS, ?VIEW_DATA, Sig},
+ RangeKey = erlfdb_tuple:pack(RangeKeyTuple, DbPrefix),
+ Range = erlfdb:wait(erlfdb:get_range_startswith(Tx, RangeKey)),
+
+ SigVal /= not_found andalso Range /= []
+ end).
+
+
+meck_intercept_job_accept(TgtDbName, ParentPid) ->
+ meck:new(fabric2_db, [passthrough]),
+ meck:expect(fabric2_db, open, fun
+ (DbName, Opts) when DbName == TgtDbName ->
+ Result = meck:passthrough([DbName, Opts]),
+ ParentPid ! {self(), triggered},
+ receive continue -> ok end,
+ meck:unload(),
+ Result;
+ (DbName, Opts) ->
+ meck:passthrough([DbName, Opts])
+ end).
+
+
+meck_intercept_job_update(_DbName, ParentPid) ->
+ meck:new(couch_jobs, [passthrough]),
+ meck:expect(couch_jobs, finish, fun(Tx, Job, Data) ->
+ ParentPid ! {self(), triggered},
+ receive continue -> ok end,
+ Result = meck:passthrough([Tx, Job, Data]),
+ meck:unload(),
+ Result
+ end).
+
+
+create_ddoc(Db, Id) ->
+ MapFunFmt = "function(doc) {var f = \"~s\"; emit(doc.val, f)}",
+ MapFun = io_lib:format(MapFunFmt, [Id]),
+ Body = {[
+ {<<"views">>, {[
+ {<<"bar">>, {[{<<"map">>, iolist_to_binary(MapFun)}]}}
+ ]}}
+ ]},
+ create_doc(Db, <<"_design/", Id/binary>>, Body).
+
+
+recreate_doc(Db, #doc{deleted = true} = Doc) ->
+ #doc{
+ id = DDocId,
+ body = Body
+ } = Doc,
+ create_doc(Db, DDocId, Body).
+
+
+create_ddocs(Db, Count) when is_integer(Count), Count > 1 ->
+ lists:map(fun(Seq) ->
+ Id = io_lib:format("~6..0b", [Seq]),
+ create_ddoc(Db, iolist_to_binary(Id))
+ end, lists:seq(1, Count)).
+
+
+create_doc(Db, Id) ->
+ create_doc(Db, Id, {[{<<"value">>, Id}]}).
+
+
+create_doc(Db, Id, Body) ->
+ Doc = #doc{
+ id = Id,
+ body = Body
+ },
+ {ok, {Pos, Rev}} = fabric2_db:update_doc(Db, Doc),
+ Doc#doc{revs = {Pos, [Rev]}}.
+
+
+create_docs(Db, Count) ->
+ create_docs(Db, Count, 0).
+
+
+create_docs(Db, Count, Offset) ->
+ lists:map(fun(Seq) ->
+ Id = io_lib:format("~6..0b", [Seq]),
+ create_doc(Db, iolist_to_binary(Id))
+ end, lists:seq(Offset + 1, Offset + Count)).
+
+
+delete_doc(Db, DDoc) ->
+ #doc{
+ revs = {_, [_ | _] = Revs}
+ } = DDoc,
+ {ok, {NewPos, Rev}} = fabric2_db:update_doc(Db, DDoc#doc{deleted = true}),
+ DDoc#doc{
+ revs = {NewPos, [Rev | Revs]},
+ deleted = true
+ }.
+
+
+job_exists(Db, DDoc) ->
+ JobId = job_id(Db, DDoc),
+ case couch_jobs:get_job_data(Db, ?INDEX_JOB_TYPE, JobId) of
+ {ok, _} -> true;
+ {error, not_found} -> false
+ end.
+
+
+job_id(Db, DDoc) ->
+ DbName = fabric2_db:name(Db),
+ {ok, #mrst{sig = Sig}} = couch_views_util:ddoc_to_mrst(DbName, DDoc),
+ HexSig = fabric2_util:to_hex(Sig),
+ <<DbName/binary, "-", HexSig/binary>>.