You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2024/12/11 10:29:35 UTC
(couchdb) branch auto-delete-2 updated (cdf2d77ef -> c455b32a0)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch auto-delete-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
from cdf2d77ef first stab at including mrview checkpoints
new 331baf291 wip
new c455b32a0 wip
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
src/couch_mrview/src/couch_mrview_cleanup.erl | 12 +++++------
src/couch_mrview/src/couch_mrview_index.erl | 29 ++++++++++++++++++---------
src/couch_mrview/src/couch_mrview_util.erl | 25 ++++++++++++++++-------
src/fabric/src/fabric.erl | 17 ++++++++--------
4 files changed, 52 insertions(+), 31 deletions(-)
(couchdb) 02/02: wip
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch auto-delete-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c455b32a0b68bcb2b2482672fd23a36466ce3a6f
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Dec 11 10:29:26 2024 +0000
wip
---
src/couch_mrview/src/couch_mrview_index.erl | 29 +++++++++++++++++++----------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview_index.erl b/src/couch_mrview/src/couch_mrview_index.erl
index 63d686cd9..85d7ce316 100644
--- a/src/couch_mrview/src/couch_mrview_index.erl
+++ b/src/couch_mrview/src/couch_mrview_index.erl
@@ -129,12 +129,14 @@ open(Db, State0) ->
NewSt = init_and_upgrade_state(Db, Fd, State, Header),
ok = commit(NewSt),
ensure_local_purge_doc(Db, NewSt),
+ write_peer_checkpoint(Db, NewSt),
{ok, NewSt};
% end of upgrade code for <= 2.x
{ok, {Sig, Header}} ->
% Matching view signatures.
NewSt = init_and_upgrade_state(Db, Fd, State, Header),
ensure_local_purge_doc(Db, NewSt),
+ write_peer_checkpoint(Db, NewSt),
check_collator_versions(DbName, NewSt),
{ok, NewSt};
{ok, {WrongSig, _}} ->
@@ -144,6 +146,7 @@ open(Db, State0) ->
),
NewSt = couch_mrview_util:reset_index(Db, Fd, State),
ensure_local_purge_doc(Db, NewSt),
+ write_peer_checkpoint(Db, NewSt),
{ok, NewSt};
{ok, Else} ->
couch_log:error(
@@ -152,10 +155,12 @@ open(Db, State0) ->
),
NewSt = couch_mrview_util:reset_index(Db, Fd, State),
ensure_local_purge_doc(Db, NewSt),
+ write_peer_checkpoint(Db, NewSt),
{ok, NewSt};
no_valid_header ->
NewSt = couch_mrview_util:reset_index(Db, Fd, State),
ensure_local_purge_doc(Db, NewSt),
+ write_peer_checkpoint(Db, NewSt),
{ok, NewSt}
end;
{error, Reason} = Error ->
@@ -332,6 +337,7 @@ update_local_purge_doc(Db, State, PSeq) ->
end,
couch_db:update_doc(Db, Doc, []).
+
init_and_upgrade_state(Db, Fd, State, Header) ->
{Commit, #mrst{} = Mrst} = couch_mrview_util:init_state(Db, Fd, State, Header),
case Commit of
@@ -369,6 +375,11 @@ check_collator_versions(DbName, #mrst{} = Mrst) ->
end.
write_peer_checkpoint(#mrst{} = State) ->
+ couch_util:with_db(State#mrst.db_name, fun(Db) ->
+ write_peer_checkpoint(Db, State)
+ end).
+
+write_peer_checkpoint(Db, #mrst{} = State) ->
Sig = ?l2b(couch_index_util:hexsig(State#mrst.sig)),
DocId = <<?LOCAL_DOC_PREFIX, "mrview-", Sig/binary>>,
BaseDoc = couch_doc:from_json_obj(
@@ -380,16 +391,14 @@ write_peer_checkpoint(#mrst{} = State) ->
{<<"seq">>, State#mrst.update_seq}
]}
),
- couch_util:with_db(State#mrst.db_name, fun(Db) ->
- Doc =
- case couch_db:open_doc(Db, DocId, []) of
- {ok, #doc{revs = Revs}} ->
- BaseDoc#doc{revs = Revs};
- {not_found, _} ->
- BaseDoc
- end,
- couch_db:update_doc(Db, Doc, [])
- end).
+ Doc =
+ case couch_db:open_doc(Db, DocId, []) of
+ {ok, #doc{revs = Revs}} ->
+ BaseDoc#doc{revs = Revs};
+ {not_found, _} ->
+ BaseDoc
+ end,
+ couch_db:update_doc(Db, Doc, []).
range(<<"shards/", _/binary>> = DbName) ->
mem3:range(DbName);
(couchdb) 01/02: wip
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch auto-delete-2
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 331baf291ab5a1bdfb3957d66c297826f1de7068
Author: Robert Newson <rn...@apache.org>
AuthorDate: Wed Dec 11 10:21:55 2024 +0000
wip
---
src/couch_mrview/src/couch_mrview_cleanup.erl | 12 ++++++------
src/couch_mrview/src/couch_mrview_util.erl | 25 ++++++++++++++++++-------
src/fabric/src/fabric.erl | 17 +++++++++--------
3 files changed, 33 insertions(+), 21 deletions(-)
diff --git a/src/couch_mrview/src/couch_mrview_cleanup.erl b/src/couch_mrview/src/couch_mrview_cleanup.erl
index 5b5afbdce..f1b671dd5 100644
--- a/src/couch_mrview/src/couch_mrview_cleanup.erl
+++ b/src/couch_mrview/src/couch_mrview_cleanup.erl
@@ -14,7 +14,7 @@
-export([
run/1,
- cleanup_purges/3,
+ cleanup_checkpoints/3,
cleanup_indices/2
]).
@@ -25,14 +25,14 @@ run(Db) ->
Checkpoints = couch_mrview_util:get_purge_checkpoints(Db),
{ok, Db1} = couch_db:reopen(Db),
Sigs = couch_mrview_util:get_signatures(Db1),
- ok = cleanup_purges(Db1, Sigs, Checkpoints),
+ ok = cleanup_checkpoints(Db1, Sigs, Checkpoints),
ok = cleanup_indices(Sigs, Indices).
-cleanup_purges(DbName, Sigs, Checkpoints) when is_binary(DbName) ->
+cleanup_checkpoints(DbName, Sigs, Checkpoints) when is_binary(DbName) ->
couch_util:with_db(DbName, fun(Db) ->
- cleanup_purges(Db, Sigs, Checkpoints)
+ cleanup_checkpoints(Db, Sigs, Checkpoints)
end);
-cleanup_purges(Db, #{} = Sigs, #{} = CheckpointsMap) ->
+cleanup_checkpoints(Db, #{} = Sigs, #{} = CheckpointsMap) ->
InactiveMap = maps:without(maps:keys(Sigs), CheckpointsMap),
InactiveCheckpoints = maps:values(InactiveMap),
DeleteFun = fun(DocId) -> delete_checkpoint(Db, DocId) end,
@@ -57,7 +57,7 @@ delete_file(File) ->
delete_checkpoint(Db, DocId) ->
DbName = couch_db:name(Db),
- LogMsg = "~p : deleting inactive purge checkpoint ~s : ~s",
+ LogMsg = "~p : deleting inactive checkpoint ~s : ~s",
couch_log:debug(LogMsg, [?MODULE, DbName, DocId]),
try couch_db:open_doc(Db, DocId, []) of
{ok, Doc = #doc{}} ->
diff --git a/src/couch_mrview/src/couch_mrview_util.erl b/src/couch_mrview/src/couch_mrview_util.erl
index a478685da..c27150453 100644
--- a/src/couch_mrview/src/couch_mrview_util.erl
+++ b/src/couch_mrview/src/couch_mrview_util.erl
@@ -36,6 +36,7 @@
-export([get_collator_versions/1]).
-export([compact_on_collator_upgrade/0]).
-export([commit_on_header_upgrade/0]).
+-export([get_mrview_checkpoints/1]).
-define(MOD, couch_mrview_index).
-define(GET_VIEW_RETRY_COUNT, 1).
@@ -114,18 +115,28 @@ get_signatures(Db) ->
% Returns a map of `Sig => DocId` elements for all the purge view
% checkpoint docs. Sig is a hex-encoded binary.
%
-get_purge_checkpoints(DbName) when is_binary(DbName) ->
- couch_util:with_db(DbName, fun get_purge_checkpoints/1);
-get_purge_checkpoints(Db) ->
+get_purge_checkpoints(DbOrName) ->
+ get_registered_peers(DbOrName, <<?LOCAL_DOC_PREFIX, "purge-mrview-">>).
+
+% Returns a map of `Sig => DocId` elements for all the mrview
+% checkpoint docs. Sig is a hex-encoded binary.
+%
+get_mrview_checkpoints(DbOrName) ->
+ get_registered_peers(DbOrName, <<?LOCAL_DOC_PREFIX, "mrview-">>).
+
+get_registered_peers(DbName, Prefix) when is_binary(DbName), is_binary(Prefix) ->
+ couch_util:with_db(DbName, fun(Db) -> get_registered_peers(Db, Prefix) end);
+get_registered_peers(Db, Prefix) ->
FoldFun = fun(#doc{id = Id}, Acc) ->
- case Id of
- <<?LOCAL_DOC_PREFIX, "purge-mrview-", Sig/binary>> ->
- {ok, Acc#{Sig => Id}};
+ case binary:match(Id, Prefix) of
+ {0, Len} ->
+ Sig = binary:part(Id, byte_size(Id) - Len),
+ {ok, Acc#{Sig => Id}};
_ ->
{stop, Acc}
end
end,
- Opts = [{start_key, <<?LOCAL_DOC_PREFIX, "purge-mrview-">>}],
+ Opts = [{start_key, Prefix}],
{ok, Signatures = #{}} = couch_db:fold_local_docs(Db, FoldFun, #{}, Opts),
Signatures.
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 343b12563..c717e0040 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -574,10 +574,10 @@ reset_validation_funs(DbName) ->
].
%% @doc clean up index files for all Dbs
--spec cleanup_index_files() -> [ok].
+-spec cleanup_index_files() -> ok.
cleanup_index_files() ->
{ok, Dbs} = fabric:all_dbs(),
- [cleanup_index_files(Db) || Db <- Dbs].
+ lists:foreach(fun cleanup_index_files/1, Dbs).
%% @doc clean up index files for a specific db
-spec cleanup_index_files(dbname()) -> ok.
@@ -595,15 +595,16 @@ cleanup_local_indices_and_purge_checkpoints([]) ->
cleanup_local_indices_and_purge_checkpoints([_ | _] = Dbs) ->
AllIndices = lists:map(fun couch_mrview_util:get_index_files/1, Dbs),
AllPurges = lists:map(fun couch_mrview_util:get_purge_checkpoints/1, Dbs),
+ AllCheckpoints = lists:map(fun couch_mrview_util:get_mrview_checkpoints/1, Dbs),
Sigs = couch_mrview_util:get_signatures(hd(Dbs)),
- ok = cleanup_purges(Sigs, AllPurges, Dbs),
+ ok = cleanup_checkpoints(Sigs, AllPurges ++ AllCheckpoints, Dbs),
ok = cleanup_indices(Sigs, AllIndices).
-cleanup_purges(Sigs, AllPurges, Dbs) ->
- Fun = fun(DbPurges, Db) ->
- couch_mrview_cleanup:cleanup_purges(Db, Sigs, DbPurges)
+cleanup_checkpoints(Sigs, AllCheckpoints, Dbs) ->
+ Fun = fun(Checkpoints, Db) ->
+ couch_mrview_cleanup:cleanup_checkpoints(Db, Sigs, Checkpoints)
end,
- lists:zipwith(Fun, AllPurges, Dbs),
+ lists:zipwith(Fun, AllCheckpoints, Dbs),
ok.
cleanup_indices(Sigs, AllIndices) ->
@@ -613,7 +614,7 @@ cleanup_indices(Sigs, AllIndices) ->
lists:foreach(Fun, AllIndices).
%% @doc clean up index files for a specific db on all nodes
--spec cleanup_index_files_all_nodes(dbname()) -> [reference()].
+-spec cleanup_index_files_all_nodes(dbname()) -> ok.
cleanup_index_files_all_nodes(DbName) ->
lists:foreach(
fun(Node) ->