You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2022/07/08 14:35:15 UTC
[couchdb] branch main updated: Implement winning_revs_only option for the replicator
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/main by this push:
new 6a455c74b Implement winning_revs_only option for the replicator
6a455c74b is described below
commit 6a455c74bd6426847698b7543322907be64b2ac0
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Jul 8 02:06:59 2022 -0400
Implement winning_revs_only option for the replicator
`winning_revs_only` option replicates only the winning revisions from the
source to the target, effectively discarding conflicts.
This option may be used as an alternative to the custom VDU strategy, or manual
_all_docs read and POST to target scripts, often used by users to discard
conflicted document revisions.
Behind the scenes the implementation simply switches the _changes feed style to
`main_only` from the replication default of `all_docs`.
Additionally, when enabled, the `winning_revs_only` option, will generate a
different replication ID and use a different set of checkpoints than the
default case. This should allows first replicating with `winning_revs_only:
true`, and then later replicate normally between the same endpoints, in
order to "backfill" the rest of the revisions.
Issue: https://github.com/apache/couchdb/issues/4034
---
.../src/couch_replicator_changes_reader.erl | 8 +-
src/couch_replicator/src/couch_replicator_docs.erl | 12 ++
src/couch_replicator/src/couch_replicator_ids.erl | 48 +++++++-
.../src/couch_replicator_js_functions.hrl | 6 +
.../eunit/couch_replicator_many_leaves_tests.erl | 132 +++++++++++++--------
5 files changed, 151 insertions(+), 55 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl b/src/couch_replicator/src/couch_replicator_changes_reader.erl
index 83080b6fb..9f8473200 100644
--- a/src/couch_replicator/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl
@@ -20,7 +20,6 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
--include("couch_replicator.hrl").
-import(couch_util, [
get_value/2
@@ -49,10 +48,15 @@ start_link(StartSeq, Db, ChangesQueue, Options) ->
read_changes(Parent, StartSeq, Db, ChangesQueue, Options) ->
Continuous = couch_util:get_value(continuous, Options),
+ Style =
+ case couch_util:get_value(winning_revs_only, Options, false) of
+ true -> main_only;
+ false -> all_docs
+ end,
try
couch_replicator_api_wrap:changes_since(
Db,
- all_docs,
+ Style,
StartSeq,
fun(Item) ->
process_change(Item, {Parent, Db, ChangesQueue, Continuous})
diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl
index 325f3eb65..270ec23be 100644
--- a/src/couch_replicator/src/couch_replicator_docs.erl
+++ b/src/couch_replicator/src/couch_replicator_docs.erl
@@ -510,6 +510,10 @@ convert_options([{<<"create_target_params">>, V} | _R]) when not is_tuple(V) ->
throw({bad_request, <<"parameter `create_target_params` must be a JSON object">>});
convert_options([{<<"create_target_params">>, V} | R]) ->
[{create_target_params, V} | convert_options(R)];
+convert_options([{<<"winning_revs_only">>, V} | _R]) when not is_boolean(V) ->
+ throw({bad_request, <<"parameter `winning_revs_only` must be a boolean">>});
+convert_options([{<<"winning_revs_only">>, V} | R]) ->
+ [{winning_revs_only, V} | convert_options(R)];
convert_options([{<<"continuous">>, V} | _R]) when not is_boolean(V) ->
throw({bad_request, <<"parameter `continuous` must be a boolean">>});
convert_options([{<<"continuous">>, V} | R]) ->
@@ -793,6 +797,10 @@ check_convert_options_pass_test() ->
[{create_target, true}],
convert_options([{<<"create_target">>, true}])
),
+ ?assertEqual(
+ [{winning_revs_only, true}],
+ convert_options([{<<"winning_revs_only">>, true}])
+ ),
?assertEqual(
[{continuous, true}],
convert_options([{<<"continuous">>, true}])
@@ -815,6 +823,10 @@ check_convert_options_fail_test() ->
{bad_request, _},
convert_options([{<<"create_target">>, <<"true">>}])
),
+ ?assertThrow(
+ {bad_request, _},
+ convert_options([{<<"winning_revs_only">>, <<"foo">>}])
+ ),
?assertThrow(
{bad_request, _},
convert_options([{<<"continuous">>, <<"true">>}])
diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl
index 6543cf069..9f4581afc 100644
--- a/src/couch_replicator/src/couch_replicator_ids.erl
+++ b/src/couch_replicator/src/couch_replicator_ids.erl
@@ -110,7 +110,17 @@ maybe_append_filters(
{error, FilterParseError} ->
throw({error, FilterParseError})
end,
- couch_util:to_hex(couch_hash:md5_hash(term_to_binary(Base2))).
+ Base3 =
+ Base2 ++
+ case couch_util:get_value(winning_revs_only, Options) of
+ undefined ->
+ [];
+ false ->
+ [];
+ true ->
+ [<<"winning_revs_only">>]
+ end,
+ couch_util:to_hex(couch_hash:md5_hash(term_to_binary(Base3))).
maybe_append_options(Options, RepOptions) ->
lists:foldl(
@@ -169,6 +179,42 @@ get_non_default_port(_Schema, Port) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch_replicator/test/eunit/couch_replicator_test.hrl").
+
+winning_revs_id_test_() ->
+ {
+ foreach,
+ fun test_util:start_couch/0,
+ fun test_util:stop_couch/1,
+ [
+ ?TDEF_FE(winning_revs_generates_new_id),
+ ?TDEF_FE(winning_revs_false_same_as_undefined)
+ ]
+ }.
+
+winning_revs_generates_new_id(_) ->
+ RepDoc1 = [
+ {<<"source">>, <<"http://foo.example.bar">>},
+ {<<"target">>, <<"http://bar.example.foo">>}
+ ],
+ Rep1 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc1}),
+ Id1 = replication_id(Rep1),
+ RepDoc2 = RepDoc1 ++ [{<<"winning_revs_only">>, true}],
+ Rep2 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc2}),
+ Id2 = replication_id(Rep2),
+ ?assertNotEqual(Id1, Id2).
+
+winning_revs_false_same_as_undefined(_) ->
+ RepDoc1 = [
+ {<<"source">>, <<"http://foo.example.bar">>},
+ {<<"target">>, <<"http://bar.example.foo">>}
+ ],
+ Rep1 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc1}),
+ Id1 = replication_id(Rep1),
+ RepDoc2 = RepDoc1 ++ [{<<"winning_revs_only">>, false}],
+ Rep2 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc2}),
+ Id2 = replication_id(Rep2),
+ ?assertEqual(Id1, Id2).
replication_id_convert_test_() ->
[
diff --git a/src/couch_replicator/src/couch_replicator_js_functions.hrl b/src/couch_replicator/src/couch_replicator_js_functions.hrl
index d41043309..4f4369075 100644
--- a/src/couch_replicator/src/couch_replicator_js_functions.hrl
+++ b/src/couch_replicator/src/couch_replicator_js_functions.hrl
@@ -76,6 +76,12 @@
reportError('The `create_target\\' field must be a boolean.');
}
+ if ((typeof newDoc.winning_revs_only !== 'undefined') &&
+ (typeof newDoc.winning_revs_only !== 'boolean')) {
+
+ reportError('The `winning_revs_only\\' field must be a boolean.');
+ }
+
if ((typeof newDoc.continuous !== 'undefined') &&
(typeof newDoc.continuous !== 'boolean')) {
diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
index 86daa808f..95ccfaa07 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
@@ -17,7 +17,7 @@
-import(couch_replicator_test_helper, [
db_url/1,
- replicate/2
+ replicate/1
]).
-define(DOCS_CONFLICTS, [
@@ -32,88 +32,100 @@
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-setup() ->
+setup_db() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
ok = couch_db:close(Db),
DbName.
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
+teardown_db(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+
+setup() ->
Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
+ Source = setup_db(),
+ Target = setup_db(),
{Ctx, {Source, Target}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
+teardown({Ctx, {Source, Target}}) ->
+ teardown_db(Source),
+ teardown_db(Target),
ok = test_util:stop_couch(Ctx).
docs_with_many_leaves_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate documents with many leaves",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup/0,
+ fun teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ fun should_populate_replicate_compact/1
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
+docs_with_many_leaves_test_winning_revs_only_test_() ->
{
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target)
- ]}
+ "Replicate winning revs only for documents with many leaves",
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun should_replicate_winning_revs_only/1
+ ]
+ }
}.
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
+should_populate_replicate_compact({_Ctx, {Source, Target}}) ->
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target, all_revs),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target, all_revs)
+ ]}.
+
+should_replicate_winning_revs_only({_Ctx, {Source, Target}}) ->
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ should_verify_target(Source, Target, winning_revs),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ should_verify_target(Source, Target, winning_revs)
+ ]}.
+
should_populate_source(Source) ->
{timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+ should_replicate(Source, Target, []).
+
+should_replicate(Source, Target, Options) ->
+ {timeout, ?TIMEOUT_EUNIT,
+ ?_test(begin
+ RepObj = {
+ [
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)}
+ ] ++ Options
+ },
+ replicate(RepObj)
+ end)}.
-should_verify_target({remote, Source}, Target) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, {remote, Target}) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, Target) ->
+should_verify_target(Source, Target, Mode) ->
{timeout, ?TIMEOUT_EUNIT,
?_test(begin
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+ verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS, Mode),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb)
end)}.
-should_add_attachments_to_source({remote, Source}) ->
- should_add_attachments_to_source(Source);
should_add_attachments_to_source(Source) ->
{timeout, ?TIMEOUT_EUNIT,
?_test(begin
@@ -131,7 +143,11 @@ populate_db(DbName) ->
id = DocId,
body = {[{<<"value">>, Value}]}
},
- {ok, _} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ {ok, {Pos, Rev}} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ % Update first initial doc rev twice to ensure it's always a winner
+ {ok, Db2} = couch_db:reopen(Db),
+ Doc1 = Doc#doc{revs = {Pos, [Rev]}},
+ {ok, _} = couch_db:update_doc(Db2, Doc1, [?ADMIN_CTX]),
{ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
end,
?DOCS_CONFLICTS
@@ -160,9 +176,9 @@ add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
[{1, Rev} | AccRevs]
).
-verify_target(_SourceDb, _TargetDb, []) ->
+verify_target(_SourceDb, _TargetDb, [], _Mode) ->
ok;
-verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
{ok, SourceLookups} = couch_db:open_doc_revs(
SourceDb,
DocId,
@@ -187,7 +203,19 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
end,
lists:zip(SourceDocs, TargetDocs)
),
- verify_target(SourceDb, TargetDb, Rest).
+ verify_target(SourceDb, TargetDb, Rest, all_revs);
+verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs) ->
+ {ok, SourceWinner} = couch_db:open_doc(SourceDb, DocId),
+ {ok, TargetWinner} = couch_db:open_doc(TargetDb, DocId),
+ SourceWinnerJson = couch_doc:to_json_obj(SourceWinner, [attachments]),
+ TargetWinnerJson = couch_doc:to_json_obj(TargetWinner, [attachments]),
+ % Source winner is the same as the target winner
+ ?assertEqual(SourceWinnerJson, TargetWinnerJson),
+ Opts = [conflicts, deleted_conflicts],
+ {ok, TargetAll} = couch_db:open_doc_revs(TargetDb, DocId, all, Opts),
+ % There is only one version on the target
+ ?assert(length(TargetAll) == 1),
+ verify_target(SourceDb, TargetDb, Rest, winning_revs).
add_attachments(_SourceDb, _NumAtts, []) ->
ok;