You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2022/07/08 13:58:47 UTC
[couchdb] branch replicate-winning-rev created (now f5482df77)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch replicate-winning-rev
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at f5482df77 Implement winning_revs_only option for the replicator
This branch includes the following new commits:
new f5482df77 Implement winning_revs_only option for the replicator
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Implement winning_revs_only option for the replicator
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch replicate-winning-rev
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit f5482df775386c8b85b4f78842cba3168c811689
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Fri Jul 8 02:06:59 2022 -0400
Implement winning_revs_only option for the replicator
`winning_revs_only` option replicates only the winning revisions from the
source to the target, effectively discarding conflicts.
This option may be used as an alternative to the custom VDU strategy, or manual
_all_docs read and POST to target scripts, often used by users to discard
conflicted document revisions.
Behind the scenes the implementation simply switches the _changes feed style to
`main_only` from the replication default of `all_docs`.
Additionally, when enabled, the `winning_revs_only` option, will generate a
different replication ID and use a different set of checkpoints than the
default case. This should allows first replicating with `winning_revs_only:
true`, and then later replicate normally between the same endpoints, in
order to "backfill" the rest of the revisions.
Issue: https://github.com/apache/couchdb/issues/4034
---
.../src/couch_replicator_changes_reader.erl | 8 +-
src/couch_replicator/src/couch_replicator_docs.erl | 12 ++
src/couch_replicator/src/couch_replicator_ids.erl | 48 +++++++-
.../src/couch_replicator_js_functions.hrl | 6 +
.../eunit/couch_replicator_many_leaves_tests.erl | 132 +++++++++++++--------
5 files changed, 151 insertions(+), 55 deletions(-)
diff --git a/src/couch_replicator/src/couch_replicator_changes_reader.erl b/src/couch_replicator/src/couch_replicator_changes_reader.erl
index 83080b6fb..9f8473200 100644
--- a/src/couch_replicator/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator/src/couch_replicator_changes_reader.erl
@@ -20,7 +20,6 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
--include("couch_replicator.hrl").
-import(couch_util, [
get_value/2
@@ -49,10 +48,15 @@ start_link(StartSeq, Db, ChangesQueue, Options) ->
read_changes(Parent, StartSeq, Db, ChangesQueue, Options) ->
Continuous = couch_util:get_value(continuous, Options),
+ Style =
+ case couch_util:get_value(winning_revs_only, Options, false) of
+ true -> main_only;
+ false -> all_docs
+ end,
try
couch_replicator_api_wrap:changes_since(
Db,
- all_docs,
+ Style,
StartSeq,
fun(Item) ->
process_change(Item, {Parent, Db, ChangesQueue, Continuous})
diff --git a/src/couch_replicator/src/couch_replicator_docs.erl b/src/couch_replicator/src/couch_replicator_docs.erl
index 325f3eb65..270ec23be 100644
--- a/src/couch_replicator/src/couch_replicator_docs.erl
+++ b/src/couch_replicator/src/couch_replicator_docs.erl
@@ -510,6 +510,10 @@ convert_options([{<<"create_target_params">>, V} | _R]) when not is_tuple(V) ->
throw({bad_request, <<"parameter `create_target_params` must be a JSON object">>});
convert_options([{<<"create_target_params">>, V} | R]) ->
[{create_target_params, V} | convert_options(R)];
+convert_options([{<<"winning_revs_only">>, V} | _R]) when not is_boolean(V) ->
+ throw({bad_request, <<"parameter `winning_revs_only` must be a boolean">>});
+convert_options([{<<"winning_revs_only">>, V} | R]) ->
+ [{winning_revs_only, V} | convert_options(R)];
convert_options([{<<"continuous">>, V} | _R]) when not is_boolean(V) ->
throw({bad_request, <<"parameter `continuous` must be a boolean">>});
convert_options([{<<"continuous">>, V} | R]) ->
@@ -793,6 +797,10 @@ check_convert_options_pass_test() ->
[{create_target, true}],
convert_options([{<<"create_target">>, true}])
),
+ ?assertEqual(
+ [{winning_revs_only, true}],
+ convert_options([{<<"winning_revs_only">>, true}])
+ ),
?assertEqual(
[{continuous, true}],
convert_options([{<<"continuous">>, true}])
@@ -815,6 +823,10 @@ check_convert_options_fail_test() ->
{bad_request, _},
convert_options([{<<"create_target">>, <<"true">>}])
),
+ ?assertThrow(
+ {bad_request, _},
+ convert_options([{<<"winning_revs_only">>, <<"foo">>}])
+ ),
?assertThrow(
{bad_request, _},
convert_options([{<<"continuous">>, <<"true">>}])
diff --git a/src/couch_replicator/src/couch_replicator_ids.erl b/src/couch_replicator/src/couch_replicator_ids.erl
index 6543cf069..9f4581afc 100644
--- a/src/couch_replicator/src/couch_replicator_ids.erl
+++ b/src/couch_replicator/src/couch_replicator_ids.erl
@@ -110,7 +110,17 @@ maybe_append_filters(
{error, FilterParseError} ->
throw({error, FilterParseError})
end,
- couch_util:to_hex(couch_hash:md5_hash(term_to_binary(Base2))).
+ Base3 =
+ Base2 ++
+ case couch_util:get_value(winning_revs_only, Options) of
+ undefined ->
+ [];
+ false ->
+ [];
+ true ->
+ [<<"winning_revs_only">>]
+ end,
+ couch_util:to_hex(couch_hash:md5_hash(term_to_binary(Base3))).
maybe_append_options(Options, RepOptions) ->
lists:foldl(
@@ -169,6 +179,42 @@ get_non_default_port(_Schema, Port) ->
-ifdef(TEST).
-include_lib("eunit/include/eunit.hrl").
+-include_lib("couch_replicator/test/eunit/couch_replicator_test.hrl").
+
+winning_revs_id_test_() ->
+ {
+ foreach,
+ fun test_util:start_couch/0,
+ fun test_util:stop_couch/1,
+ [
+ ?TDEF_FE(winning_revs_generates_new_id),
+ ?TDEF_FE(winning_revs_false_same_as_undefined)
+ ]
+ }.
+
+winning_revs_generates_new_id(_) ->
+ RepDoc1 = [
+ {<<"source">>, <<"http://foo.example.bar">>},
+ {<<"target">>, <<"http://bar.example.foo">>}
+ ],
+ Rep1 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc1}),
+ Id1 = replication_id(Rep1),
+ RepDoc2 = RepDoc1 ++ [{<<"winning_revs_only">>, true}],
+ Rep2 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc2}),
+ Id2 = replication_id(Rep2),
+ ?assertNotEqual(Id1, Id2).
+
+winning_revs_false_same_as_undefined(_) ->
+ RepDoc1 = [
+ {<<"source">>, <<"http://foo.example.bar">>},
+ {<<"target">>, <<"http://bar.example.foo">>}
+ ],
+ Rep1 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc1}),
+ Id1 = replication_id(Rep1),
+ RepDoc2 = RepDoc1 ++ [{<<"winning_revs_only">>, false}],
+ Rep2 = couch_replicator_docs:parse_rep_doc_without_id({RepDoc2}),
+ Id2 = replication_id(Rep2),
+ ?assertEqual(Id1, Id2).
replication_id_convert_test_() ->
[
diff --git a/src/couch_replicator/src/couch_replicator_js_functions.hrl b/src/couch_replicator/src/couch_replicator_js_functions.hrl
index d41043309..4f4369075 100644
--- a/src/couch_replicator/src/couch_replicator_js_functions.hrl
+++ b/src/couch_replicator/src/couch_replicator_js_functions.hrl
@@ -76,6 +76,12 @@
reportError('The `create_target\\' field must be a boolean.');
}
+ if ((typeof newDoc.winning_revs_only !== 'undefined') &&
+ (typeof newDoc.winning_revs_only !== 'boolean')) {
+
+ reportError('The `winning_revs_only\\' field must be a boolean.');
+ }
+
if ((typeof newDoc.continuous !== 'undefined') &&
(typeof newDoc.continuous !== 'boolean')) {
diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
index 86daa808f..95ccfaa07 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
@@ -17,7 +17,7 @@
-import(couch_replicator_test_helper, [
db_url/1,
- replicate/2
+ replicate/1
]).
-define(DOCS_CONFLICTS, [
@@ -32,88 +32,100 @@
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-setup() ->
+setup_db() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
ok = couch_db:close(Db),
DbName.
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
+teardown_db(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+
+setup() ->
Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
+ Source = setup_db(),
+ Target = setup_db(),
{Ctx, {Source, Target}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
+teardown({Ctx, {Source, Target}}) ->
+ teardown_db(Source),
+ teardown_db(Target),
ok = test_util:stop_couch(Ctx).
docs_with_many_leaves_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate documents with many leaves",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup/0,
+ fun teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ fun should_populate_replicate_compact/1
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
+docs_with_many_leaves_test_winning_revs_only_test_() ->
{
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target)
- ]}
+ "Replicate winning revs only for documents with many leaves",
+ {
+ foreach,
+ fun setup/0,
+ fun teardown/1,
+ [
+ fun should_replicate_winning_revs_only/1
+ ]
+ }
}.
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
+should_populate_replicate_compact({_Ctx, {Source, Target}}) ->
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target, all_revs),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target, all_revs)
+ ]}.
+
+should_replicate_winning_revs_only({_Ctx, {Source, Target}}) ->
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ should_verify_target(Source, Target, winning_revs),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ should_verify_target(Source, Target, winning_revs)
+ ]}.
+
should_populate_source(Source) ->
{timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+ should_replicate(Source, Target, []).
+
+should_replicate(Source, Target, Options) ->
+ {timeout, ?TIMEOUT_EUNIT,
+ ?_test(begin
+ RepObj = {
+ [
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)}
+ ] ++ Options
+ },
+ replicate(RepObj)
+ end)}.
-should_verify_target({remote, Source}, Target) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, {remote, Target}) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, Target) ->
+should_verify_target(Source, Target, Mode) ->
{timeout, ?TIMEOUT_EUNIT,
?_test(begin
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+ verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS, Mode),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb)
end)}.
-should_add_attachments_to_source({remote, Source}) ->
- should_add_attachments_to_source(Source);
should_add_attachments_to_source(Source) ->
{timeout, ?TIMEOUT_EUNIT,
?_test(begin
@@ -131,7 +143,11 @@ populate_db(DbName) ->
id = DocId,
body = {[{<<"value">>, Value}]}
},
- {ok, _} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ {ok, {Pos, Rev}} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ % Update first initial doc rev twice to ensure it's always a winner
+ {ok, Db2} = couch_db:reopen(Db),
+ Doc1 = Doc#doc{revs = {Pos, [Rev]}},
+ {ok, _} = couch_db:update_doc(Db2, Doc1, [?ADMIN_CTX]),
{ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
end,
?DOCS_CONFLICTS
@@ -160,9 +176,9 @@ add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
[{1, Rev} | AccRevs]
).
-verify_target(_SourceDb, _TargetDb, []) ->
+verify_target(_SourceDb, _TargetDb, [], _Mode) ->
ok;
-verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
{ok, SourceLookups} = couch_db:open_doc_revs(
SourceDb,
DocId,
@@ -187,7 +203,19 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
end,
lists:zip(SourceDocs, TargetDocs)
),
- verify_target(SourceDb, TargetDb, Rest).
+ verify_target(SourceDb, TargetDb, Rest, all_revs);
+verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs) ->
+ {ok, SourceWinner} = couch_db:open_doc(SourceDb, DocId),
+ {ok, TargetWinner} = couch_db:open_doc(TargetDb, DocId),
+ SourceWinnerJson = couch_doc:to_json_obj(SourceWinner, [attachments]),
+ TargetWinnerJson = couch_doc:to_json_obj(TargetWinner, [attachments]),
+ % Source winner is the same as the target winner
+ ?assertEqual(SourceWinnerJson, TargetWinnerJson),
+ Opts = [conflicts, deleted_conflicts],
+ {ok, TargetAll} = couch_db:open_doc_revs(TargetDb, DocId, all, Opts),
+ % There is only one version on the target
+ ?assert(length(TargetAll) == 1),
+ verify_target(SourceDb, TargetDb, Rest, winning_revs).
add_attachments(_SourceDb, _NumAtts, []) ->
ok;