You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2022/08/19 01:52:35 UTC
[couchdb] branch refactor-replication-tests created (now 3c4d713ce)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
at 3c4d713ce Update couch_replicator_use_checkpoints_tests
This branch includes the following new commits:
new 0c1676d9d Add some utility functions to couch_replicator_test_helper
new 8bd64e8da Update couch_replicator_attachments_too_large to use fabric
new 35da93ac1 Update couch_replicator_compact_tests
new c79cf582e Update couch_replicator_connection_tests
new b15409c1f Update couch_replicator_create_target_with_options_tests
new 0702d09b1 Update couch_replicator_error_reporting_tests
new c8f811d69 Update couch_replicator_filtered_tests
new 4765b097a Update couch_replicator_httpc_pool_tests
new 6add4086e Update couch_replicator_id_too_long_tests
new 35a2e80ae Update couch_replicator_large_atts_tests
new 2468a8682 Update couch_replicator_many_leaves_tests
new 65a9d0262 Update couch_replicator_missing_stubs_tests
new abdc7f2ee Update couch_replicator_proxy_tests
new c262a0e59 Update couch_replicator_rate_limiter_tests
new c0a0fec15 Update couch_replicator_retain_stats_between_job_runs
new 1623a4fa9 Update couch_replicator_selector_tests
new 4a2ce6676 Update couch_replicator_small_max_request_size_target
new 3c4d713ce Update couch_replicator_use_checkpoints_tests
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 17/18: Update couch_replicator_small_max_request_size_target
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4a2ce66761fd6f06e54b6a00aaf15cd1f112c4d5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:41:14 2022 -0400
Update couch_replicator_small_max_request_size_target
Use the clustered version of the source and target endoints and switch to using
common test setup and teardown function functions. Overall it added to quite a
few number of lines saved.
---
...ch_replicator_small_max_request_size_target.erl | 158 ++++++---------------
1 file changed, 43 insertions(+), 115 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
index 3b020927d..4a905850d 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
@@ -2,137 +2,61 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1,
- compare_dbs/3
-]).
+-include("couch_replicator_test.hrl").
-define(TIMEOUT_EUNIT, 360).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("chttpd", "max_http_request_size", "10000", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
reduce_max_request_size_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs when target has a small max_http_request_size",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_all_docs/2}
- || Pair <- Pairs
- ] ++
- [
- {Pair, fun should_replicate_one/2}
- || Pair <- Pairs
- ] ++
- % Disabled. See issue 574. Sometimes PUTs with a doc and
- % attachment which exceed maximum request size are simply
- % closed instead of returning a 413 request. That makes these
- % tests flaky.
- [
- {Pair, fun should_replicate_one_with_attachment/2}
- || Pair <- Pairs
- ]
+ ?TDEF_FE(should_replicate_all_docs, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one_with_attachment, ?TIMEOUT_EUNIT)
+ ]
}
}.
% Test documents which are below max_http_request_size but when batched, batch size
% will be greater than max_http_request_size. Replicator could automatically split
% the batch into smaller batches and POST those separately.
-should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [])
- ]}
- }.
+should_replicate_all_docs({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, []).
% If a document is too large to post as a single request, that document is
% skipped but replication overall will make progress and not crash.
-should_replicate_one({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_one_small(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_one_small(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
% If a document has an attachment > 64 * 1024 bytes, replicator will switch to
% POST-ing individual documents directly and skip bulk_docs. Test that case
% separately
% See note in main test function why this was disabled.
-should_replicate_one_with_attachment({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_attachment(Source),
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one_with_attachment({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_attachment(Source),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
+
+populate_source(Source) ->
+ add_docs(Source, 5, 3000, 0).
+
+populate_source_one_large_one_small(Source) ->
+ one_large_one_small(Source, 12000, 3000).
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(add_docs(Source, 5, 3000, 0))}.
-
-should_populate_source_one_large_one_small({remote, Source}) ->
- should_populate_source_one_large_one_small(Source);
-should_populate_source_one_large_one_small(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_one_small(Source, 12000, 3000))}.
-
-should_populate_source_one_large_attachment({remote, Source}) ->
- should_populate_source_one_large_attachment(Source);
-should_populate_source_one_large_attachment(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_attachment(Source, 70000, 70000))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, {remote, Target}, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, Target, ExceptIds) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target, ExceptIds))}.
+populate_source_one_large_attachment(Source) ->
+ one_large_attachment(Source, 70000, 70000).
binary_chunk(Size) when is_integer(Size), Size > 0 ->
<<<<"x">> || _ <- lists:seq(1, Size)>>.
@@ -155,11 +79,9 @@ one_large_attachment(DbName, Size, AttSize) ->
add_doc(DbName, <<"doc0">>, Size, AttSize).
add_doc(DbName, DocId, Size, AttSize) when is_binary(DocId) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Doc0 = #doc{id = DocId, body = {[{<<"x">>, binary_chunk(Size)}]}},
Doc = Doc0#doc{atts = atts(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
atts(0) ->
[];
@@ -173,12 +95,18 @@ atts(Size) ->
])
].
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
- % This make batch_size predictable
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
+ % This makes batch_size more predictable
{<<"worker_processes">>, "1"}
]}
).
+
+compare(Source, Target, ExceptIds) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target, ExceptIds).
[couchdb] 09/18: Update couch_replicator_id_too_long_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 6add4086edad72c73073967e9bb83263e829a32f
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:23:47 2022 -0400
Update couch_replicator_id_too_long_tests
Use common setup and teardown helpers along with some local replicate/2 and db_url/2 functions.
Remove foreachx goop and use TDEF_FE for consistency with other tests.
---
.../eunit/couch_replicator_id_too_long_tests.erl | 82 +++++++---------------
1 file changed, 26 insertions(+), 56 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
index 9ed415a29..08454bd71 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
@@ -14,73 +14,43 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:set("replicator", "max_document_id_length", "infinity"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
id_too_long_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Doc id too long tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "5"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
-
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "4"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "5", _Persist = false),
+ replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "4", _Persist = false),
+ replicate(Source, Target),
+ ?assertError({not_found, <<"12345">>}, compare(Source, Target)).
create_doc(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = couch_doc:from_json_obj({[{<<"_id">>, <<"12345">>}]}),
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 11/18: Update couch_replicator_many_leaves_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 2468a8682e6d03b20cc043b3534e5e17e3b04183
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:28:13 2022 -0400
Update couch_replicator_many_leaves_tests
Use comon setup and teardown function and the TDEF_FE macros.
Also, remove quite a bit of foreachx and remote boilerplate which is not needed
any longer.
Most of the changes however consisted in update all the db operations to use
fabric instead of couch. Luckily, most of those have fabric equivalents, and
fabric calls are even shorter as they don't need open, re-open and close
operations.
---
.../eunit/couch_replicator_many_leaves_tests.erl | 190 ++++++++-------------
1 file changed, 67 insertions(+), 123 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
index cdc90e2ea..aa5752650 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
@@ -14,11 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_CONFLICTS, [
{<<"doc1">>, 10},
@@ -32,35 +28,15 @@
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
-setup() ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup_db(),
- Target = setup_db(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {Source, Target}}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok = test_util:stop_couch(Ctx).
-
docs_with_many_leaves_test_() ->
{
"Replicate documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_populate_replicate_compact/1
+ ?TDEF_FE(should_populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
@@ -70,72 +46,40 @@ docs_with_many_leaves_test_winning_revs_only_test_() ->
"Replicate winning revs only for documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_replicate_winning_revs_only/1
+ ?TDEF_FE(should_replicate_winning_revs_only, ?TIMEOUT_EUNIT)
]
}
}.
should_populate_replicate_compact({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs)
- ]}.
+ populate_db(Source),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs).
should_replicate_winning_revs_only({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs)
- ]}.
-
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate(Source, Target) ->
- should_replicate(Source, Target, []).
-
-should_replicate(Source, Target, Options) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- RepObj = {
- [
- {<<"source">>, db_url(Source)},
- {<<"target">>, db_url(Target)}
- ] ++ Options
- },
- replicate(RepObj)
- end)}.
-
-should_verify_target(Source, Target, Mode) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS, Mode),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
-
-should_add_attachments_to_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]),
- add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
- ok = couch_db:close(SourceDb)
- end)}.
+ populate_db(Source),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs).
+
+replicate(Source, Target, Options) ->
+ RepObj = {
+ [
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)}
+ ] ++ Options
+ },
+ couch_replicator_test_helper:replicate(RepObj).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
lists:foreach(
fun({DocId, NumConflicts}) ->
Value = <<"0">>,
@@ -143,22 +87,20 @@ populate_db(DbName) ->
id = DocId,
body = {[{<<"value">>, Value}]}
},
- {ok, {Pos, Rev}} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ {ok, {Pos, Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]),
% Update first initial doc rev twice to ensure it's always a winner
- {ok, Db2} = couch_db:reopen(Db),
Doc1 = Doc#doc{revs = {Pos, [Rev]}},
- {ok, _} = couch_db:update_doc(Db2, Doc1, [?ADMIN_CTX]),
- {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ {ok, _} = fabric:update_doc(DbName, Doc1, [?ADMIN_CTX]),
+ {ok, _} = add_doc_siblings(DbName, DocId, NumConflicts)
end,
?DOCS_CONFLICTS
- ),
- couch_db:close(Db).
+ ).
add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
add_doc_siblings(Db, DocId, NumLeaves, [], []).
add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], ?REPLICATED_CHANGES),
+ {ok, []} = fabric:update_docs(Db, AccDocs, [?REPLICATED_CHANGES, ?ADMIN_CTX]),
{ok, AccRevs};
add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
Value = ?l2b(?i2l(NumLeaves)),
@@ -179,18 +121,8 @@ add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
verify_target(_SourceDb, _TargetDb, [], _Mode) ->
ok;
verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
- {ok, TargetLookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
+ SourceLookups = open_revs_conflicts(SourceDb, DocId),
+ TargetLookups = open_revs_conflicts(TargetDb, DocId),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
Total = NumConflicts + 1,
@@ -205,14 +137,13 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
),
verify_target(SourceDb, TargetDb, Rest, all_revs);
verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs) ->
- {ok, SourceWinner} = couch_db:open_doc(SourceDb, DocId),
- {ok, TargetWinner} = couch_db:open_doc(TargetDb, DocId),
+ SourceWinner = open_doc(SourceDb, DocId),
+ TargetWinner = open_doc(TargetDb, DocId),
SourceWinnerJson = couch_doc:to_json_obj(SourceWinner, [attachments]),
TargetWinnerJson = couch_doc:to_json_obj(TargetWinner, [attachments]),
% Source winner is the same as the target winner
?assertEqual(SourceWinnerJson, TargetWinnerJson),
- Opts = [conflicts, deleted_conflicts],
- {ok, TargetAll} = couch_db:open_doc_revs(TargetDb, DocId, all, Opts),
+ TargetAll = open_revs_conflicts(TargetDb, DocId),
% There is only one version on the target
?assert(length(TargetAll) == 1),
verify_target(SourceDb, TargetDb, Rest, winning_revs).
@@ -220,7 +151,7 @@ verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs)
add_attachments(_SourceDb, _NumAtts, []) ->
ok;
add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceLookups = open_revs(SourceDb, DocId, []),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
Total = NumConflicts + 1,
?assertEqual(Total, length(SourceDocs)),
@@ -228,20 +159,7 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
NewAtts = lists:foldl(
fun(I, AttAcc) ->
- AttData = crypto:strong_rand_bytes(100),
- NewAtt = couch_att:new([
- {name,
- ?io2b([
- "att_",
- ?i2l(I),
- "_",
- couch_doc:rev_to_str({Pos, Rev})
- ])},
- {type, <<"application/foobar">>},
- {att_len, byte_size(AttData)},
- {data, AttData}
- ]),
- [NewAtt | AttAcc]
+ [make_att(I, Pos, Rev, 100) | AttAcc]
end,
[],
lists:seq(1, NumAtts)
@@ -251,7 +169,33 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
[],
SourceDocs
),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ {ok, UpdateResults} = fabric:update_docs(SourceDb, NewDocs, [?ADMIN_CTX]),
NewRevs = [R || {ok, R} <- UpdateResults],
?assertEqual(length(NewDocs), length(NewRevs)),
add_attachments(SourceDb, NumAtts, Rest).
+
+make_att(Id, Pos, Rev, Size) ->
+ AttData = crypto:strong_rand_bytes(Size),
+ RevStr = couch_doc:rev_to_str({Pos, Rev}),
+ couch_att:new([
+ {name, ?io2b(["att_", ?i2l(Id), "_", RevStr])},
+ {type, <<"application/foobar">>},
+ {att_len, byte_size(AttData)},
+ {data, AttData}
+ ]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+open_revs_conflicts(DbName, Id) ->
+ Opts = [conflicts, deleted_conflicts],
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_revs(DbName, Id, Opts) ->
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_doc(DbName, Id) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ Doc.
[couchdb] 03/18: Update couch_replicator_compact_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 35da93ac185daf5844ee826fd24478115a8e7373
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:10:15 2022 -0400
Update couch_replicator_compact_tests
Compactor tests are the only tests which continue using the local ports since
they deals with triggering and managing low level compaction processes.
However, it was still possible to improve the tests somewhat by using the
TDEF_FE macros and removing some left-over foreachx cruft.
---
.../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------
1 file changed, 131 insertions(+), 183 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
index 1c093d58c..1f241c753 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
@@ -15,11 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- get_pid/1
-]).
+-include("couch_replicator_test.hrl").
-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
-define(DELAY, 500).
@@ -28,92 +24,60 @@
-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 70).
-define(WRITE_BATCH_SIZE, 25).
-setup() ->
+setup_db() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
ok = couch_db:close(Db),
DbName.
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
+teardown_db(DbName) ->
ok = couch_server:delete(DbName, [?ADMIN_CTX]),
ok.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
+test_setup() ->
+ Ctx = test_util:start_couch([couch_replicator]),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ teardown_db(Source),
+ teardown_db(Target),
ok = test_util:stop_couch(Ctx).
compact_test_() ->
- Pairs = [{remote, remote}],
{
"Compaction during replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun test_setup/0,
+ fun test_teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
+populate_replicate_compact({_Ctx, {Source, Target}}) ->
{ok, RepPid, RepId} = replicate(Source, Target),
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_run_replication(RepPid, RepId, Source, Target),
- should_all_processes_be_alive(RepPid, Source, Target),
- should_populate_and_compact(RepPid, Source, Target, 50, 3),
- should_wait_target_in_sync(Source, Target),
- should_ensure_replication_still_running(RepPid, RepId, Source, Target),
- should_cancel_replication(RepId, RepPid),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_all_processes_be_alive(RepPid, Source, Target) ->
- ?_test(begin
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb)))
- end).
-
-should_run_replication(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
-
-should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+ check_active_tasks(RepPid, RepId, Source, Target),
+ all_processes_are_alive(RepPid, Source, Target),
+ populate_and_compact(RepPid, Source, Target, 50, 3),
+ wait_target_in_sync(Source, Target),
+ check_active_tasks(RepPid, RepId, Source, Target),
+ cancel_replication(RepId, RepPid),
+ compare_databases(Source, Target).
+
+all_processes_are_alive(RepPid, Source, Target) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))).
check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
- Source =
- case Src of
- {remote, NameSrc} ->
- <<(db_url(NameSrc))/binary, $/>>;
- _ ->
- Src
- end,
- Target =
- case Tgt of
- {remote, NameTgt} ->
- <<(db_url(NameTgt))/binary, $/>>;
- _ ->
- Tgt
- end,
+ Source = <<(db_url(Src))/binary, $/>>,
+ Target = <<(db_url(Tgt))/binary, $/>>,
FullRepId = ?l2b(BaseId ++ Ext),
Pid = ?l2b(pid_to_list(RepPid)),
RepTasks = wait_for_task_status(),
@@ -152,71 +116,59 @@ wait_for_task_status() ->
end
end).
-should_cancel_replication(RepId, RepPid) ->
- ?_assertNot(begin
- ok = couch_replicator_scheduler:remove_job(RepId),
- is_process_alive(RepPid)
- end).
+cancel_replication(RepId, RepPid) ->
+ ok = couch_replicator_scheduler:remove_job(RepId),
+ ?assertNot(is_process_alive(RepPid)).
+
+populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
+ {ok, SourceDb0} = reopen_db(Source),
+ Writer = spawn_writer(SourceDb0),
+ lists:foreach(
+ fun(N) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ pause_writer(Writer),
+
+ compact_db("source", SourceDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ wait_for_compaction("source", SourceDb),
+
+ compact_db("target", TargetDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
+ wait_for_compaction("target", TargetDb),
+
+ {ok, SourceDb2} = reopen_db(SourceDb),
+ {ok, TargetDb2} = reopen_db(TargetDb),
+
+ resume_writer(Writer),
+ wait_writer(Writer, BatchSize * N),
+
+ compact_db("source", SourceDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("source", SourceDb2),
+ resume_writer(Writer),
+
+ compact_db("target", TargetDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("target", TargetDb2),
+ resume_writer(Writer)
+ end,
+ lists:seq(1, Rounds)
+ ),
+ stop_writer(Writer).
-should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb0} = reopen_db(Source),
- Writer = spawn_writer(SourceDb0),
- lists:foreach(
- fun(N) ->
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- pause_writer(Writer),
-
- compact_db("source", SourceDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- wait_for_compaction("source", SourceDb),
-
- compact_db("target", TargetDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
- wait_for_compaction("target", TargetDb),
-
- {ok, SourceDb2} = reopen_db(SourceDb),
- {ok, TargetDb2} = reopen_db(TargetDb),
-
- resume_writer(Writer),
- wait_writer(Writer, BatchSize * N),
-
- compact_db("source", SourceDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
- pause_writer(Writer),
- wait_for_compaction("source", SourceDb2),
- resume_writer(Writer),
-
- compact_db("target", TargetDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
- pause_writer(Writer),
- wait_for_compaction("target", TargetDb2),
- resume_writer(Writer)
- end,
- lists:seq(1, Rounds)
- ),
- stop_writer(Writer)
- end)}.
-
-should_wait_target_in_sync({remote, Source}, Target) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, {remote, Target}) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_assert(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
- wait_target_in_sync_loop(SourceDocCount, Target, 300)
- end)}.
+wait_target_in_sync(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
+ ok = couch_db:close(SourceDb),
+ SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
erlang:error(
@@ -226,8 +178,6 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
{reason, "Could not get source and target databases in sync"}
]}
);
-wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
{ok, Target} = couch_db:open_int(TargetName, []),
{ok, TargetInfo} = couch_db:get_db_info(Target),
@@ -241,49 +191,40 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, 35,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
+compare_databases(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget =
+ case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed, [
+ {module, ?MODULE},
+ {line, ?LINE},
+ {reason,
+ lists:concat([
+ "Error opening document '",
+ ?b2l(DocId),
+ "' from target: ",
+ couch_util:to_list(Error)
+ ])}
+ ]}
+ )
end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
-reopen_db({remote, Db}) ->
- reopen_db(Db);
reopen_db(DbName) when is_binary(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
@@ -357,21 +298,17 @@ wait_for_compaction(Type, Db) ->
)
end.
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
replicate(Source, Target) ->
RepObject =
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
- Pid = get_pid(Rep#rep.id),
+ Pid = couch_replicator_test_helper:get_pid(Rep#rep.id),
{ok, Pid, Rep#rep.id}.
wait_writer(Pid, NumDocs) ->
@@ -521,3 +458,14 @@ maybe_pause(Parent, Counter) ->
after 0 ->
ok
end.
+
+db_url(DbName) ->
+ % Note we're returning the backend (local) URL here
+ iolist_to_binary([
+ "http://",
+ config:get("httpd", "bind_address", "127.0.0.1"),
+ ":",
+ integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/",
+ DbName
+ ]).
[couchdb] 18/18: Update couch_replicator_use_checkpoints_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 3c4d713ced4743b460a8467016e5f1bbad1328e5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:45:29 2022 -0400
Update couch_replicator_use_checkpoints_tests
Use the clustered version of source and target endpoints.
Also, use common setup and teardown functions. The test was previously quite
clever in trying to save an extra few lines by parameterizing the "use
checkpoing" vs "don't use checkpoints" scenarios with a foreachx and a custom
notifier function. Instead, opt for more clarity and use the usual TDEF_FE
macro and just setup two separate test one which uses checkpoints and one which
doesn't.
Another major win is is using the utility db comparison function instead of a
duplicate copy of it.
---
.../couch_replicator_use_checkpoints_tests.erl | 181 ++++++---------------
1 file changed, 48 insertions(+), 133 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
index a23f415c0..d96a1c9d7 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
@@ -14,133 +14,77 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_COUNT, 100).
-define(TIMEOUT_EUNIT, 30).
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-start(false) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
- (_) ->
- ok
- end;
-start(true) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertNotEqual(
- false,
- lists:keyfind(
- <<"session_id">>,
- 1,
- CheckpointHistory
- )
- );
- (_) ->
- ok
- end.
-
-stop(_, _) ->
- ok.
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+setup_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_checkpoint_fun/1,
+ {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+ {Ctx, {Source, Target, Listener}}.
-setup(remote) ->
- {remote, setup()};
-setup({_, Fun, {A, B}}) ->
- Ctx = test_util:start_couch([couch_replicator]),
+setup_no_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_no_checkpoint_fun/1,
{ok, Listener} = couch_replicator_notifier:start_link(Fun),
- Source = setup(A),
- Target = setup(B),
{Ctx, {Source, Target, Listener}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+teardown({Ctx, {Source, Target, Listener}}) ->
+ couch_replicator_notifier:stop(Listener),
+ couch_replicator_test_helper:test_teardown({Ctx, {Source, Target}}).
-teardown(_, {Ctx, {Source, Target, Listener}}) ->
- teardown(Source),
- teardown(Target),
+notifier_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ SId = lists:keyfind(<<"session_id">>, 1, CheckpointHistory),
+ ?assertNotEqual(false, SId);
+notifier_checkpoint_fun(_) ->
+ ok.
- couch_replicator_notifier:stop(Listener),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+notifier_no_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
+notifier_no_checkpoint_fun(_) ->
+ ok.
use_checkpoints_test_() ->
{
- "Replication use_checkpoints feature tests",
+ "Replication test using checkpoints",
{
- foreachx,
- fun start/1,
- fun stop/2,
+ foreach,
+ fun setup_checkpoints/0,
+ fun teardown/1,
[
- {UseCheckpoints, fun use_checkpoints_tests/2}
- || UseCheckpoints <- [false, true]
+ ?TDEF_FE(use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-use_checkpoints_tests(UseCheckpoints, Fun) ->
- Pairs = [{remote, remote}],
+dont_use_checkpoints_test_() ->
{
- "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+ "Replication test without using checkpoints",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup_no_checkpoints/0,
+ fun teardown/1,
[
- {{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
- || Pair <- Pairs
+ ?TDEF_FE(dont_use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-should_test_checkpoints({UseCheckpoints, _, {From, To}}, {_Ctx, {Source, Target, _}}) ->
- should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
-should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source, ?DOCS_COUNT),
- should_replicate(Source, Target, UseCheckpoints),
- should_compare_databases(Source, Target)
- ]}
- }.
+use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, true),
+ compare_dbs(Source, Target).
-should_populate_source({remote, Source}, DocCount) ->
- should_populate_source(Source, DocCount);
-should_populate_source(Source, DocCount) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
-
-should_replicate({remote, Source}, Target, UseCheckpoints) ->
- should_replicate(db_url(Source), Target, UseCheckpoints);
-should_replicate(Source, {remote, Target}, UseCheckpoints) ->
- should_replicate(Source, db_url(Target), UseCheckpoints);
-should_replicate(Source, Target, UseCheckpoints) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+dont_use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, false),
+ compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
@@ -154,48 +98,19 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
- end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
replicate(Source, Target, UseCheckpoints) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"use_checkpoints">>, UseCheckpoints}
]}
).
[couchdb] 14/18: Update couch_replicator_rate_limiter_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c262a0e593c4d7d6090e71da0c0facbc07afc141
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:35:50 2022 -0400
Update couch_replicator_rate_limiter_tests
Use the TDEF_FE macro and cleanup ?_test(begin...end) instances.
---
.../eunit/couch_replicator_rate_limiter_tests.erl | 77 ++++++++++------------
1 file changed, 33 insertions(+), 44 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
index a214d4607..0b7c48b62 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
@@ -1,6 +1,7 @@
-module(couch_replicator_rate_limiter_tests).
-include_lib("couch/include/couch_eunit.hrl").
+-include("couch_replicator_test.hrl").
rate_limiter_test_() ->
{
@@ -8,58 +9,46 @@ rate_limiter_test_() ->
fun setup/0,
fun teardown/1,
[
- t_new_key(),
- t_1_failure(),
- t_2_failures_back_to_back(),
- t_2_failures(),
- t_success_threshold(),
- t_1_failure_2_successes()
+ ?TDEF_FE(t_new_key),
+ ?TDEF_FE(t_1_failure),
+ ?TDEF_FE(t_2_failures_back_to_back),
+ ?TDEF_FE(t_2_failures),
+ ?TDEF_FE(t_success_threshold),
+ ?TDEF_FE(t_1_failure_2_successes)
]
}.
-t_new_key() ->
- ?_test(begin
- ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get}))
- end).
+t_new_key(_) ->
+ ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get})).
-t_1_failure() ->
- ?_test(begin
- ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get}))
- end).
+t_1_failure(_) ->
+ ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get})).
-t_2_failures() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(29, Interval)
- end).
+t_2_failures(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(29, Interval).
-t_2_failures_back_to_back() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(24, Interval)
- end).
+t_2_failures_back_to_back(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(24, Interval).
-t_success_threshold() ->
- ?_test(begin
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval),
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval)
- end).
+t_success_threshold(_) ->
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval),
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval).
-t_1_failure_2_successes() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(20, Succ1),
- low_pass_filter_delay(),
- Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Succ2)
- end).
+t_1_failure_2_successes(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(20, Succ1),
+ low_pass_filter_delay(),
+ Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Succ2).
low_pass_filter_delay() ->
timer:sleep(100).
[couchdb] 05/18: Update couch_replicator_create_target_with_options_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit b15409c1f0fda06855d8eb034fd9c2079719f55f
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:15:02 2022 -0400
Update couch_replicator_create_target_with_options_tests
Start using the common setup and tear down functions from the test helper.
Also using the test definitions to use the TDEF_FE macro.
Since the setup function already creates a target endpoint database and the
test is also in charge of creating test database, we just remove the target db
before the replication jobs start.
---
...replicator_create_target_with_options_tests.erl | 74 +++++-----------------
1 file changed, 17 insertions(+), 57 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
index 8adcd25bd..fabb6480e 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
@@ -14,35 +14,20 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
--define(USERNAME, "rep_admin").
--define(PASSWORD, "secret").
-
-setup() ->
- Ctx = test_util:start_couch([fabric, mem3, couch_replicator, chttpd]),
- Hashed = couch_passwords:hash_admin_password(?PASSWORD),
- ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
- Source = ?tempdb(),
- Target = ?tempdb(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {_Source, _Target}}) ->
- config:delete("admins", ?USERNAME),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
create_target_with_options_replication_test_() ->
{
"Create target with range partitions tests",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_create_target_with_q_4/1,
- fun should_create_target_with_q_2_n_1/1,
- fun should_create_target_with_default/1,
- fun should_not_create_target_with_q_any/1
+ ?TDEF_FE(should_create_target_with_q_4),
+ ?TDEF_FE(should_create_target_with_q_2_n_1),
+ ?TDEF_FE(should_create_target_with_default),
+ ?TDEF_FE(should_not_create_target_with_q_any)
]
}
}.
@@ -55,15 +40,12 @@ should_create_target_with_q_4({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"4">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(4, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(4, couch_util:get_value(q, ClusterInfo)).
should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
RepObject =
@@ -73,18 +55,13 @@ should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"2">>}, {<<"n">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- [
- ?_assertEqual(2, couch_util:get_value(q, ClusterInfo)),
- ?_assertEqual(1, couch_util:get_value(n, ClusterInfo))
- ].
+ ?assertEqual(2, couch_util:get_value(q, ClusterInfo)),
+ ?assertEqual(1, couch_util:get_value(n, ClusterInfo)).
should_create_target_with_default({_Ctx, {Source, Target}}) ->
RepObject =
@@ -93,16 +70,13 @@ should_create_target_with_default({_Ctx, {Source, Target}}) ->
{<<"target">>, db_url(Target)},
{<<"create_target">>, true}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
Q = config:get_integer("cluster", "q", 2),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
RepObject =
@@ -112,33 +86,19 @@ should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
{<<"create_target">>, false},
{<<"create_target_params">>, {[{<<"q">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{error, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
DbExist = is_list(catch mem3:shards(Target)),
- delete_db(Source),
- ?_assertEqual(false, DbExist).
+ ?assertEqual(false, DbExist).
create_doc(DbName) ->
Body = {[{<<"foo">>, <<"bar">>}]},
NewDoc = #doc{body = Body},
{ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]).
-create_db(DbName) ->
- ok = fabric:create_db(DbName, [?ADMIN_CTX]).
-
delete_db(DbName) ->
ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
db_url(DbName) ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- ?l2b(
- io_lib:format("http://~s:~s@~s:~b/~s", [
- ?USERNAME,
- ?PASSWORD,
- Addr,
- Port,
- DbName
- ])
- ).
+ couch_replicator_test_helper:cluster_db_url(DbName).
[couchdb] 04/18: Update couch_replicator_connection_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c79cf582e61288546c6291b363a062321406d58a
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:13:44 2022 -0400
Update couch_replicator_connection_tests
The main changes are just using the TDEF_FE macros and removng the
_test(begin...end) silliness.
---
.../eunit/couch_replicator_connection_tests.erl | 264 ++++++++++-----------
1 file changed, 123 insertions(+), 141 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
index 7adbb6852..e8c90d02a 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
@@ -13,9 +13,7 @@
-module(couch_replicator_connection_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
-
--define(TIMEOUT, 1000).
+-include("couch_replicator_test.hrl").
setup() ->
Host = config:get("httpd", "bind_address", "127.0.0.1"),
@@ -37,164 +35,148 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun connections_shared_after_release/1,
- fun connections_not_shared_after_owner_death/1,
- fun idle_connections_closed/1,
- fun test_owner_monitors/1,
- fun worker_discards_creds_on_create/1,
- fun worker_discards_url_creds_after_request/1,
- fun worker_discards_creds_in_headers_after_request/1,
- fun worker_discards_proxy_creds_after_request/1
+ ?TDEF_FE(connections_shared_after_release),
+ ?TDEF_FE(connections_not_shared_after_owner_death),
+ ?TDEF_FE(idle_connections_closed),
+ ?TDEF_FE(test_owner_monitors),
+ ?TDEF_FE(worker_discards_creds_on_create),
+ ?TDEF_FE(worker_discards_url_creds_after_request),
+ ?TDEF_FE(worker_discards_creds_in_headers_after_request),
+ ?TDEF_FE(worker_discards_proxy_creds_after_request)
]
}
}
}.
connections_shared_after_release({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection:release(Pid),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL)
- end),
- receive
- {ok, Pid2} ->
- ?assertEqual(Pid, Pid2)
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection:release(Pid),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL)
+ end),
+ receive
+ {ok, Pid2} ->
+ ?assertEqual(Pid, Pid2)
+ end.
connections_not_shared_after_owner_death({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL),
- error("simulate division by zero without compiler warning")
- end),
- receive
- {ok, Pid} ->
- {ok, Pid2} = couch_replicator_connection:acquire(URL),
- ?assertNotEqual(Pid, Pid2),
- MRef = monitor(process, Pid),
- receive
- {'DOWN', MRef, process, Pid, _Reason} ->
- ?assert(not is_process_alive(Pid));
- Other ->
- throw(Other)
- end
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL),
+ error("simulate division by zero without compiler warning")
+ end),
+ receive
+ {ok, Pid} ->
+ {ok, Pid2} = couch_replicator_connection:acquire(URL),
+ ?assertNotEqual(Pid, Pid2),
+ MRef = monitor(process, Pid),
+ receive
+ {'DOWN', MRef, process, Pid, _Reason} ->
+ ?assert(not is_process_alive(Pid));
+ Other ->
+ throw(Other)
+ end
+ end.
idle_connections_closed({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection ! close_idle_connections,
- ?assert(ets:member(couch_replicator_connection, Pid)),
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- couch_replicator_connection:release(Pid),
- couch_replicator_connection ! close_idle_connections,
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- ?assert(not ets:member(couch_replicator_connection, Pid))
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection ! close_idle_connections,
+ ?assert(ets:member(couch_replicator_connection, Pid)),
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ couch_replicator_connection:release(Pid),
+ couch_replicator_connection ! close_idle_connections,
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ ?assert(not ets:member(couch_replicator_connection, Pid)).
test_owner_monitors({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Worker0} = couch_replicator_connection:acquire(URL),
- assert_monitors_equal([{process, self()}]),
- couch_replicator_connection:release(Worker0),
- assert_monitors_equal([]),
- {Workers, Monitors} = lists:foldl(
- fun(_, {WAcc, MAcc}) ->
- {ok, Worker1} = couch_replicator_connection:acquire(URL),
- MAcc1 = [{process, self()} | MAcc],
- assert_monitors_equal(MAcc1),
- {[Worker1 | WAcc], MAcc1}
- end,
- {[], []},
- lists:seq(1, 5)
- ),
- lists:foldl(
- fun(Worker2, Acc) ->
- [_ | NewAcc] = Acc,
- couch_replicator_connection:release(Worker2),
- assert_monitors_equal(NewAcc),
- NewAcc
- end,
- Monitors,
- Workers
- )
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Worker0} = couch_replicator_connection:acquire(URL),
+ assert_monitors_equal([{process, self()}]),
+ couch_replicator_connection:release(Worker0),
+ assert_monitors_equal([]),
+ {Workers, Monitors} = lists:foldl(
+ fun(_, {WAcc, MAcc}) ->
+ {ok, Worker1} = couch_replicator_connection:acquire(URL),
+ MAcc1 = [{process, self()} | MAcc],
+ assert_monitors_equal(MAcc1),
+ {[Worker1 | WAcc], MAcc1}
+ end,
+ {[], []},
+ lists:seq(1, 5)
+ ),
+ lists:foldl(
+ fun(Worker2, Acc) ->
+ [_ | NewAcc] = Acc,
+ couch_replicator_connection:release(Worker2),
+ assert_monitors_equal(NewAcc),
+ NewAcc
+ end,
+ Monitors,
+ Workers
+ ).
worker_discards_creds_on_create({Host, Port}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0).
worker_discards_url_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_creds_in_headers_after_request({Host, _}) ->
- ?_test(begin
- {_User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Headers = [{"Authorization", "Basic " ++ B64Auth}],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {_User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Headers = [{"Authorization", "Basic " ++ B64Auth}],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_proxy_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Opts = [
- {proxy_host, Host},
- {proxy_port, Port},
- {proxy_user, User},
- {proxy_pass, Pass}
- ],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Opts = [
+ {proxy_host, Host},
+ {proxy_port, Port},
+ {proxy_user, User},
+ {proxy_pass, Pass}
+ ],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
send_req(WPid, URL, Headers, Opts) ->
ibrowse:send_req_direct(WPid, URL, Headers, get, [], Opts).
[couchdb] 16/18: Update couch_replicator_selector_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1623a4fa913958f627f6b0ba53c1e453e494e986
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:39:17 2022 -0400
Update couch_replicator_selector_tests
Use the clustered versions of endpoints for the test.
Also, use the common setup and teardown helpers and remove the foreachx
silliness.
---
.../test/eunit/couch_replicator_selector_tests.erl | 121 ++++++++-------------
1 file changed, 44 insertions(+), 77 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
index 8f61a638c..31c5dc93c 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
@@ -14,92 +14,62 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
selector_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Selector filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
+ replicate(
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"selector">>, {[{<<"_id">>, <<"doc2">>}]}}
- ]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ ]}
+ ),
%% FilteredFun is an Erlang version of following mango selector
- FilterFun = fun(_DocId, {Props}) ->
- couch_util:get_value(<<"_id">>, Props) == <<"doc2">>
+ FilterFun = fun(DocId, #doc{}) ->
+ DocId == <<"doc2">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"All the docs selected as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs selected as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc1 = couch_doc:from_json_obj(
{[
{<<"_id">>, <<"doc1">>}
@@ -110,13 +80,10 @@ create_docs(DbName) ->
{<<"_id">>, <<"doc2">>}
]}
),
- {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2]),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, [Doc1, Doc2], [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 15/18: Update couch_replicator_retain_stats_between_job_runs
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c0a0fec152b1fc32e93791da3a2291e723c777e9
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:37:32 2022 -0400
Update couch_replicator_retain_stats_between_job_runs
Switch the test to use the clustered endpoints.
Use the common test setup and teardown functions as well as the TDEF_FE macros.
---
...ch_replicator_retain_stats_between_job_runs.erl | 154 ++++++++-------------
1 file changed, 55 insertions(+), 99 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
index 9ffcc9e2c..3b8bd6452 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
@@ -15,104 +15,70 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DELAY, 500).
-define(TIMEOUT, 60000).
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
-
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
-
-setup() ->
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
-
-teardown({Source, Target}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok.
-
stats_retained_test_() ->
{
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_stats_retained_by_scheduler/1,
- fun t_stats_retained_on_job_removal/1
- ]
- }
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_stats_retained_by_scheduler),
+ ?TDEF_FE(t_stats_retained_on_job_removal)
+ ]
}.
-t_stats_retained_by_scheduler({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, RepPid, RepId} = replicate(Source, Target),
- wait_target_in_sync(6, Target),
-
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+t_stats_retained_by_scheduler({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, RepPid, RepId} = replicate(Source, Target),
+ wait_target_in_sync(6, Target),
- stop_job(RepPid),
- check_scheduler_jobs(10, 5, 5),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- start_job(),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
- couch_replicator_scheduler:remove_job(RepId)
- end).
-
-t_stats_retained_on_job_removal({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, _, RepId} = replicate(Source, Target),
- % 5 + 1 vdu
- wait_target_in_sync(6, Target),
+ stop_job(RepPid),
+ check_scheduler_jobs(10, 5, 5),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+ start_job(),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
+ couch_replicator_scheduler:remove_job(RepId).
- couch_replicator_scheduler:remove_job(RepId),
+t_stats_retained_on_job_removal({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 5 + 1 vdu
+ wait_target_in_sync(6, Target),
- populate_db_reject_even_docs(Source, 11, 20),
- {ok, _, RepId} = replicate(Source, Target),
- % 6 + 5
- wait_target_in_sync(11, Target),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- check_scheduler_jobs(20, 10, 10),
- check_active_tasks(20, 10, 10),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId),
+ populate_db_reject_even_docs(Source, 11, 20),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 6 + 5
+ wait_target_in_sync(11, Target),
- populate_db_reject_even_docs(Source, 21, 30),
- {ok, _, RepId} = replicate(Source, Target),
- % 11 + 5
- wait_target_in_sync(16, Target),
+ check_scheduler_jobs(20, 10, 10),
+ check_active_tasks(20, 10, 10),
- check_scheduler_jobs(30, 15, 15),
- check_active_tasks(30, 15, 15),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId)
- end).
+ populate_db_reject_even_docs(Source, 21, 30),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 11 + 5
+ wait_target_in_sync(16, Target),
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ check_scheduler_jobs(30, 15, 15),
+ check_active_tasks(30, 15, 15),
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ couch_replicator_scheduler:remove_job(RepId).
stop_job(RepPid) ->
Ref = erlang:monitor(process, RepPid),
@@ -201,7 +167,6 @@ populate_db_reject_even_docs(DbName, Start, End) ->
populate_db(DbName, Start, End, BodyFun).
populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -211,8 +176,7 @@ populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(DocCount, Target) when is_integer(DocCount) ->
wait_target_in_sync_loop(DocCount, Target, 300).
@@ -226,10 +190,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -238,13 +199,14 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
@@ -254,10 +216,9 @@ replicate(Source, Target) ->
{ok, Pid, Rep#rep.id}.
scheduler_jobs() ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- Url = lists:flatten(io_lib:format("http://~s:~b/_scheduler/jobs", [Addr, Port])),
- {ok, 200, _, Body} = test_request:get(Url, []),
+ ClusterUrl = couch_replicator_test_helper:cluster_url(),
+ JobsUrl = <<ClusterUrl/binary, "/_scheduler/jobs">>,
+ {ok, 200, _, Body} = test_request:get(?b2l(JobsUrl), []),
Json = jiffy:decode(Body, [return_maps]),
maps:get(<<"jobs">>, Json).
@@ -279,9 +240,4 @@ add_vdu(DbName) ->
{<<"validate_doc_update">>, vdu()}
],
Doc = couch_doc:from_json_obj({DocProps}, []),
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
- try
- {ok, _Rev} = couch_db:update_doc(Db, Doc, [])
- after
- couch_db:close(Db)
- end.
+ {ok, _Rev} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
[couchdb] 13/18: Update couch_replicator_proxy_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit abdc7f2ee67b0e63aa5b27448b2b96cf52de3be4
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:34:26 2022 -0400
Update couch_replicator_proxy_tests
Use the TDEF_FE macro and remove the ugly ?_test(begin...end) construct.
---
.../test/eunit/couch_replicator_proxy_tests.erl | 137 ++++++++++-----------
1 file changed, 64 insertions(+), 73 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
index ca1816b33..184d81aaf 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
@@ -15,6 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include("couch_replicator_test.hrl").
setup() ->
ok.
@@ -34,90 +35,80 @@ replicator_proxy_test_() ->
fun setup/0,
fun teardown/1,
[
- fun parse_rep_doc_without_proxy/1,
- fun parse_rep_doc_with_proxy/1,
- fun parse_rep_source_target_proxy/1,
- fun mutually_exclusive_proxy_and_source_proxy/1,
- fun mutually_exclusive_proxy_and_target_proxy/1
+ ?TDEF_FE(parse_rep_doc_without_proxy),
+ ?TDEF_FE(parse_rep_doc_with_proxy),
+ ?TDEF_FE(parse_rep_source_target_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_source_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_target_proxy)
]
}
}
}.
parse_rep_doc_without_proxy(_) ->
- ?_test(begin
- NoProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined)
- end).
+ NoProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined).
parse_rep_doc_with_proxy(_) ->
- ?_test(begin
- ProxyURL = <<"http://myproxy.com">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, ProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL))
- end).
+ ProxyURL = <<"http://myproxy.com">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, ProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL)).
parse_rep_source_target_proxy(_) ->
- ?_test(begin
- SrcProxyURL = <<"http://mysrcproxy.com">>,
- TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"source_proxy">>, SrcProxyURL},
- {<<"target_proxy">>, TgtProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual(
- (Rep#rep.source)#httpdb.proxy_url,
- binary_to_list(SrcProxyURL)
- ),
- ?assertEqual(
- (Rep#rep.target)#httpdb.proxy_url,
- binary_to_list(TgtProxyURL)
- )
- end).
+ SrcProxyURL = <<"http://mysrcproxy.com">>,
+ TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"source_proxy">>, SrcProxyURL},
+ {<<"target_proxy">>, TgtProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual(
+ (Rep#rep.source)#httpdb.proxy_url,
+ binary_to_list(SrcProxyURL)
+ ),
+ ?assertEqual(
+ (Rep#rep.target)#httpdb.proxy_url,
+ binary_to_list(TgtProxyURL)
+ ).
mutually_exclusive_proxy_and_source_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"source_proxy">>, <<"sourceproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"source_proxy">>, <<"sourceproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
mutually_exclusive_proxy_and_target_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"target_proxy">>, <<"targetproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"target_proxy">>, <<"targetproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
[couchdb] 12/18: Update couch_replicator_missing_stubs_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 65a9d026250bc8b130619e67cae9b71a3fd2d258
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:31:46 2022 -0400
Update couch_replicator_missing_stubs_tests
Use common setup and teardown helpers, TDEF_FE macros and remove all the
foreachx nonsense.
---
.../eunit/couch_replicator_missing_stubs_tests.erl | 146 ++++++---------------
1 file changed, 39 insertions(+), 107 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
index ff3b5ee98..b0e205729 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
@@ -14,106 +14,34 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(REVS_LIMIT, 3).
-define(TIMEOUT_EUNIT, 30).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
missing_stubs_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with missing stubs (COUCHDB-1365)",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_docs_with_missed_att_stubs/2}
- || Pair <- Pairs
+ ?TDEF_FE(replicate_docs_with_missing_att_stubs, ?TIMEOUT_EUNIT)
]
}
}.
-should_replicate_docs_with_missed_att_stubs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_set_target_revs_limit(Target, ?REVS_LIMIT),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target),
- should_update_source_docs(Source, ?REVS_LIMIT * 2),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_set_target_revs_limit({remote, Target}, RevsLimit) ->
- should_set_target_revs_limit(Target, RevsLimit);
-should_set_target_revs_limit(Target, RevsLimit) ->
- ?_test(begin
- {ok, Db} = couch_db:open_int(Target, [?ADMIN_CTX]),
- ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
- ok = couch_db:close(Db)
- end).
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-should_update_source_docs({remote, Source}, Times) ->
- should_update_source_docs(Source, Times);
-should_update_source_docs(Source, Times) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+replicate_docs_with_missing_att_stubs({_Ctx, {Source, Target}}) ->
+ populate_db(Source),
+ fabric:set_revs_limit(Target, ?REVS_LIMIT, [?ADMIN_CTX]),
+ replicate(Source, Target),
+ compare(Source, Target),
+ update_docs(Source, ?REVS_LIMIT * 2),
+ replicate(Source, Target),
+ compare(Source, Target).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
AttData = crypto:strong_rand_bytes(6000),
Doc = #doc{
id = <<"doc1">>,
@@ -126,34 +54,38 @@ populate_db(DbName) ->
])
]
},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
-update_db_docs(DbName, Times) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _} = couch_db:fold_docs(
- Db,
- fun(FDI, Acc) -> db_fold_fun(FDI, Acc) end,
- {DbName, Times},
- []
- ),
- ok = couch_db:close(Db).
+update_docs(DbName, Times) ->
+ lists:foreach(
+ fun({Id, _Rev}) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ update_doc(DbName, Doc, Times)
+ end,
+ couch_replicator_test_helper:cluster_doc_revs(DbName)
+ ).
-db_fold_fun(FullDocInfo, {DbName, Times}) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+update_doc(DbName, Doc, Times) ->
+ {Pos0, [Rev0 | _]} = Doc#doc.revs,
lists:foldl(
fun(_, {Pos, RevId}) ->
- {ok, Db2} = couch_db:reopen(Db),
- NewDocVersion = Doc#doc{
+ Val = base64:encode(crypto:strong_rand_bytes(100)),
+ NewDoc = Doc#doc{
revs = {Pos, [RevId]},
- body = {[{<<"value">>, base64:encode(crypto:strong_rand_bytes(100))}]}
+ body = {[{<<"value">>, Val}]}
},
- {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+ {ok, NewRev} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]),
NewRev
end,
- {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+ {Pos0, Rev0},
lists:seq(1, Times)
- ),
- ok = couch_db:close(Db),
- {ok, {DbName, Times}}.
+ ).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
+
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 02/18: Update couch_replicator_attachments_too_large to use fabric
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8bd64e8dad8765b1440e30fa94d16ec37f930721
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:08:17 2022 -0400
Update couch_replicator_attachments_too_large to use fabric
Switch test module to use the clustered enpdpoints and TDEF_FE test
macros.
---
.../couch_replicator_attachments_too_large.erl | 79 +++++++---------------
1 file changed, 24 insertions(+), 55 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
index 2d58f847e..4c4ff9c14 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
@@ -14,69 +14,37 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc_with_attachment(Source, <<"doc">>, 1000),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:delete("couchdb", "max_attachment_size"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
attachment_too_large_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Attachment size too large replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "1000", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
+ ok = replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "999", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
-
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ ok = replicate(Source, Target),
+ ?assertError({not_found, <<"doc">>}, compare(Source, Target)).
create_doc_with_attachment(DbName, DocId, AttSize) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = #doc{id = DocId, atts = att(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db),
- ok.
+ {ok, {1, _Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
att(Size) when is_integer(Size), Size >= 1 ->
[
@@ -90,10 +58,11 @@ att(Size) when is_integer(Size), Size >= 1 ->
])
].
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 07/18: Update couch_replicator_filtered_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit c8f811d69e99fe58aee86766aa3454dda2ee3ad5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:20:15 2022 -0400
Update couch_replicator_filtered_tests
Take advantage of the helper setup and teardown functions.
Switching to a simpler TDEF_FE macro instead of foreachx and inorder setup
cruft also saves some lines of code.
---
.../test/eunit/couch_replicator_filtered_tests.erl | 257 ++++++++-------------
1 file changed, 100 insertions(+), 157 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
index b77b83daa..267c4fab6 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
@@ -14,7 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DDOC,
{[
@@ -57,206 +57,149 @@
]}
).
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
filtered_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_succeed_with_query),
+ ?TDEF_FE(should_succeed_with_view)
+ ]
}
}.
-query_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with query replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_query/2} || Pair <- Pairs]
- }
- }.
-
-view_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with a view replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_view/2} || Pair <- Pairs]
- }
- }.
-
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/testfilter">>}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ replicate(RepObject),
%% FilteredFun is an Erlang version of following JS function
%% function(doc, req){if (doc.class == 'mammal') return true;}
- FilterFun = fun(_DocId, {Props}) ->
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs,
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_query({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/queryfilter">>},
{<<"query_params">>,
{[
{<<"starts">>, <<"a">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
case couch_util:get_value(<<"class">>, Props) of
<<"a", _/binary>> -> true;
_ -> false
end
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(2, TargetDocCount),
+ % All the docs filtered as expected,
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_view({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_view({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"_view">>},
{<<"query_params">>,
{[
{<<"view">>, <<"filter_ddoc/mammals">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
- DDoc = couch_doc:from_json_obj(?DDOC),
- Doc1 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc1">>},
- {<<"class">>, <<"mammal">>},
- {<<"value">>, 1}
- ]}
- ),
- Doc2 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc2">>},
- {<<"class">>, <<"amphibians">>},
- {<<"value">>, 2}
- ]}
- ),
- Doc3 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc3">>},
- {<<"class">>, <<"reptiles">>},
- {<<"value">>, 3}
- ]}
- ),
- Doc4 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc4">>},
- {<<"class">>, <<"arthropods">>},
- {<<"value">>, 2}
- ]}
- ),
- {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3, Doc4]),
- couch_db:close(Db).
+ Docs = [
+ couch_doc:from_json_obj(?DDOC),
+ #doc{
+ id = <<"doc1">>,
+ body =
+ {[
+ {<<"class">>, <<"mammal">>},
+ {<<"value">>, 1}
+ ]}
+ },
+ #doc{
+ id = <<"doc2">>,
+ body =
+ {[
+ {<<"class">>, <<"amphibians">>},
+ {<<"value">>, 2}
+ ]}
+ },
+ #doc{
+ id = <<"doc3">>,
+ body =
+ {[
+ {<<"class">>, <<"reptiles">>},
+ {<<"value">>, 3}
+ ]}
+ },
+ #doc{
+ id = <<"doc4">>,
+ body =
+ {[
+ {<<"class">>, <<"arthropods">>},
+ {<<"value">>, 2}
+ ]}
+ }
+ ],
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 06/18: Update couch_replicator_error_reporting_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0702d09b1d5f19367d203b7ae683b38f30594674
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:17:37 2022 -0400
Update couch_replicator_error_reporting_tests
Use the TDEF_FE macro and clean up ?_test(begin...end) cruft.
---
.../couch_replicator_error_reporting_tests.erl | 272 +++++++++------------
1 file changed, 111 insertions(+), 161 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
index b0863614c..7e198562f 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
@@ -15,149 +15,115 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
+error_reporting_test_() ->
+ {
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_fail_bulk_docs),
+ ?TDEF_FE(t_fail_changes_reader),
+ ?TDEF_FE(t_fail_revs_diff),
+ ?TDEF_FE(t_fail_changes_queue),
+ ?TDEF_FE(t_fail_changes_manager),
+ ?TDEF_FE(t_fail_changes_reader_proc)
+ ]
+ }.
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
+t_fail_bulk_docs({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
-setup() ->
- meck:unload(),
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
-teardown({Source, Target}) ->
- meck:unload(),
- teardown_db(Source),
- teardown_db(Target),
- ok.
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-error_reporting_test_() ->
- {
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_fail_bulk_docs/1,
- fun t_fail_changes_reader/1,
- fun t_fail_revs_diff/1,
- fun t_fail_changes_queue/1,
- fun t_fail_changes_manager/1,
- fun t_fail_changes_reader_proc/1
- ]
- }
- }.
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_revs_diff({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
-t_fail_bulk_docs({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_revs_diff({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_queue({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesQueue = element(20, State),
- ?assert(is_process_alive(ChangesQueue)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesQueue, boom),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_queue_died, boom}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_manager({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesManager = element(21, State),
- ?assert(is_process_alive(ChangesManager)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesManager, bam),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_manager_died, bam}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader_proc({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesReader = element(22, State),
- ?assert(is_process_alive(ChangesReader)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesReader, kapow),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_reader_died, kapow}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
+t_fail_changes_queue({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesQueue = element(20, State),
+ ?assert(is_process_alive(ChangesQueue)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesQueue, boom),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_queue_died, boom}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_manager({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesManager = element(21, State),
+ ?assert(is_process_alive(ChangesManager)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesManager, bam),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_manager_died, bam}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader_proc({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesReader = element(22, State),
+ ?assert(is_process_alive(ChangesReader)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesReader, kapow),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_reader_died, kapow}, Result),
+ couch_replicator_notifier:stop(Listener).
mock_fail_req(Path, Return) ->
meck:expect(
@@ -190,17 +156,7 @@ wait_rep_result(RepId) ->
{error, RepId, Reason} -> {error, Reason}
end.
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
populate_db(DbName, Start, End) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -210,14 +166,10 @@ populate_db(DbName, Start, End) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ {ok, SourceDocCount} = fabric:get_doc_count(Source),
wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
@@ -229,10 +181,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -242,12 +191,10 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
end.
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, url(Source)},
+ {<<"target">>, url(Target)},
{<<"continuous">>, true},
{<<"worker_processes">>, 1},
{<<"retries_per_request">>, 1},
@@ -258,3 +205,6 @@ replicate(Source, Target) ->
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
{ok, Rep#rep.id}.
+
+url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
[couchdb] 01/18: Add some utility functions to couch_replicator_test_helper
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 0c1676d9df5f57d0967a6db86ad8a5b4eb80ea81
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:03:22 2022 -0400
Add some utility functions to couch_replicator_test_helper
In preparation to start using chttpd (fabric) endpoints add some common utility
functions to the replication test helper module.
Since `couch_db:fold_docs/4` doesn't exit for fabric, use the changes feed to
get all the revision leafs. That is used when comparing database endpoints.
It turns our the majority replication tests can use the exact same setup,
teardown and db_url functions so make sure those are also available in the
helper module.
---
.../test/eunit/couch_replicator_test_helper.erl | 131 +++++++++++++++------
1 file changed, 97 insertions(+), 34 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
index 4044e7c72..9888f1b84 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
@@ -4,39 +4,73 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-define(USERNAME, "rep_test_user").
+-define(PASSWORD, "rep_test_pass").
+
-export([
- compare_dbs/2,
- compare_dbs/3,
- db_url/1,
+ cluster_compare_dbs/2,
+ cluster_compare_dbs/3,
+ cluster_doc_revs/1,
+ cluster_open_rev/3,
+ cluster_url/0,
+ cluster_db_url/1,
replicate/1,
get_pid/1,
- replicate/2
+ replicate/2,
+ test_setup/0,
+ test_teardown/1,
+ setup_db/0,
+ teardown_db/1
]).
-compare_dbs(Source, Target) ->
- compare_dbs(Source, Target, []).
-
-compare_dbs(Source, Target, ExceptIds) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
-
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
- Id = DocSource#doc.id,
- case lists:member(Id, ExceptIds) of
- true ->
- ?assertEqual(not_found, couch_db:get_doc_info(TargetDb, Id));
- false ->
- {ok, TDoc} = couch_db:open_doc(TargetDb, Id),
- compare_docs(DocSource, TDoc)
+cluster_compare_dbs(Source, Target) ->
+ cluster_compare_dbs(Source, Target, []).
+
+cluster_compare_dbs(Source, Target, ExceptIds) ->
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Source)),
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Target)),
+ lists:foreach(
+ fun({Id, Rev}) ->
+ SrcDoc = cluster_open_rev(Source, Id, Rev),
+ TgtDoc = cluster_open_rev(Target, Id, Rev),
+ case lists:member(Id, ExceptIds) of
+ true ->
+ ?assertEqual(not_found, TgtDoc);
+ false ->
+ compare_docs(SrcDoc, TgtDoc)
+ end
end,
- {ok, Acc}
- end,
+ cluster_doc_revs(Source)
+ ).
+
+cluster_open_rev(DbName, Id, Rev) ->
+ {ok, [Result]} = fabric:open_revs(DbName, Id, [Rev], []),
+ case Result of
+ {ok, #doc{} = Doc} ->
+ Doc;
+ {{not_found, missing}, _} ->
+ not_found
+ end.
+
+cluster_doc_revs(DbName) ->
+ Opts = [{style, all_docs}],
+ {ok, Acc} = fabric_util:isolate(fun() ->
+ fabric:changes(DbName, fun changes_callback/2, [], Opts)
+ end),
+ Acc.
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+changes_callback(start, Acc) ->
+ {ok, Acc};
+changes_callback({change, {Change}}, Acc) ->
+ Id = proplists:get_value(id, Change),
+ Revs = proplists:get_value(changes, Change),
+ IdRevs = [{Id, couch_doc:parse_rev(R)} || {[{<<"rev">>, R}]} <- Revs],
+ {ok, IdRevs ++ Acc};
+changes_callback({stop, _EndSeq, _Pending}, Acc) ->
+ {ok, Acc}.
+compare_docs(#doc{} = Doc1, not_found) ->
+ error({not_found, Doc1#doc.id});
compare_docs(Doc1, Doc2) ->
?assertEqual(Doc1#doc.body, Doc2#doc.body),
#doc{atts = Atts1} = Doc1,
@@ -111,15 +145,16 @@ att_decoded_md5(Att) ->
),
couch_hash:md5_hash_final(Md50).
-db_url(DbName) ->
- iolist_to_binary([
- "http://",
- config:get("httpd", "bind_address", "127.0.0.1"),
- ":",
- integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/",
- DbName
- ]).
+cluster_url() ->
+ Fmt = "http://~s:~s@~s:~b",
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Args = [?USERNAME, ?PASSWORD, Addr, Port],
+ ?l2b(io_lib:format(Fmt, Args)).
+
+cluster_db_url(DbName) when is_binary(DbName) ->
+ ClusterUrl = cluster_url(),
+ <<ClusterUrl/binary, "/", DbName/binary>>.
get_pid(RepId) ->
Pid = global:whereis_name({couch_replicator_scheduler_job, RepId}),
@@ -145,3 +180,31 @@ replicate({[_ | _]} = RepObject) ->
ok
end,
ok = couch_replicator_scheduler:remove_job(Rep#rep.id).
+
+setup_db() ->
+ DbName = ?tempdb(),
+ ok = fabric:create_db(DbName, [{q, 1}, {n, 1}, ?ADMIN_CTX]),
+ DbName.
+
+teardown_db(DbName) ->
+ try
+ ok = fabric:delete_db(DbName, [?ADMIN_CTX])
+ catch
+ error:database_does_not_exist ->
+ ok
+ end.
+
+test_setup() ->
+ Ctx = test_util:start_couch([fabric, mem3, chttpd, couch_replicator]),
+ Hashed = couch_passwords:hash_admin_password(?PASSWORD),
+ ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ meck:unload(),
+ teardown_db(Source),
+ teardown_db(Target),
+ config:delete("admins", ?USERNAME, _Persist = false),
+ ok = test_util:stop_couch(Ctx).
[couchdb] 10/18: Update couch_replicator_large_atts_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 35a2e80ae15a9e882a2fcfab52138b3a05185835
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:25:37 2022 -0400
Update couch_replicator_large_atts_tests
Use commong setup functions and TDEF_FE macro.
Removing the foreachx and the remote vs local junk really trimmed down the
size. The test content was tiny compared to the clunky EUnit setup logic.
---
.../eunit/couch_replicator_large_atts_tests.erl | 92 ++++++----------------
1 file changed, 23 insertions(+), 69 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
index 2f0e2a1f0..8190c7205 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
@@ -14,12 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(ATT_SIZE_1, 2 * 1024 * 1024).
-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
@@ -27,83 +22,37 @@
-define(TIMEOUT_EUNIT, 120).
setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("attachments", "compressible_types", "text/*", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ Ctx = couch_replicator_test_helper:test_setup(),
+ config:set("attachments", "compressible_types", "text/*", _Persist = false),
+ Ctx.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
-
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+teardown(Ctx) ->
+ config:delete("attachments", "compressible_types", _Persist = false),
+ couch_replicator_test_helper:test_teardown(Ctx).
large_atts_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with large attachments",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup/0,
+ fun teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(should_replicate_atts, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+should_replicate_atts({_Ctx, {Source, Target}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ ?assertEqual(ok, replicate(Source, Target)),
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Doc = #doc{
- id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ id = integer_to_binary(DocIdCounter),
body = {[]},
atts = [
att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
@@ -115,8 +64,7 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
att(Name, Size, Type) ->
couch_att:new([
@@ -125,3 +73,9 @@ att(Name, Size, Type) ->
{att_len, Size},
{data, fun(Count) -> crypto:strong_rand_bytes(Count) end}
]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 08/18: Update couch_replicator_httpc_pool_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch refactor-replication-tests
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 4765b097a3d992210c59d43ec2fff0708a9787f2
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:22:30 2022 -0400
Update couch_replicator_httpc_pool_tests
Use the TDEF_FE macro and remove the ?_test(begin...end) construct.
---
.../eunit/couch_replicator_httpc_pool_tests.erl | 96 +++++++++++-----------
1 file changed, 46 insertions(+), 50 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
index 31f1da48e..5fce5e886 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
@@ -13,7 +13,7 @@
-module(couch_replicator_httpc_pool_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
+-include("couch_replicator_test.hrl").
-define(TIMEOUT, 1000).
@@ -35,71 +35,67 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun should_block_new_clients_when_full/1,
- fun should_replace_worker_on_death/1
+ ?TDEF_FE(should_block_new_clients_when_full),
+ ?TDEF_FE(should_replace_worker_on_death)
]
}
}
}.
should_block_new_clients_when_full(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
+ Client1 = spawn_client(Pool),
+ Client2 = spawn_client(Pool),
+ Client3 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- ?assertEqual(ok, ping_client(Client2)),
- ?assertEqual(ok, ping_client(Client3)),
+ ?assertEqual(ok, ping_client(Client1)),
+ ?assertEqual(ok, ping_client(Client2)),
+ ?assertEqual(ok, ping_client(Client3)),
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
+ Worker1 = get_client_worker(Client1, "1"),
+ Worker2 = get_client_worker(Client2, "2"),
+ Worker3 = get_client_worker(Client3, "3"),
- ?assert(is_process_alive(Worker1)),
- ?assert(is_process_alive(Worker2)),
- ?assert(is_process_alive(Worker3)),
+ ?assert(is_process_alive(Worker1)),
+ ?assert(is_process_alive(Worker2)),
+ ?assert(is_process_alive(Worker3)),
- ?assertNotEqual(Worker1, Worker2),
- ?assertNotEqual(Worker2, Worker3),
- ?assertNotEqual(Worker3, Worker1),
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertNotEqual(Worker2, Worker3),
+ ?assertNotEqual(Worker3, Worker1),
- Client4 = spawn_client(Pool),
- ?assertEqual(timeout, ping_client(Client4)),
+ Client4 = spawn_client(Pool),
+ ?assertEqual(timeout, ping_client(Client4)),
- ?assertEqual(ok, stop_client(Client1)),
- ?assertEqual(ok, ping_client(Client4)),
+ ?assertEqual(ok, stop_client(Client1)),
+ ?assertEqual(ok, ping_client(Client4)),
- Worker4 = get_client_worker(Client4, "4"),
- ?assertEqual(Worker1, Worker4),
+ Worker4 = get_client_worker(Client4, "4"),
+ ?assertEqual(Worker1, Worker4),
- lists:foreach(
- fun(C) ->
- ?assertEqual(ok, stop_client(C))
- end,
- [Client2, Client3, Client4]
- )
- end).
+ lists:foreach(
+ fun(C) ->
+ ?assertEqual(ok, stop_client(C))
+ end,
+ [Client2, Client3, Client4]
+ ).
should_replace_worker_on_death(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- Worker1 = get_client_worker(Client1, "1"),
- ?assert(is_process_alive(Worker1)),
-
- ?assertEqual(ok, kill_client_worker(Client1)),
- ?assertNot(is_process_alive(Worker1)),
- ?assertEqual(ok, stop_client(Client1)),
-
- Client2 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client2)),
- Worker2 = get_client_worker(Client2, "2"),
- ?assert(is_process_alive(Worker2)),
-
- ?assertNotEqual(Worker1, Worker2),
- ?assertEqual(ok, stop_client(Client2))
- end).
+ Client1 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client1)),
+ Worker1 = get_client_worker(Client1, "1"),
+ ?assert(is_process_alive(Worker1)),
+
+ ?assertEqual(ok, kill_client_worker(Client1)),
+ ?assertNot(is_process_alive(Worker1)),
+ ?assertEqual(ok, stop_client(Client1)),
+
+ Client2 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client2)),
+ Worker2 = get_client_worker(Client2, "2"),
+ ?assert(is_process_alive(Worker2)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertEqual(ok, stop_client(Client2)).
spawn_client(Pool) ->
Parent = self(),