You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2022/08/19 17:19:05 UTC
[couchdb] branch main updated (9a5d0de97 -> e6db35b6b)
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
from 9a5d0de97 Update couchdb-mochiweb to v3.1.0
new 264ad11f3 Add some utility functions to couch_replicator_test_helper
new 1ec76df18 Update couch_replicator_attachments_too_large to use fabric
new e1a947531 Update couch_replicator_compact_tests
new 620bdea36 Update couch_replicator_connection_tests
new 31185647f Update couch_replicator_create_target_with_options_tests
new 381fe30d8 Update couch_replicator_error_reporting_tests
new cad96318f Update couch_replicator_filtered_tests
new 75e73da04 Update couch_replicator_httpc_pool_tests
new 065b212e5 Update couch_replicator_id_too_long_tests
new 9a6875f6b Update couch_replicator_large_atts_tests
new ca46fa61c Update couch_replicator_many_leaves_tests
new 033b8d5de Update couch_replicator_missing_stubs_tests
new 9243298ef Update couch_replicator_proxy_tests
new d831ec70e Update couch_replicator_rate_limiter_tests
new 1e69eaac9 Update couch_replicator_retain_stats_between_job_runs
new 8e6158972 Update couch_replicator_selector_tests
new 609e7cc79 Update couch_replicator_small_max_request_size_target
new e6db35b6b Update couch_replicator_use_checkpoints_tests
The 18 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../couch_replicator_attachments_too_large.erl | 79 ++----
.../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------
.../eunit/couch_replicator_connection_tests.erl | 264 ++++++++---------
...replicator_create_target_with_options_tests.erl | 74 ++---
.../couch_replicator_error_reporting_tests.erl | 272 ++++++++----------
.../test/eunit/couch_replicator_filtered_tests.erl | 257 +++++++----------
.../eunit/couch_replicator_httpc_pool_tests.erl | 96 +++----
.../eunit/couch_replicator_id_too_long_tests.erl | 82 ++----
.../eunit/couch_replicator_large_atts_tests.erl | 92 ++----
.../eunit/couch_replicator_many_leaves_tests.erl | 190 +++++--------
.../eunit/couch_replicator_missing_stubs_tests.erl | 146 +++-------
.../test/eunit/couch_replicator_proxy_tests.erl | 137 +++++----
.../eunit/couch_replicator_rate_limiter_tests.erl | 77 +++--
...ch_replicator_retain_stats_between_job_runs.erl | 153 ++++------
.../test/eunit/couch_replicator_selector_tests.erl | 121 +++-----
...ch_replicator_small_max_request_size_target.erl | 158 +++--------
.../test/eunit/couch_replicator_test_helper.erl | 134 ++++++---
.../couch_replicator_use_checkpoints_tests.erl | 182 ++++--------
18 files changed, 1094 insertions(+), 1734 deletions(-)
[couchdb] 06/18: Update couch_replicator_error_reporting_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 381fe30d88a83becc2ac92627c73345bc647ecfc
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:17:37 2022 -0400
Update couch_replicator_error_reporting_tests
Use the TDEF_FE macro and clean up ?_test(begin...end) cruft.
---
.../couch_replicator_error_reporting_tests.erl | 272 +++++++++------------
1 file changed, 111 insertions(+), 161 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
index b0863614c..7e198562f 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_error_reporting_tests.erl
@@ -15,149 +15,115 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
+error_reporting_test_() ->
+ {
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_fail_bulk_docs),
+ ?TDEF_FE(t_fail_changes_reader),
+ ?TDEF_FE(t_fail_revs_diff),
+ ?TDEF_FE(t_fail_changes_queue),
+ ?TDEF_FE(t_fail_changes_manager),
+ ?TDEF_FE(t_fail_changes_reader_proc)
+ ]
+ }.
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
+t_fail_bulk_docs({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
-setup() ->
- meck:unload(),
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
-teardown({Source, Target}) ->
- meck:unload(),
- teardown_db(Source),
- teardown_db(Target),
- ok.
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-error_reporting_test_() ->
- {
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_fail_bulk_docs/1,
- fun t_fail_changes_reader/1,
- fun t_fail_revs_diff/1,
- fun t_fail_changes_queue/1,
- fun t_fail_changes_manager/1,
- fun t_fail_changes_reader_proc/1
- ]
- }
- }.
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_revs_diff({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
+ populate_db(Source, 6, 6),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
+
+ couch_replicator_notifier:stop(Listener).
-t_fail_bulk_docs({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_bulk_docs", {ok, "403", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({bulk_docs_failed, 403, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_changes", {ok, "418", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_req_failed, 418, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_revs_diff({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- {ok, Listener} = rep_result_listener(RepId),
- mock_fail_req("/_revs_diff", {ok, "407", [], [<<"{\"x\":\"y\"}">>]}),
- populate_db(Source, 6, 6),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({revs_diff_failed, 407, {[{<<"x">>, <<"y">>}]}}, Result),
-
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_queue({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesQueue = element(20, State),
- ?assert(is_process_alive(ChangesQueue)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesQueue, boom),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_queue_died, boom}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_manager({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesManager = element(21, State),
- ?assert(is_process_alive(ChangesManager)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesManager, bam),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_manager_died, bam}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
-
-t_fail_changes_reader_proc({Source, Target}) ->
- ?_test(begin
- populate_db(Source, 1, 5),
- {ok, RepId} = replicate(Source, Target),
- wait_target_in_sync(Source, Target),
-
- RepPid = couch_replicator_test_helper:get_pid(RepId),
- State = sys:get_state(RepPid),
- ChangesReader = element(22, State),
- ?assert(is_process_alive(ChangesReader)),
-
- {ok, Listener} = rep_result_listener(RepId),
- exit(ChangesReader, kapow),
-
- {error, Result} = wait_rep_result(RepId),
- ?assertEqual({changes_reader_died, kapow}, Result),
- couch_replicator_notifier:stop(Listener)
- end).
+t_fail_changes_queue({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesQueue = element(20, State),
+ ?assert(is_process_alive(ChangesQueue)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesQueue, boom),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_queue_died, boom}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_manager({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesManager = element(21, State),
+ ?assert(is_process_alive(ChangesManager)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesManager, bam),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_manager_died, bam}, Result),
+ couch_replicator_notifier:stop(Listener).
+
+t_fail_changes_reader_proc({_Ctx, {Source, Target}}) ->
+ populate_db(Source, 1, 5),
+ {ok, RepId} = replicate(Source, Target),
+ wait_target_in_sync(Source, Target),
+
+ RepPid = couch_replicator_test_helper:get_pid(RepId),
+ State = sys:get_state(RepPid),
+ ChangesReader = element(22, State),
+ ?assert(is_process_alive(ChangesReader)),
+
+ {ok, Listener} = rep_result_listener(RepId),
+ exit(ChangesReader, kapow),
+
+ {error, Result} = wait_rep_result(RepId),
+ ?assertEqual({changes_reader_died, kapow}, Result),
+ couch_replicator_notifier:stop(Listener).
mock_fail_req(Path, Return) ->
meck:expect(
@@ -190,17 +156,7 @@ wait_rep_result(RepId) ->
{error, RepId, Reason} -> {error, Reason}
end.
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
populate_db(DbName, Start, End) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -210,14 +166,10 @@ populate_db(DbName, Start, End) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ {ok, SourceDocCount} = fabric:get_doc_count(Source),
wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
@@ -229,10 +181,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -242,12 +191,10 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
end.
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, url(Source)},
+ {<<"target">>, url(Target)},
{<<"continuous">>, true},
{<<"worker_processes">>, 1},
{<<"retries_per_request">>, 1},
@@ -258,3 +205,6 @@ replicate(Source, Target) ->
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
{ok, Rep#rep.id}.
+
+url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
[couchdb] 03/18: Update couch_replicator_compact_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e1a947531d7bb70a4ca2ca3f007b92661fef3935
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:10:15 2022 -0400
Update couch_replicator_compact_tests
Compactor tests are the only tests which continue using the local ports since
they deals with triggering and managing low level compaction processes.
However, it was still possible to improve the tests somewhat by using the
TDEF_FE macros and removing some left-over foreachx cruft.
---
.../test/eunit/couch_replicator_compact_tests.erl | 314 +++++++++------------
1 file changed, 131 insertions(+), 183 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
index 1c093d58c..1f241c753 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_compact_tests.erl
@@ -15,11 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- get_pid/1
-]).
+-include("couch_replicator_test.hrl").
-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
-define(DELAY, 500).
@@ -28,92 +24,60 @@
-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 70).
-define(WRITE_BATCH_SIZE, 25).
-setup() ->
+setup_db() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
ok = couch_db:close(Db),
DbName.
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
+teardown_db(DbName) ->
ok = couch_server:delete(DbName, [?ADMIN_CTX]),
ok.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
+test_setup() ->
+ Ctx = test_util:start_couch([couch_replicator]),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ teardown_db(Source),
+ teardown_db(Target),
ok = test_util:stop_couch(Ctx).
compact_test_() ->
- Pairs = [{remote, remote}],
{
"Compaction during replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun test_setup/0,
+ fun test_teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
+populate_replicate_compact({_Ctx, {Source, Target}}) ->
{ok, RepPid, RepId} = replicate(Source, Target),
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_run_replication(RepPid, RepId, Source, Target),
- should_all_processes_be_alive(RepPid, Source, Target),
- should_populate_and_compact(RepPid, Source, Target, 50, 3),
- should_wait_target_in_sync(Source, Target),
- should_ensure_replication_still_running(RepPid, RepId, Source, Target),
- should_cancel_replication(RepId, RepPid),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_all_processes_be_alive(RepPid, Source, Target) ->
- ?_test(begin
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb)))
- end).
-
-should_run_replication(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
-
-should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+ check_active_tasks(RepPid, RepId, Source, Target),
+ all_processes_are_alive(RepPid, Source, Target),
+ populate_and_compact(RepPid, Source, Target, 50, 3),
+ wait_target_in_sync(Source, Target),
+ check_active_tasks(RepPid, RepId, Source, Target),
+ cancel_replication(RepId, RepPid),
+ compare_databases(Source, Target).
+
+all_processes_are_alive(RepPid, Source, Target) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))).
check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
- Source =
- case Src of
- {remote, NameSrc} ->
- <<(db_url(NameSrc))/binary, $/>>;
- _ ->
- Src
- end,
- Target =
- case Tgt of
- {remote, NameTgt} ->
- <<(db_url(NameTgt))/binary, $/>>;
- _ ->
- Tgt
- end,
+ Source = <<(db_url(Src))/binary, $/>>,
+ Target = <<(db_url(Tgt))/binary, $/>>,
FullRepId = ?l2b(BaseId ++ Ext),
Pid = ?l2b(pid_to_list(RepPid)),
RepTasks = wait_for_task_status(),
@@ -152,71 +116,59 @@ wait_for_task_status() ->
end
end).
-should_cancel_replication(RepId, RepPid) ->
- ?_assertNot(begin
- ok = couch_replicator_scheduler:remove_job(RepId),
- is_process_alive(RepPid)
- end).
+cancel_replication(RepId, RepPid) ->
+ ok = couch_replicator_scheduler:remove_job(RepId),
+ ?assertNot(is_process_alive(RepPid)).
+
+populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
+ {ok, SourceDb0} = reopen_db(Source),
+ Writer = spawn_writer(SourceDb0),
+ lists:foreach(
+ fun(N) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ pause_writer(Writer),
+
+ compact_db("source", SourceDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ wait_for_compaction("source", SourceDb),
+
+ compact_db("target", TargetDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
+ wait_for_compaction("target", TargetDb),
+
+ {ok, SourceDb2} = reopen_db(SourceDb),
+ {ok, TargetDb2} = reopen_db(TargetDb),
+
+ resume_writer(Writer),
+ wait_writer(Writer, BatchSize * N),
+
+ compact_db("source", SourceDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("source", SourceDb2),
+ resume_writer(Writer),
+
+ compact_db("target", TargetDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
+ pause_writer(Writer),
+ wait_for_compaction("target", TargetDb2),
+ resume_writer(Writer)
+ end,
+ lists:seq(1, Rounds)
+ ),
+ stop_writer(Writer).
-should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb0} = reopen_db(Source),
- Writer = spawn_writer(SourceDb0),
- lists:foreach(
- fun(N) ->
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- pause_writer(Writer),
-
- compact_db("source", SourceDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
- wait_for_compaction("source", SourceDb),
-
- compact_db("target", TargetDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
- wait_for_compaction("target", TargetDb),
-
- {ok, SourceDb2} = reopen_db(SourceDb),
- {ok, TargetDb2} = reopen_db(TargetDb),
-
- resume_writer(Writer),
- wait_writer(Writer, BatchSize * N),
-
- compact_db("source", SourceDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
- pause_writer(Writer),
- wait_for_compaction("source", SourceDb2),
- resume_writer(Writer),
-
- compact_db("target", TargetDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
- pause_writer(Writer),
- wait_for_compaction("target", TargetDb2),
- resume_writer(Writer)
- end,
- lists:seq(1, Rounds)
- ),
- stop_writer(Writer)
- end)}.
-
-should_wait_target_in_sync({remote, Source}, Target) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, {remote, Target}) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_assert(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
- wait_target_in_sync_loop(SourceDocCount, Target, 300)
- end)}.
+wait_target_in_sync(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
+ ok = couch_db:close(SourceDb),
+ SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ wait_target_in_sync_loop(SourceDocCount, Target, 300).
wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
erlang:error(
@@ -226,8 +178,6 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
{reason, "Could not get source and target databases in sync"}
]}
);
-wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
{ok, Target} = couch_db:open_int(TargetName, []),
{ok, TargetInfo} = couch_db:get_db_info(Target),
@@ -241,49 +191,40 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, 35,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
+compare_databases(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget =
+ case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed, [
+ {module, ?MODULE},
+ {line, ?LINE},
+ {reason,
+ lists:concat([
+ "Error opening document '",
+ ?b2l(DocId),
+ "' from target: ",
+ couch_util:to_list(Error)
+ ])}
+ ]}
+ )
end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
-reopen_db({remote, Db}) ->
- reopen_db(Db);
reopen_db(DbName) when is_binary(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
@@ -357,21 +298,17 @@ wait_for_compaction(Type, Db) ->
)
end.
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
replicate(Source, Target) ->
RepObject =
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
ok = couch_replicator_scheduler:add_job(Rep),
couch_replicator_scheduler:reschedule(),
- Pid = get_pid(Rep#rep.id),
+ Pid = couch_replicator_test_helper:get_pid(Rep#rep.id),
{ok, Pid, Rep#rep.id}.
wait_writer(Pid, NumDocs) ->
@@ -521,3 +458,14 @@ maybe_pause(Parent, Counter) ->
after 0 ->
ok
end.
+
+db_url(DbName) ->
+ % Note we're returning the backend (local) URL here
+ iolist_to_binary([
+ "http://",
+ config:get("httpd", "bind_address", "127.0.0.1"),
+ ":",
+ integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/",
+ DbName
+ ]).
[couchdb] 18/18: Update couch_replicator_use_checkpoints_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit e6db35b6b5737a7397e10758a87c06c10f729200
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:45:29 2022 -0400
Update couch_replicator_use_checkpoints_tests
Use the clustered version of source and target endpoints.
Also, use common setup and teardown functions. The test was previously quite
clever in trying to save an extra few lines by parameterizing the "use
checkpoing" vs "don't use checkpoints" scenarios with a foreachx and a custom
notifier function. Instead, opt for more clarity and use the usual TDEF_FE
macro and just setup two separate test one which uses checkpoints and one which
doesn't.
Another major win is is using the utility db comparison function instead of a
duplicate copy of it.
---
.../couch_replicator_use_checkpoints_tests.erl | 182 ++++++---------------
1 file changed, 49 insertions(+), 133 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
index a23f415c0..193855a3d 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_use_checkpoints_tests.erl
@@ -14,133 +14,78 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_COUNT, 100).
-define(TIMEOUT_EUNIT, 30).
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-start(false) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
- (_) ->
- ok
- end;
-start(true) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertNotEqual(
- false,
- lists:keyfind(
- <<"session_id">>,
- 1,
- CheckpointHistory
- )
- );
- (_) ->
- ok
- end.
-
-stop(_, _) ->
- ok.
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+setup_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_checkpoint_fun/1,
+ {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+ {Ctx, {Source, Target, Listener}}.
-setup(remote) ->
- {remote, setup()};
-setup({_, Fun, {A, B}}) ->
- Ctx = test_util:start_couch([couch_replicator]),
+setup_no_checkpoints() ->
+ {Ctx, {Source, Target}} = couch_replicator_test_helper:test_setup(),
+ Fun = fun notifier_no_checkpoint_fun/1,
{ok, Listener} = couch_replicator_notifier:start_link(Fun),
- Source = setup(A),
- Target = setup(B),
{Ctx, {Source, Target, Listener}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+teardown({Ctx, {Source, Target, Listener}}) ->
+ couch_replicator_notifier:stop(Listener),
+ couch_replicator_test_helper:test_teardown({Ctx, {Source, Target}}).
-teardown(_, {Ctx, {Source, Target, Listener}}) ->
- teardown(Source),
- teardown(Target),
+notifier_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ SId = lists:keyfind(<<"session_id">>, 1, CheckpointHistory),
+ SId =/= false orelse ?debugFmt("~nsession_id not found when using checkpoints", []),
+ ?assertNotEqual(false, SId);
+notifier_checkpoint_fun(_) ->
+ ok.
- couch_replicator_notifier:stop(Listener),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+notifier_no_checkpoint_fun({finished, _, {CheckpointHistory}}) ->
+ ?assertEqual([{<<"use_checkpoints">>, false}], CheckpointHistory);
+notifier_no_checkpoint_fun(_) ->
+ ok.
use_checkpoints_test_() ->
{
- "Replication use_checkpoints feature tests",
+ "Replication test using checkpoints",
{
- foreachx,
- fun start/1,
- fun stop/2,
+ foreach,
+ fun setup_checkpoints/0,
+ fun teardown/1,
[
- {UseCheckpoints, fun use_checkpoints_tests/2}
- || UseCheckpoints <- [false, true]
+ ?TDEF_FE(use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-use_checkpoints_tests(UseCheckpoints, Fun) ->
- Pairs = [{remote, remote}],
+dont_use_checkpoints_test_() ->
{
- "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+ "Replication test without using checkpoints",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup_no_checkpoints/0,
+ fun teardown/1,
[
- {{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
- || Pair <- Pairs
+ ?TDEF_FE(dont_use_checkpoints, ?TIMEOUT_EUNIT)
]
}
}.
-should_test_checkpoints({UseCheckpoints, _, {From, To}}, {_Ctx, {Source, Target, _}}) ->
- should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
-should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source, ?DOCS_COUNT),
- should_replicate(Source, Target, UseCheckpoints),
- should_compare_databases(Source, Target)
- ]}
- }.
+use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, true),
+ compare_dbs(Source, Target).
-should_populate_source({remote, Source}, DocCount) ->
- should_populate_source(Source, DocCount);
-should_populate_source(Source, DocCount) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
-
-should_replicate({remote, Source}, Target, UseCheckpoints) ->
- should_replicate(db_url(Source), Target, UseCheckpoints);
-should_replicate(Source, {remote, Target}, UseCheckpoints) ->
- should_replicate(Source, db_url(Target), UseCheckpoints);
-should_replicate(Source, Target, UseCheckpoints) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+dont_use_checkpoints({_Ctx, {Source, Target, _}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ replicate(Source, Target, false),
+ compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
@@ -154,48 +99,19 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget =
- case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed, [
- {module, ?MODULE},
- {line, ?LINE},
- {reason,
- lists:concat([
- "Error opening document '",
- ?b2l(DocId),
- "' from target: ",
- couch_util:to_list(Error)
- ])}
- ]}
- )
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
- end,
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
replicate(Source, Target, UseCheckpoints) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"use_checkpoints">>, UseCheckpoints}
]}
).
[couchdb] 17/18: Update couch_replicator_small_max_request_size_target
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 609e7cc79c9001eb10bda16453f7fda3c64d8e78
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:41:14 2022 -0400
Update couch_replicator_small_max_request_size_target
Use the clustered version of the source and target endoints and switch to using
common test setup and teardown function functions. Overall it added to quite a
few number of lines saved.
---
...ch_replicator_small_max_request_size_target.erl | 158 ++++++---------------
1 file changed, 43 insertions(+), 115 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
index 3b020927d..4a905850d 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_small_max_request_size_target.erl
@@ -2,137 +2,61 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1,
- compare_dbs/3
-]).
+-include("couch_replicator_test.hrl").
-define(TIMEOUT_EUNIT, 360).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("chttpd", "max_http_request_size", "10000", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
reduce_max_request_size_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs when target has a small max_http_request_size",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_all_docs/2}
- || Pair <- Pairs
- ] ++
- [
- {Pair, fun should_replicate_one/2}
- || Pair <- Pairs
- ] ++
- % Disabled. See issue 574. Sometimes PUTs with a doc and
- % attachment which exceed maximum request size are simply
- % closed instead of returning a 413 request. That makes these
- % tests flaky.
- [
- {Pair, fun should_replicate_one_with_attachment/2}
- || Pair <- Pairs
- ]
+ ?TDEF_FE(should_replicate_all_docs, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one, ?TIMEOUT_EUNIT),
+ ?TDEF_FE(should_replicate_one_with_attachment, ?TIMEOUT_EUNIT)
+ ]
}
}.
% Test documents which are below max_http_request_size but when batched, batch size
% will be greater than max_http_request_size. Replicator could automatically split
% the batch into smaller batches and POST those separately.
-should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [])
- ]}
- }.
+should_replicate_all_docs({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, []).
% If a document is too large to post as a single request, that document is
% skipped but replication overall will make progress and not crash.
-should_replicate_one({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_one_small(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_one_small(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
% If a document has an attachment > 64 * 1024 bytes, replicator will switch to
% POST-ing individual documents directly and skip bulk_docs. Test that case
% separately
% See note in main test function why this was disabled.
-should_replicate_one_with_attachment({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source_one_large_attachment(Source),
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target, [<<"doc0">>])
- ]}
- }.
+should_replicate_one_with_attachment({_Ctx, {Source, Target}}) ->
+ config:set("chttpd", "max_http_request_size", "10000", false),
+ populate_source_one_large_attachment(Source),
+ populate_source(Source),
+ replicate(Source, Target),
+ compare(Source, Target, [<<"doc0">>]).
+
+populate_source(Source) ->
+ add_docs(Source, 5, 3000, 0).
+
+populate_source_one_large_one_small(Source) ->
+ one_large_one_small(Source, 12000, 3000).
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(add_docs(Source, 5, 3000, 0))}.
-
-should_populate_source_one_large_one_small({remote, Source}) ->
- should_populate_source_one_large_one_small(Source);
-should_populate_source_one_large_one_small(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_one_small(Source, 12000, 3000))}.
-
-should_populate_source_one_large_attachment({remote, Source}) ->
- should_populate_source_one_large_attachment(Source);
-should_populate_source_one_large_attachment(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(one_large_attachment(Source, 70000, 70000))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, {remote, Target}, ExceptIds) ->
- should_compare_databases(Source, Target, ExceptIds);
-should_compare_databases(Source, Target, ExceptIds) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target, ExceptIds))}.
+populate_source_one_large_attachment(Source) ->
+ one_large_attachment(Source, 70000, 70000).
binary_chunk(Size) when is_integer(Size), Size > 0 ->
<<<<"x">> || _ <- lists:seq(1, Size)>>.
@@ -155,11 +79,9 @@ one_large_attachment(DbName, Size, AttSize) ->
add_doc(DbName, <<"doc0">>, Size, AttSize).
add_doc(DbName, DocId, Size, AttSize) when is_binary(DocId) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Doc0 = #doc{id = DocId, body = {[{<<"x">>, binary_chunk(Size)}]}},
Doc = Doc0#doc{atts = atts(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
atts(0) ->
[];
@@ -173,12 +95,18 @@ atts(Size) ->
])
].
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- replicate(
+ couch_replicator_test_helper:replicate(
{[
- {<<"source">>, Source},
- {<<"target">>, Target},
- % This make batch_size predictable
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
+ % This makes batch_size more predictable
{<<"worker_processes">>, "1"}
]}
).
+
+compare(Source, Target, ExceptIds) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target, ExceptIds).
[couchdb] 04/18: Update couch_replicator_connection_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 620bdea367e3bce208e982ce43442ca72cc36988
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:13:44 2022 -0400
Update couch_replicator_connection_tests
The main changes are just using the TDEF_FE macros and removng the
_test(begin...end) silliness.
---
.../eunit/couch_replicator_connection_tests.erl | 264 ++++++++++-----------
1 file changed, 123 insertions(+), 141 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
index 7adbb6852..e8c90d02a 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_connection_tests.erl
@@ -13,9 +13,7 @@
-module(couch_replicator_connection_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
-
--define(TIMEOUT, 1000).
+-include("couch_replicator_test.hrl").
setup() ->
Host = config:get("httpd", "bind_address", "127.0.0.1"),
@@ -37,164 +35,148 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun connections_shared_after_release/1,
- fun connections_not_shared_after_owner_death/1,
- fun idle_connections_closed/1,
- fun test_owner_monitors/1,
- fun worker_discards_creds_on_create/1,
- fun worker_discards_url_creds_after_request/1,
- fun worker_discards_creds_in_headers_after_request/1,
- fun worker_discards_proxy_creds_after_request/1
+ ?TDEF_FE(connections_shared_after_release),
+ ?TDEF_FE(connections_not_shared_after_owner_death),
+ ?TDEF_FE(idle_connections_closed),
+ ?TDEF_FE(test_owner_monitors),
+ ?TDEF_FE(worker_discards_creds_on_create),
+ ?TDEF_FE(worker_discards_url_creds_after_request),
+ ?TDEF_FE(worker_discards_creds_in_headers_after_request),
+ ?TDEF_FE(worker_discards_proxy_creds_after_request)
]
}
}
}.
connections_shared_after_release({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection:release(Pid),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL)
- end),
- receive
- {ok, Pid2} ->
- ?assertEqual(Pid, Pid2)
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection:release(Pid),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL)
+ end),
+ receive
+ {ok, Pid2} ->
+ ?assertEqual(Pid, Pid2)
+ end.
connections_not_shared_after_owner_death({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- Self = self(),
- spawn(fun() ->
- Self ! couch_replicator_connection:acquire(URL),
- error("simulate division by zero without compiler warning")
- end),
- receive
- {ok, Pid} ->
- {ok, Pid2} = couch_replicator_connection:acquire(URL),
- ?assertNotEqual(Pid, Pid2),
- MRef = monitor(process, Pid),
- receive
- {'DOWN', MRef, process, Pid, _Reason} ->
- ?assert(not is_process_alive(Pid));
- Other ->
- throw(Other)
- end
- end
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ Self = self(),
+ spawn(fun() ->
+ Self ! couch_replicator_connection:acquire(URL),
+ error("simulate division by zero without compiler warning")
+ end),
+ receive
+ {ok, Pid} ->
+ {ok, Pid2} = couch_replicator_connection:acquire(URL),
+ ?assertNotEqual(Pid, Pid2),
+ MRef = monitor(process, Pid),
+ receive
+ {'DOWN', MRef, process, Pid, _Reason} ->
+ ?assert(not is_process_alive(Pid));
+ Other ->
+ throw(Other)
+ end
+ end.
idle_connections_closed({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Pid} = couch_replicator_connection:acquire(URL),
- couch_replicator_connection ! close_idle_connections,
- ?assert(ets:member(couch_replicator_connection, Pid)),
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- couch_replicator_connection:release(Pid),
- couch_replicator_connection ! close_idle_connections,
- % block until idle connections have closed
- sys:get_status(couch_replicator_connection),
- ?assert(not ets:member(couch_replicator_connection, Pid))
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Pid} = couch_replicator_connection:acquire(URL),
+ couch_replicator_connection ! close_idle_connections,
+ ?assert(ets:member(couch_replicator_connection, Pid)),
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ couch_replicator_connection:release(Pid),
+ couch_replicator_connection ! close_idle_connections,
+ % block until idle connections have closed
+ sys:get_status(couch_replicator_connection),
+ ?assert(not ets:member(couch_replicator_connection, Pid)).
test_owner_monitors({Host, Port}) ->
- ?_test(begin
- URL = "http://" ++ Host ++ ":" ++ Port,
- {ok, Worker0} = couch_replicator_connection:acquire(URL),
- assert_monitors_equal([{process, self()}]),
- couch_replicator_connection:release(Worker0),
- assert_monitors_equal([]),
- {Workers, Monitors} = lists:foldl(
- fun(_, {WAcc, MAcc}) ->
- {ok, Worker1} = couch_replicator_connection:acquire(URL),
- MAcc1 = [{process, self()} | MAcc],
- assert_monitors_equal(MAcc1),
- {[Worker1 | WAcc], MAcc1}
- end,
- {[], []},
- lists:seq(1, 5)
- ),
- lists:foldl(
- fun(Worker2, Acc) ->
- [_ | NewAcc] = Acc,
- couch_replicator_connection:release(Worker2),
- assert_monitors_equal(NewAcc),
- NewAcc
- end,
- Monitors,
- Workers
- )
- end).
+ URL = "http://" ++ Host ++ ":" ++ Port,
+ {ok, Worker0} = couch_replicator_connection:acquire(URL),
+ assert_monitors_equal([{process, self()}]),
+ couch_replicator_connection:release(Worker0),
+ assert_monitors_equal([]),
+ {Workers, Monitors} = lists:foldl(
+ fun(_, {WAcc, MAcc}) ->
+ {ok, Worker1} = couch_replicator_connection:acquire(URL),
+ MAcc1 = [{process, self()} | MAcc],
+ assert_monitors_equal(MAcc1),
+ {[Worker1 | WAcc], MAcc1}
+ end,
+ {[], []},
+ lists:seq(1, 5)
+ ),
+ lists:foldl(
+ fun(Worker2, Acc) ->
+ [_ | NewAcc] = Acc,
+ couch_replicator_connection:release(Worker2),
+ assert_monitors_equal(NewAcc),
+ NewAcc
+ end,
+ Monitors,
+ Workers
+ ).
worker_discards_creds_on_create({Host, Port}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ Port,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0).
worker_discards_url_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ User ++ ":" ++ Pass ++ "@" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_creds_in_headers_after_request({Host, _}) ->
- ?_test(begin
- {_User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Headers = [{"Authorization", "Basic " ++ B64Auth}],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {_User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Headers = [{"Authorization", "Basic " ++ B64Auth}],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, Headers, [])),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
worker_discards_proxy_creds_after_request({Host, _}) ->
- ?_test(begin
- {User, Pass, B64Auth} = user_pass(),
- {Port, ServerPid} = server(),
- PortStr = integer_to_list(Port),
- URL = "http://" ++ Host ++ ":" ++ PortStr,
- {ok, WPid} = couch_replicator_connection:acquire(URL),
- Opts = [
- {proxy_host, Host},
- {proxy_port, Port},
- {proxy_user, User},
- {proxy_pass, Pass}
- ],
- ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
- Internals = worker_internals(WPid),
- ?assert(string:str(Internals, B64Auth) =:= 0),
- ?assert(string:str(Internals, Pass) =:= 0),
- couch_replicator_connection:release(WPid),
- unlink(ServerPid),
- exit(ServerPid, kill)
- end).
+ {User, Pass, B64Auth} = user_pass(),
+ {Port, ServerPid} = server(),
+ PortStr = integer_to_list(Port),
+ URL = "http://" ++ Host ++ ":" ++ PortStr,
+ {ok, WPid} = couch_replicator_connection:acquire(URL),
+ Opts = [
+ {proxy_host, Host},
+ {proxy_port, Port},
+ {proxy_user, User},
+ {proxy_pass, Pass}
+ ],
+ ?assertMatch({ok, "200", _, _}, send_req(WPid, URL, [], Opts)),
+ Internals = worker_internals(WPid),
+ ?assert(string:str(Internals, B64Auth) =:= 0),
+ ?assert(string:str(Internals, Pass) =:= 0),
+ couch_replicator_connection:release(WPid),
+ unlink(ServerPid),
+ exit(ServerPid, kill).
send_req(WPid, URL, Headers, Opts) ->
ibrowse:send_req_direct(WPid, URL, Headers, get, [], Opts).
[couchdb] 14/18: Update couch_replicator_rate_limiter_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit d831ec70edb709b5f672adf1fabf4c7869f50d26
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:35:50 2022 -0400
Update couch_replicator_rate_limiter_tests
Use the TDEF_FE macro and cleanup ?_test(begin...end) instances.
---
.../eunit/couch_replicator_rate_limiter_tests.erl | 77 ++++++++++------------
1 file changed, 33 insertions(+), 44 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
index a214d4607..0b7c48b62 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_rate_limiter_tests.erl
@@ -1,6 +1,7 @@
-module(couch_replicator_rate_limiter_tests).
-include_lib("couch/include/couch_eunit.hrl").
+-include("couch_replicator_test.hrl").
rate_limiter_test_() ->
{
@@ -8,58 +9,46 @@ rate_limiter_test_() ->
fun setup/0,
fun teardown/1,
[
- t_new_key(),
- t_1_failure(),
- t_2_failures_back_to_back(),
- t_2_failures(),
- t_success_threshold(),
- t_1_failure_2_successes()
+ ?TDEF_FE(t_new_key),
+ ?TDEF_FE(t_1_failure),
+ ?TDEF_FE(t_2_failures_back_to_back),
+ ?TDEF_FE(t_2_failures),
+ ?TDEF_FE(t_success_threshold),
+ ?TDEF_FE(t_1_failure_2_successes)
]
}.
-t_new_key() ->
- ?_test(begin
- ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get}))
- end).
+t_new_key(_) ->
+ ?assertEqual(0, couch_replicator_rate_limiter:interval({"foo", get})).
-t_1_failure() ->
- ?_test(begin
- ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get}))
- end).
+t_1_failure(_) ->
+ ?assertEqual(24, couch_replicator_rate_limiter:failure({"foo", get})).
-t_2_failures() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(29, Interval)
- end).
+t_2_failures(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(29, Interval).
-t_2_failures_back_to_back() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- Interval = couch_replicator_rate_limiter:failure({"foo", get}),
- ?assertEqual(24, Interval)
- end).
+t_2_failures_back_to_back(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ Interval = couch_replicator_rate_limiter:failure({"foo", get}),
+ ?assertEqual(24, Interval).
-t_success_threshold() ->
- ?_test(begin
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval),
- Interval = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Interval)
- end).
+t_success_threshold(_) ->
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval),
+ Interval = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Interval).
-t_1_failure_2_successes() ->
- ?_test(begin
- couch_replicator_rate_limiter:failure({"foo", get}),
- low_pass_filter_delay(),
- Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(20, Succ1),
- low_pass_filter_delay(),
- Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
- ?assertEqual(0, Succ2)
- end).
+t_1_failure_2_successes(_) ->
+ couch_replicator_rate_limiter:failure({"foo", get}),
+ low_pass_filter_delay(),
+ Succ1 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(20, Succ1),
+ low_pass_filter_delay(),
+ Succ2 = couch_replicator_rate_limiter:success({"foo", get}),
+ ?assertEqual(0, Succ2).
low_pass_filter_delay() ->
timer:sleep(100).
[couchdb] 07/18: Update couch_replicator_filtered_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit cad96318fc41f62c67b7fd4cc75b01ebdd2c4ba3
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:20:15 2022 -0400
Update couch_replicator_filtered_tests
Take advantage of the helper setup and teardown functions.
Switching to a simpler TDEF_FE macro instead of foreachx and inorder setup
cruft also saves some lines of code.
---
.../test/eunit/couch_replicator_filtered_tests.erl | 257 ++++++++-------------
1 file changed, 100 insertions(+), 157 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
index b77b83daa..267c4fab6 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_filtered_tests.erl
@@ -14,7 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DDOC,
{[
@@ -57,206 +57,149 @@
]}
).
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
filtered_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_succeed_with_query),
+ ?TDEF_FE(should_succeed_with_view)
+ ]
}
}.
-query_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with query replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_query/2} || Pair <- Pairs]
- }
- }.
-
-view_filtered_replication_test_() ->
- Pairs = [{remote, remote}],
- {
- "Filtered with a view replication tests",
- {
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed_with_view/2} || Pair <- Pairs]
- }
- }.
-
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/testfilter">>}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ replicate(RepObject),
%% FilteredFun is an Erlang version of following JS function
%% function(doc, req){if (doc.class == 'mammal') return true;}
- FilterFun = fun(_DocId, {Props}) ->
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs,
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_query({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"filter_ddoc/queryfilter">>},
{<<"query_params">>,
{[
{<<"starts">>, <<"a">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
case couch_util:get_value(<<"class">>, Props) of
<<"a", _/binary>> -> true;
_ -> false
end
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(2, TargetDocCount),
+ % All the docs filtered as expected,
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
-should_succeed_with_view({From, To}, {_Ctx, {Source, Target}}) ->
+should_succeed_with_view({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
RepObject =
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"filter">>, <<"_view">>},
{<<"query_params">>,
{[
{<<"view">>, <<"filter_ddoc/mammals">>}
]}}
]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- FilterFun = fun(_DocId, {Props}) ->
+ replicate(RepObject),
+ FilterFun = fun(_DocId, #doc{body = {Props}}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"Target DB doesn't have deleted docs",
- ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
- {"All the docs filtered as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs filtered as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
- DDoc = couch_doc:from_json_obj(?DDOC),
- Doc1 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc1">>},
- {<<"class">>, <<"mammal">>},
- {<<"value">>, 1}
- ]}
- ),
- Doc2 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc2">>},
- {<<"class">>, <<"amphibians">>},
- {<<"value">>, 2}
- ]}
- ),
- Doc3 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc3">>},
- {<<"class">>, <<"reptiles">>},
- {<<"value">>, 3}
- ]}
- ),
- Doc4 = couch_doc:from_json_obj(
- {[
- {<<"_id">>, <<"doc4">>},
- {<<"class">>, <<"arthropods">>},
- {<<"value">>, 2}
- ]}
- ),
- {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3, Doc4]),
- couch_db:close(Db).
+ Docs = [
+ couch_doc:from_json_obj(?DDOC),
+ #doc{
+ id = <<"doc1">>,
+ body =
+ {[
+ {<<"class">>, <<"mammal">>},
+ {<<"value">>, 1}
+ ]}
+ },
+ #doc{
+ id = <<"doc2">>,
+ body =
+ {[
+ {<<"class">>, <<"amphibians">>},
+ {<<"value">>, 2}
+ ]}
+ },
+ #doc{
+ id = <<"doc3">>,
+ body =
+ {[
+ {<<"class">>, <<"reptiles">>},
+ {<<"value">>, 3}
+ ]}
+ },
+ #doc{
+ id = <<"doc4">>,
+ body =
+ {[
+ {<<"class">>, <<"arthropods">>},
+ {<<"value">>, 2}
+ ]}
+ }
+ ],
+ {ok, [_ | _]} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 05/18: Update couch_replicator_create_target_with_options_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 31185647f5a1483abe8b0cd4b7da114a15869d8b
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:15:02 2022 -0400
Update couch_replicator_create_target_with_options_tests
Start using the common setup and tear down functions from the test helper.
Also using the test definitions to use the TDEF_FE macro.
Since the setup function already creates a target endpoint database and the
test is also in charge of creating test database, we just remove the target db
before the replication jobs start.
---
...replicator_create_target_with_options_tests.erl | 74 +++++-----------------
1 file changed, 17 insertions(+), 57 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
index 8adcd25bd..fabb6480e 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_create_target_with_options_tests.erl
@@ -14,35 +14,20 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
--define(USERNAME, "rep_admin").
--define(PASSWORD, "secret").
-
-setup() ->
- Ctx = test_util:start_couch([fabric, mem3, couch_replicator, chttpd]),
- Hashed = couch_passwords:hash_admin_password(?PASSWORD),
- ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
- Source = ?tempdb(),
- Target = ?tempdb(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {_Source, _Target}}) ->
- config:delete("admins", ?USERNAME),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
create_target_with_options_replication_test_() ->
{
"Create target with range partitions tests",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_create_target_with_q_4/1,
- fun should_create_target_with_q_2_n_1/1,
- fun should_create_target_with_default/1,
- fun should_not_create_target_with_q_any/1
+ ?TDEF_FE(should_create_target_with_q_4),
+ ?TDEF_FE(should_create_target_with_q_2_n_1),
+ ?TDEF_FE(should_create_target_with_default),
+ ?TDEF_FE(should_not_create_target_with_q_any)
]
}
}.
@@ -55,15 +40,12 @@ should_create_target_with_q_4({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"4">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(4, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(4, couch_util:get_value(q, ClusterInfo)).
should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
RepObject =
@@ -73,18 +55,13 @@ should_create_target_with_q_2_n_1({_Ctx, {Source, Target}}) ->
{<<"create_target">>, true},
{<<"create_target_params">>, {[{<<"q">>, <<"2">>}, {<<"n">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
- delete_db(Source),
- delete_db(Target),
- [
- ?_assertEqual(2, couch_util:get_value(q, ClusterInfo)),
- ?_assertEqual(1, couch_util:get_value(n, ClusterInfo))
- ].
+ ?assertEqual(2, couch_util:get_value(q, ClusterInfo)),
+ ?assertEqual(1, couch_util:get_value(n, ClusterInfo)).
should_create_target_with_default({_Ctx, {Source, Target}}) ->
RepObject =
@@ -93,16 +70,13 @@ should_create_target_with_default({_Ctx, {Source, Target}}) ->
{<<"target">>, db_url(Target)},
{<<"create_target">>, true}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
-
{ok, TargetInfo} = fabric:get_db_info(Target),
{ClusterInfo} = couch_util:get_value(cluster, TargetInfo),
Q = config:get_integer("cluster", "q", 2),
- delete_db(Source),
- delete_db(Target),
- ?_assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
+ ?assertEqual(Q, couch_util:get_value(q, ClusterInfo)).
should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
RepObject =
@@ -112,33 +86,19 @@ should_not_create_target_with_q_any({_Ctx, {Source, Target}}) ->
{<<"create_target">>, false},
{<<"create_target_params">>, {[{<<"q">>, <<"1">>}]}}
]},
- create_db(Source),
create_doc(Source),
+ delete_db(Target),
{error, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
DbExist = is_list(catch mem3:shards(Target)),
- delete_db(Source),
- ?_assertEqual(false, DbExist).
+ ?assertEqual(false, DbExist).
create_doc(DbName) ->
Body = {[{<<"foo">>, <<"bar">>}]},
NewDoc = #doc{body = Body},
{ok, _} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]).
-create_db(DbName) ->
- ok = fabric:create_db(DbName, [?ADMIN_CTX]).
-
delete_db(DbName) ->
ok = fabric:delete_db(DbName, [?ADMIN_CTX]).
db_url(DbName) ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- ?l2b(
- io_lib:format("http://~s:~s@~s:~b/~s", [
- ?USERNAME,
- ?PASSWORD,
- Addr,
- Port,
- DbName
- ])
- ).
+ couch_replicator_test_helper:cluster_db_url(DbName).
[couchdb] 10/18: Update couch_replicator_large_atts_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9a6875f6b2e1b83bd4128f72fc4c4d01d14dfc7c
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:25:37 2022 -0400
Update couch_replicator_large_atts_tests
Use commong setup functions and TDEF_FE macro.
Removing the foreachx and the remote vs local junk really trimmed down the
size. The test content was tiny compared to the clunky EUnit setup logic.
---
.../eunit/couch_replicator_large_atts_tests.erl | 92 ++++++----------------
1 file changed, 23 insertions(+), 69 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
index 2f0e2a1f0..8190c7205 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_large_atts_tests.erl
@@ -14,12 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(ATT_SIZE_1, 2 * 1024 * 1024).
-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
@@ -27,83 +22,37 @@
-define(TIMEOUT_EUNIT, 120).
setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- config:set("attachments", "compressible_types", "text/*", false),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ Ctx = couch_replicator_test_helper:test_setup(),
+ config:set("attachments", "compressible_types", "text/*", _Persist = false),
+ Ctx.
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
-
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+teardown(Ctx) ->
+ config:delete("attachments", "compressible_types", _Persist = false),
+ couch_replicator_test_helper:test_teardown(Ctx).
large_atts_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with large attachments",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun setup/0,
+ fun teardown/1,
[
- {Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs
+ ?TDEF_FE(should_replicate_atts, ?TIMEOUT_EUNIT)
]
}
}.
-should_populate_replicate_compact({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+should_replicate_atts({_Ctx, {Source, Target}}) ->
+ populate_db(Source, ?DOCS_COUNT),
+ ?assertEqual(ok, replicate(Source, Target)),
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Doc = #doc{
- id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ id = integer_to_binary(DocIdCounter),
body = {[]},
atts = [
att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
@@ -115,8 +64,7 @@ populate_db(DbName, DocCount) ->
[],
lists:seq(1, DocCount)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
att(Name, Size, Type) ->
couch_att:new([
@@ -125,3 +73,9 @@ att(Name, Size, Type) ->
{att_len, Size},
{data, fun(Count) -> crypto:strong_rand_bytes(Count) end}
]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 13/18: Update couch_replicator_proxy_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 9243298efe65ac71a38f443009423be7becce716
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:34:26 2022 -0400
Update couch_replicator_proxy_tests
Use the TDEF_FE macro and remove the ugly ?_test(begin...end) construct.
---
.../test/eunit/couch_replicator_proxy_tests.erl | 137 ++++++++++-----------
1 file changed, 64 insertions(+), 73 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
index ca1816b33..184d81aaf 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_proxy_tests.erl
@@ -15,6 +15,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-include_lib("couch_replicator/include/couch_replicator_api_wrap.hrl").
+-include("couch_replicator_test.hrl").
setup() ->
ok.
@@ -34,90 +35,80 @@ replicator_proxy_test_() ->
fun setup/0,
fun teardown/1,
[
- fun parse_rep_doc_without_proxy/1,
- fun parse_rep_doc_with_proxy/1,
- fun parse_rep_source_target_proxy/1,
- fun mutually_exclusive_proxy_and_source_proxy/1,
- fun mutually_exclusive_proxy_and_target_proxy/1
+ ?TDEF_FE(parse_rep_doc_without_proxy),
+ ?TDEF_FE(parse_rep_doc_with_proxy),
+ ?TDEF_FE(parse_rep_source_target_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_source_proxy),
+ ?TDEF_FE(mutually_exclusive_proxy_and_target_proxy)
]
}
}
}.
parse_rep_doc_without_proxy(_) ->
- ?_test(begin
- NoProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined)
- end).
+ NoProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(NoProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, undefined),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, undefined).
parse_rep_doc_with_proxy(_) ->
- ?_test(begin
- ProxyURL = <<"http://myproxy.com">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, ProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
- ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL))
- end).
+ ProxyURL = <<"http://myproxy.com">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, ProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual((Rep#rep.source)#httpdb.proxy_url, binary_to_list(ProxyURL)),
+ ?assertEqual((Rep#rep.target)#httpdb.proxy_url, binary_to_list(ProxyURL)).
parse_rep_source_target_proxy(_) ->
- ?_test(begin
- SrcProxyURL = <<"http://mysrcproxy.com">>,
- TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"source_proxy">>, SrcProxyURL},
- {<<"target_proxy">>, TgtProxyURL}
- ]},
- Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
- ?assertEqual(
- (Rep#rep.source)#httpdb.proxy_url,
- binary_to_list(SrcProxyURL)
- ),
- ?assertEqual(
- (Rep#rep.target)#httpdb.proxy_url,
- binary_to_list(TgtProxyURL)
- )
- end).
+ SrcProxyURL = <<"http://mysrcproxy.com">>,
+ TgtProxyURL = <<"http://mytgtproxy.com:9999">>,
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"source_proxy">>, SrcProxyURL},
+ {<<"target_proxy">>, TgtProxyURL}
+ ]},
+ Rep = couch_replicator_docs:parse_rep_doc(ProxyDoc),
+ ?assertEqual(
+ (Rep#rep.source)#httpdb.proxy_url,
+ binary_to_list(SrcProxyURL)
+ ),
+ ?assertEqual(
+ (Rep#rep.target)#httpdb.proxy_url,
+ binary_to_list(TgtProxyURL)
+ ).
mutually_exclusive_proxy_and_source_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"source_proxy">>, <<"sourceproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"source_proxy">>, <<"sourceproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
mutually_exclusive_proxy_and_target_proxy(_) ->
- ?_test(begin
- ProxyDoc =
- {[
- {<<"source">>, <<"http://unproxied.com">>},
- {<<"target">>, <<"http://otherunproxied.com">>},
- {<<"proxy">>, <<"oldstyleproxy.local">>},
- {<<"target_proxy">>, <<"targetproxy.local">>}
- ]},
- ?assertThrow(
- {bad_rep_doc, _},
- couch_replicator_docs:parse_rep_doc(ProxyDoc)
- )
- end).
+ ProxyDoc =
+ {[
+ {<<"source">>, <<"http://unproxied.com">>},
+ {<<"target">>, <<"http://otherunproxied.com">>},
+ {<<"proxy">>, <<"oldstyleproxy.local">>},
+ {<<"target_proxy">>, <<"targetproxy.local">>}
+ ]},
+ ?assertThrow(
+ {bad_rep_doc, _},
+ couch_replicator_docs:parse_rep_doc(ProxyDoc)
+ ).
[couchdb] 16/18: Update couch_replicator_selector_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 8e6158972149eb01548a28cf8c4f82d739b621ee
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:39:17 2022 -0400
Update couch_replicator_selector_tests
Use the clustered versions of endpoints for the test.
Also, use the common setup and teardown helpers and remove the foreachx
silliness.
---
.../test/eunit/couch_replicator_selector_tests.erl | 121 ++++++++-------------
1 file changed, 44 insertions(+), 77 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
index 8f61a638c..31c5dc93c 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_selector_tests.erl
@@ -14,92 +14,62 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_docs(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
selector_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Selector filtered replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_docs(Source),
+ replicate(
{[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"selector">>, {[{<<"_id">>, <<"doc2">>}]}}
- ]},
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ ]}
+ ),
%% FilteredFun is an Erlang version of following mango selector
- FilterFun = fun(_DocId, {Props}) ->
- couch_util:get_value(<<"_id">>, Props) == <<"doc2">>
+ FilterFun = fun(DocId, #doc{}) ->
+ DocId == <<"doc2">>
end,
- {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
- {"Target DB has proper number of docs",
- ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
- {"All the docs selected as expected",
- ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
- ]}.
+ {TargetDocCount, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ % Target DB has proper number of docs
+ ?assertEqual(1, TargetDocCount),
+ % All the docs selected as expected
+ ?assert(lists:all(fun(Valid) -> Valid end, AllReplies)).
compare_dbs(Source, Target, FilterFun) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
- TargetReply = read_doc(TargetDb, DocId),
- case FilterFun(DocId, SourceDoc) of
- true ->
- ValidReply = {ok, DocId, SourceDoc} == TargetReply,
- {ok, [ValidReply | Acc]};
- false ->
- ValidReply = {not_found, missing} == TargetReply,
- {ok, [ValidReply | Acc]}
- end
- end,
- {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb),
- {ok, TargetDbInfo, AllReplies}.
-
-read_doc(Db, DocIdOrInfo) ->
- case couch_db:open_doc(Db, DocIdOrInfo) of
- {ok, Doc} ->
- {Props} = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- {ok, DocId, {Props}};
- Error ->
- Error
- end.
+ {ok, TargetDocCount} = fabric:get_doc_count(Target),
+ Replies = lists:foldl(
+ fun({Id, Rev}, Acc) ->
+ SrcDoc = read_doc(Source, Id, Rev),
+ TgtDoc = read_doc(Target, Id, Rev),
+ case FilterFun(Id, SrcDoc) of
+ true ->
+ [is_record(TgtDoc, doc) | Acc];
+ false ->
+ [TgtDoc =:= not_found | Acc]
+ end
+ end,
+ [],
+ couch_replicator_test_helper:cluster_doc_revs(Source)
+ ),
+ {TargetDocCount, Replies}.
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+read_doc(Db, DocId, Rev) ->
+ couch_replicator_test_helper:cluster_open_rev(Db, DocId, Rev).
create_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc1 = couch_doc:from_json_obj(
{[
{<<"_id">>, <<"doc1">>}
@@ -110,13 +80,10 @@ create_docs(DbName) ->
{<<"_id">>, <<"doc2">>}
]}
),
- {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2]),
- couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, [Doc1, Doc2], [?ADMIN_CTX]).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(RepObject) ->
+ couch_replicator_test_helper:replicate(RepObject).
[couchdb] 12/18: Update couch_replicator_missing_stubs_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 033b8d5de026fb6c71bde0246e1b03c2933e56ff
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:31:46 2022 -0400
Update couch_replicator_missing_stubs_tests
Use common setup and teardown helpers, TDEF_FE macros and remove all the
foreachx nonsense.
---
.../eunit/couch_replicator_missing_stubs_tests.erl | 146 ++++++---------------
1 file changed, 39 insertions(+), 107 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
index ff3b5ee98..b0e205729 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_missing_stubs_tests.erl
@@ -14,106 +14,34 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/2,
- compare_dbs/2
-]).
+-include("couch_replicator_test.hrl").
-define(REVS_LIMIT, 3).
-define(TIMEOUT_EUNIT, 30).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
- {Ctx, {Source, Target}}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
- ok = application:stop(couch_replicator),
- ok = test_util:stop_couch(Ctx).
-
missing_stubs_test_() ->
- Pairs = [{remote, remote}],
{
"Replicate docs with missing stubs (COUCHDB-1365)",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- {Pair, fun should_replicate_docs_with_missed_att_stubs/2}
- || Pair <- Pairs
+ ?TDEF_FE(replicate_docs_with_missing_att_stubs, ?TIMEOUT_EUNIT)
]
}
}.
-should_replicate_docs_with_missed_att_stubs({From, To}, {_Ctx, {Source, Target}}) ->
- {
- lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_set_target_revs_limit(Target, ?REVS_LIMIT),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target),
- should_update_source_docs(Source, ?REVS_LIMIT * 2),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}
- }.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_set_target_revs_limit({remote, Target}, RevsLimit) ->
- should_set_target_revs_limit(Target, RevsLimit);
-should_set_target_revs_limit(Target, RevsLimit) ->
- ?_test(begin
- {ok, Db} = couch_db:open_int(Target, [?ADMIN_CTX]),
- ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
- ok = couch_db:close(Db)
- end).
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-should_update_source_docs({remote, Source}, Times) ->
- should_update_source_docs(Source, Times);
-should_update_source_docs(Source, Times) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+replicate_docs_with_missing_att_stubs({_Ctx, {Source, Target}}) ->
+ populate_db(Source),
+ fabric:set_revs_limit(Target, ?REVS_LIMIT, [?ADMIN_CTX]),
+ replicate(Source, Target),
+ compare(Source, Target),
+ update_docs(Source, ?REVS_LIMIT * 2),
+ replicate(Source, Target),
+ compare(Source, Target).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
AttData = crypto:strong_rand_bytes(6000),
Doc = #doc{
id = <<"doc1">>,
@@ -126,34 +54,38 @@ populate_db(DbName) ->
])
]
},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
-update_db_docs(DbName, Times) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _} = couch_db:fold_docs(
- Db,
- fun(FDI, Acc) -> db_fold_fun(FDI, Acc) end,
- {DbName, Times},
- []
- ),
- ok = couch_db:close(Db).
+update_docs(DbName, Times) ->
+ lists:foreach(
+ fun({Id, _Rev}) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ update_doc(DbName, Doc, Times)
+ end,
+ couch_replicator_test_helper:cluster_doc_revs(DbName)
+ ).
-db_fold_fun(FullDocInfo, {DbName, Times}) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+update_doc(DbName, Doc, Times) ->
+ {Pos0, [Rev0 | _]} = Doc#doc.revs,
lists:foldl(
fun(_, {Pos, RevId}) ->
- {ok, Db2} = couch_db:reopen(Db),
- NewDocVersion = Doc#doc{
+ Val = base64:encode(crypto:strong_rand_bytes(100)),
+ NewDoc = Doc#doc{
revs = {Pos, [RevId]},
- body = {[{<<"value">>, base64:encode(crypto:strong_rand_bytes(100))}]}
+ body = {[{<<"value">>, Val}]}
},
- {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+ {ok, NewRev} = fabric:update_doc(DbName, NewDoc, [?ADMIN_CTX]),
NewRev
end,
- {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+ {Pos0, Rev0},
lists:seq(1, Times)
- ),
- ok = couch_db:close(Db),
- {ok, {DbName, Times}}.
+ ).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
+
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 11/18: Update couch_replicator_many_leaves_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit ca46fa61c728004ead41f768bfc85eb30861360e
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:28:13 2022 -0400
Update couch_replicator_many_leaves_tests
Use comon setup and teardown function and the TDEF_FE macros.
Also, remove quite a bit of foreachx and remote boilerplate which is not needed
any longer.
Most of the changes however consisted in update all the db operations to use
fabric instead of couch. Luckily, most of those have fabric equivalents, and
fabric calls are even shorter as they don't need open, re-open and close
operations.
---
.../eunit/couch_replicator_many_leaves_tests.erl | 190 ++++++++-------------
1 file changed, 67 insertions(+), 123 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
index cdc90e2ea..aa5752650 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_many_leaves_tests.erl
@@ -14,11 +14,7 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-
--import(couch_replicator_test_helper, [
- db_url/1,
- replicate/1
-]).
+-include("couch_replicator_test.hrl").
-define(DOCS_CONFLICTS, [
{<<"doc1">>, 10},
@@ -32,35 +28,15 @@
-define(i2l(I), integer_to_list(I)).
-define(io2b(Io), iolist_to_binary(Io)).
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
-
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-
-setup() ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = setup_db(),
- Target = setup_db(),
- {Ctx, {Source, Target}}.
-
-teardown({Ctx, {Source, Target}}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok = test_util:stop_couch(Ctx).
-
docs_with_many_leaves_test_() ->
{
"Replicate documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_populate_replicate_compact/1
+ ?TDEF_FE(should_populate_replicate_compact, ?TIMEOUT_EUNIT)
]
}
}.
@@ -70,72 +46,40 @@ docs_with_many_leaves_test_winning_revs_only_test_() ->
"Replicate winning revs only for documents with many leaves",
{
foreach,
- fun setup/0,
- fun teardown/1,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
[
- fun should_replicate_winning_revs_only/1
+ ?TDEF_FE(should_replicate_winning_revs_only, ?TIMEOUT_EUNIT)
]
}
}.
should_populate_replicate_compact({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target, all_revs)
- ]}.
+ populate_db(Source),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, []),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, all_revs).
should_replicate_winning_revs_only({_Ctx, {Source, Target}}) ->
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
- should_verify_target(Source, Target, winning_revs)
- ]}.
-
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate(Source, Target) ->
- should_replicate(Source, Target, []).
-
-should_replicate(Source, Target, Options) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- RepObj = {
- [
- {<<"source">>, db_url(Source)},
- {<<"target">>, db_url(Target)}
- ] ++ Options
- },
- replicate(RepObj)
- end)}.
-
-should_verify_target(Source, Target, Mode) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS, Mode),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
-
-should_add_attachments_to_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT,
- ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, [?ADMIN_CTX]),
- add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
- ok = couch_db:close(SourceDb)
- end)}.
+ populate_db(Source),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs),
+ add_attachments(Source, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ replicate(Source, Target, [{<<"winning_revs_only">>, true}]),
+ verify_target(Source, Target, ?DOCS_CONFLICTS, winning_revs).
+
+replicate(Source, Target, Options) ->
+ RepObj = {
+ [
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)}
+ ] ++ Options
+ },
+ couch_replicator_test_helper:replicate(RepObj).
populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
lists:foreach(
fun({DocId, NumConflicts}) ->
Value = <<"0">>,
@@ -143,22 +87,20 @@ populate_db(DbName) ->
id = DocId,
body = {[{<<"value">>, Value}]}
},
- {ok, {Pos, Rev}} = couch_db:update_doc(Db, Doc, [?ADMIN_CTX]),
+ {ok, {Pos, Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]),
% Update first initial doc rev twice to ensure it's always a winner
- {ok, Db2} = couch_db:reopen(Db),
Doc1 = Doc#doc{revs = {Pos, [Rev]}},
- {ok, _} = couch_db:update_doc(Db2, Doc1, [?ADMIN_CTX]),
- {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ {ok, _} = fabric:update_doc(DbName, Doc1, [?ADMIN_CTX]),
+ {ok, _} = add_doc_siblings(DbName, DocId, NumConflicts)
end,
?DOCS_CONFLICTS
- ),
- couch_db:close(Db).
+ ).
add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
add_doc_siblings(Db, DocId, NumLeaves, [], []).
add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], ?REPLICATED_CHANGES),
+ {ok, []} = fabric:update_docs(Db, AccDocs, [?REPLICATED_CHANGES, ?ADMIN_CTX]),
{ok, AccRevs};
add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
Value = ?l2b(?i2l(NumLeaves)),
@@ -179,18 +121,8 @@ add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
verify_target(_SourceDb, _TargetDb, [], _Mode) ->
ok;
verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
- {ok, TargetLookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]
- ),
+ SourceLookups = open_revs_conflicts(SourceDb, DocId),
+ TargetLookups = open_revs_conflicts(TargetDb, DocId),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
Total = NumConflicts + 1,
@@ -205,14 +137,13 @@ verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest], all_revs) ->
),
verify_target(SourceDb, TargetDb, Rest, all_revs);
verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs) ->
- {ok, SourceWinner} = couch_db:open_doc(SourceDb, DocId),
- {ok, TargetWinner} = couch_db:open_doc(TargetDb, DocId),
+ SourceWinner = open_doc(SourceDb, DocId),
+ TargetWinner = open_doc(TargetDb, DocId),
SourceWinnerJson = couch_doc:to_json_obj(SourceWinner, [attachments]),
TargetWinnerJson = couch_doc:to_json_obj(TargetWinner, [attachments]),
% Source winner is the same as the target winner
?assertEqual(SourceWinnerJson, TargetWinnerJson),
- Opts = [conflicts, deleted_conflicts],
- {ok, TargetAll} = couch_db:open_doc_revs(TargetDb, DocId, all, Opts),
+ TargetAll = open_revs_conflicts(TargetDb, DocId),
% There is only one version on the target
?assert(length(TargetAll) == 1),
verify_target(SourceDb, TargetDb, Rest, winning_revs).
@@ -220,7 +151,7 @@ verify_target(SourceDb, TargetDb, [{DocId, _NumConflicts} | Rest], winning_revs)
add_attachments(_SourceDb, _NumAtts, []) ->
ok;
add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceLookups = open_revs(SourceDb, DocId, []),
SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
Total = NumConflicts + 1,
?assertEqual(Total, length(SourceDocs)),
@@ -228,20 +159,7 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
NewAtts = lists:foldl(
fun(I, AttAcc) ->
- AttData = crypto:strong_rand_bytes(100),
- NewAtt = couch_att:new([
- {name,
- ?io2b([
- "att_",
- ?i2l(I),
- "_",
- couch_doc:rev_to_str({Pos, Rev})
- ])},
- {type, <<"application/foobar">>},
- {att_len, byte_size(AttData)},
- {data, AttData}
- ]),
- [NewAtt | AttAcc]
+ [make_att(I, Pos, Rev, 100) | AttAcc]
end,
[],
lists:seq(1, NumAtts)
@@ -251,7 +169,33 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
[],
SourceDocs
),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ {ok, UpdateResults} = fabric:update_docs(SourceDb, NewDocs, [?ADMIN_CTX]),
NewRevs = [R || {ok, R} <- UpdateResults],
?assertEqual(length(NewDocs), length(NewRevs)),
add_attachments(SourceDb, NumAtts, Rest).
+
+make_att(Id, Pos, Rev, Size) ->
+ AttData = crypto:strong_rand_bytes(Size),
+ RevStr = couch_doc:rev_to_str({Pos, Rev}),
+ couch_att:new([
+ {name, ?io2b(["att_", ?i2l(Id), "_", RevStr])},
+ {type, <<"application/foobar">>},
+ {att_len, byte_size(AttData)},
+ {data, AttData}
+ ]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+open_revs_conflicts(DbName, Id) ->
+ Opts = [conflicts, deleted_conflicts],
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_revs(DbName, Id, Opts) ->
+ {ok, Lookups} = fabric:open_revs(DbName, Id, all, Opts),
+ Lookups.
+
+open_doc(DbName, Id) ->
+ {ok, Doc} = fabric:open_doc(DbName, Id, [?ADMIN_CTX]),
+ Doc.
[couchdb] 15/18: Update couch_replicator_retain_stats_between_job_runs
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1e69eaac9f77411522766a8e8bd7045ffd75eff9
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:37:32 2022 -0400
Update couch_replicator_retain_stats_between_job_runs
Switch the test to use the clustered endpoints.
Use the common test setup and teardown functions as well as the TDEF_FE macros.
---
...ch_replicator_retain_stats_between_job_runs.erl | 153 ++++++++-------------
1 file changed, 54 insertions(+), 99 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
index 9ffcc9e2c..1da4dfa02 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_retain_stats_between_job_runs.erl
@@ -15,104 +15,70 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-include("couch_replicator_test.hrl").
-define(DELAY, 500).
-define(TIMEOUT, 60000).
-setup_all() ->
- test_util:start_couch([couch_replicator, chttpd, mem3, fabric]).
-
-teardown_all(Ctx) ->
- ok = test_util:stop_couch(Ctx).
-
-setup() ->
- Source = setup_db(),
- Target = setup_db(),
- {Source, Target}.
-
-teardown({Source, Target}) ->
- teardown_db(Source),
- teardown_db(Target),
- ok.
-
stats_retained_test_() ->
{
- setup,
- fun setup_all/0,
- fun teardown_all/1,
- {
- foreach,
- fun setup/0,
- fun teardown/1,
- [
- fun t_stats_retained_by_scheduler/1,
- fun t_stats_retained_on_job_removal/1
- ]
- }
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(t_stats_retained_by_scheduler),
+ ?TDEF_FE(t_stats_retained_on_job_removal)
+ ]
}.
-t_stats_retained_by_scheduler({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, RepPid, RepId} = replicate(Source, Target),
- wait_target_in_sync(6, Target),
-
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+t_stats_retained_by_scheduler({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, RepPid, RepId} = replicate(Source, Target),
+ wait_target_in_sync(6, Target),
- stop_job(RepPid),
- check_scheduler_jobs(10, 5, 5),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- start_job(),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
- couch_replicator_scheduler:remove_job(RepId)
- end).
-
-t_stats_retained_on_job_removal({Source, Target}) ->
- ?_test(begin
- {ok, _} = add_vdu(Target),
- populate_db_reject_even_docs(Source, 1, 10),
- {ok, _, RepId} = replicate(Source, Target),
- % 5 + 1 vdu
- wait_target_in_sync(6, Target),
+ stop_job(RepPid),
+ check_scheduler_jobs(10, 5, 5),
- check_active_tasks(10, 5, 5),
- check_scheduler_jobs(10, 5, 5),
+ start_job(),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
+ couch_replicator_scheduler:remove_job(RepId).
- couch_replicator_scheduler:remove_job(RepId),
+t_stats_retained_on_job_removal({_Ctx, {Source, Target}}) ->
+ {ok, _} = add_vdu(Target),
+ populate_db_reject_even_docs(Source, 1, 10),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 5 + 1 vdu
+ wait_target_in_sync(6, Target),
- populate_db_reject_even_docs(Source, 11, 20),
- {ok, _, RepId} = replicate(Source, Target),
- % 6 + 5
- wait_target_in_sync(11, Target),
+ check_active_tasks(10, 5, 5),
+ check_scheduler_jobs(10, 5, 5),
- check_scheduler_jobs(20, 10, 10),
- check_active_tasks(20, 10, 10),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId),
+ populate_db_reject_even_docs(Source, 11, 20),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 6 + 5
+ wait_target_in_sync(11, Target),
- populate_db_reject_even_docs(Source, 21, 30),
- {ok, _, RepId} = replicate(Source, Target),
- % 11 + 5
- wait_target_in_sync(16, Target),
+ check_scheduler_jobs(20, 10, 10),
+ check_active_tasks(20, 10, 10),
- check_scheduler_jobs(30, 15, 15),
- check_active_tasks(30, 15, 15),
+ couch_replicator_scheduler:remove_job(RepId),
- couch_replicator_scheduler:remove_job(RepId)
- end).
+ populate_db_reject_even_docs(Source, 21, 30),
+ {ok, _, RepId} = replicate(Source, Target),
+ % 11 + 5
+ wait_target_in_sync(16, Target),
-setup_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ check_scheduler_jobs(30, 15, 15),
+ check_active_tasks(30, 15, 15),
-teardown_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
+ couch_replicator_scheduler:remove_job(RepId).
stop_job(RepPid) ->
Ref = erlang:monitor(process, RepPid),
@@ -201,7 +167,6 @@ populate_db_reject_even_docs(DbName, Start, End) ->
populate_db(DbName, Start, End, BodyFun).
populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
- {ok, Db} = couch_db:open_int(DbName, []),
Docs = lists:foldl(
fun(DocIdCounter, Acc) ->
Id = integer_to_binary(DocIdCounter),
@@ -211,8 +176,7 @@ populate_db(DbName, Start, End, BodyFun) when is_function(BodyFun, 1) ->
[],
lists:seq(Start, End)
),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
+ {ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).
wait_target_in_sync(DocCount, Target) when is_integer(DocCount) ->
wait_target_in_sync_loop(DocCount, Target, 300).
@@ -226,10 +190,7 @@ wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
]}
);
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ {ok, TargetDocCount} = fabric:get_doc_count(TargetName),
case TargetDocCount == DocCount of
true ->
true;
@@ -238,13 +199,14 @@ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
end.
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
replicate(Source, Target) ->
- SrcUrl = couch_replicator_test_helper:db_url(Source),
- TgtUrl = couch_replicator_test_helper:db_url(Target),
RepObject =
{[
- {<<"source">>, SrcUrl},
- {<<"target">>, TgtUrl},
+ {<<"source">>, db_url(Source)},
+ {<<"target">>, db_url(Target)},
{<<"continuous">>, true}
]},
{ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
@@ -254,10 +216,8 @@ replicate(Source, Target) ->
{ok, Pid, Rep#rep.id}.
scheduler_jobs() ->
- Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(chttpd, port),
- Url = lists:flatten(io_lib:format("http://~s:~b/_scheduler/jobs", [Addr, Port])),
- {ok, 200, _, Body} = test_request:get(Url, []),
+ JobsUrl = couch_replicator_test_helper:cluster_db_url(<<"_scheduler/jobs">>),
+ {ok, 200, _, Body} = test_request:get(?b2l(JobsUrl), []),
Json = jiffy:decode(Body, [return_maps]),
maps:get(<<"jobs">>, Json).
@@ -279,9 +239,4 @@ add_vdu(DbName) ->
{<<"validate_doc_update">>, vdu()}
],
Doc = couch_doc:from_json_obj({DocProps}, []),
- {ok, Db} = couch_db:open_int(DbName, [?ADMIN_CTX]),
- try
- {ok, _Rev} = couch_db:update_doc(Db, Doc, [])
- after
- couch_db:close(Db)
- end.
+ {ok, _Rev} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
[couchdb] 01/18: Add some utility functions to couch_replicator_test_helper
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 264ad11f37340e6b1c7e5b1acc21bad5a81d0bec
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:03:22 2022 -0400
Add some utility functions to couch_replicator_test_helper
In preparation to start using chttpd (fabric) endpoints add some common utility
functions to the replication test helper module.
Since `couch_db:fold_docs/4` doesn't exit for fabric, use the changes feed to
get all the revision leafs. That is used when comparing database endpoints.
It turns our the majority replication tests can use the exact same setup,
teardown and db_url functions so make sure those are also available in the
helper module.
---
.../test/eunit/couch_replicator_test_helper.erl | 134 +++++++++++++++------
1 file changed, 100 insertions(+), 34 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
index 4044e7c72..16423e85c 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_test_helper.erl
@@ -4,39 +4,75 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
+-define(USERNAME, "rep_test_user").
+-define(PASSWORD, "rep_test_pass").
+
-export([
- compare_dbs/2,
- compare_dbs/3,
- db_url/1,
+ cluster_compare_dbs/2,
+ cluster_compare_dbs/3,
+ cluster_doc_revs/1,
+ cluster_open_rev/3,
+ cluster_url/0,
+ cluster_db_url/1,
replicate/1,
get_pid/1,
- replicate/2
+ replicate/2,
+ test_setup/0,
+ test_teardown/1,
+ setup_db/0,
+ teardown_db/1
]).
-compare_dbs(Source, Target) ->
- compare_dbs(Source, Target, []).
-
-compare_dbs(Source, Target, ExceptIds) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
-
- Fun = fun(FullDocInfo, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
- Id = DocSource#doc.id,
- case lists:member(Id, ExceptIds) of
- true ->
- ?assertEqual(not_found, couch_db:get_doc_info(TargetDb, Id));
- false ->
- {ok, TDoc} = couch_db:open_doc(TargetDb, Id),
- compare_docs(DocSource, TDoc)
+cluster_compare_dbs(Source, Target) ->
+ cluster_compare_dbs(Source, Target, []).
+
+cluster_compare_dbs(Source, Target, ExceptIds) ->
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Source)),
+ ?assertMatch({ok, [_ | _]}, fabric:get_db_info(Target)),
+ lists:foreach(
+ fun({Id, Rev}) ->
+ SrcDoc = cluster_open_rev(Source, Id, Rev),
+ TgtDoc = cluster_open_rev(Target, Id, Rev),
+ case lists:member(Id, ExceptIds) of
+ true ->
+ ?assertEqual(not_found, TgtDoc);
+ false ->
+ compare_docs(SrcDoc, TgtDoc)
+ end
end,
- {ok, Acc}
- end,
+ cluster_doc_revs(Source)
+ ).
+
+cluster_open_rev(DbName, Id, Rev) ->
+ {ok, [Result]} = fabric:open_revs(DbName, Id, [Rev], []),
+ case Result of
+ {ok, #doc{} = Doc} ->
+ Doc;
+ {{not_found, missing}, _} ->
+ not_found
+ end.
+
+cluster_doc_revs(DbName) ->
+ Opts = [{style, all_docs}],
+ {ok, Acc} = fabric_util:isolate(fun() ->
+ fabric:changes(DbName, fun changes_callback/2, [], Opts)
+ end),
+ Acc.
- {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
+changes_callback(start, Acc) ->
+ {ok, Acc};
+changes_callback({change, {Change}}, Acc) ->
+ Id = proplists:get_value(id, Change),
+ Revs = proplists:get_value(changes, Change),
+ IdRevs = [{Id, couch_doc:parse_rev(R)} || {[{<<"rev">>, R}]} <- Revs],
+ {ok, IdRevs ++ Acc};
+changes_callback(timeout, Acc) ->
+ {ok, Acc};
+changes_callback({stop, _EndSeq, _Pending}, Acc) ->
+ {ok, Acc}.
+compare_docs(#doc{} = Doc1, not_found) ->
+ error({not_found, Doc1#doc.id});
compare_docs(Doc1, Doc2) ->
?assertEqual(Doc1#doc.body, Doc2#doc.body),
#doc{atts = Atts1} = Doc1,
@@ -111,15 +147,17 @@ att_decoded_md5(Att) ->
),
couch_hash:md5_hash_final(Md50).
-db_url(DbName) ->
- iolist_to_binary([
- "http://",
- config:get("httpd", "bind_address", "127.0.0.1"),
- ":",
- integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/",
- DbName
- ]).
+cluster_url() ->
+ Fmt = "http://~s:~s@~s:~b",
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Args = [?USERNAME, ?PASSWORD, Addr, Port],
+ ?l2b(io_lib:format(Fmt, Args)).
+
+cluster_db_url(<<"/", _/binary>> = Path) ->
+ <<(cluster_url())/binary, Path/binary>>;
+cluster_db_url(Path) ->
+ <<(cluster_url())/binary, "/", Path/binary>>.
get_pid(RepId) ->
Pid = global:whereis_name({couch_replicator_scheduler_job, RepId}),
@@ -145,3 +183,31 @@ replicate({[_ | _]} = RepObject) ->
ok
end,
ok = couch_replicator_scheduler:remove_job(Rep#rep.id).
+
+setup_db() ->
+ DbName = ?tempdb(),
+ ok = fabric:create_db(DbName, [{q, 1}, {n, 1}, ?ADMIN_CTX]),
+ DbName.
+
+teardown_db(DbName) ->
+ try
+ ok = fabric:delete_db(DbName, [?ADMIN_CTX])
+ catch
+ error:database_does_not_exist ->
+ ok
+ end.
+
+test_setup() ->
+ Ctx = test_util:start_couch([fabric, mem3, chttpd, couch_replicator]),
+ Hashed = couch_passwords:hash_admin_password(?PASSWORD),
+ ok = config:set("admins", ?USERNAME, ?b2l(Hashed), _Persist = false),
+ Source = setup_db(),
+ Target = setup_db(),
+ {Ctx, {Source, Target}}.
+
+test_teardown({Ctx, {Source, Target}}) ->
+ meck:unload(),
+ teardown_db(Source),
+ teardown_db(Target),
+ config:delete("admins", ?USERNAME, _Persist = false),
+ ok = test_util:stop_couch(Ctx).
[couchdb] 02/18: Update couch_replicator_attachments_too_large to use fabric
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 1ec76df18450a00c775aa44d02abd75f24209bd5
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:08:17 2022 -0400
Update couch_replicator_attachments_too_large to use fabric
Switch test module to use the clustered enpdpoints and TDEF_FE test
macros.
---
.../couch_replicator_attachments_too_large.erl | 79 +++++++---------------
1 file changed, 24 insertions(+), 55 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
index 2d58f847e..4c4ff9c14 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_attachments_too_large.erl
@@ -14,69 +14,37 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc_with_attachment(Source, <<"doc">>, 1000),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:delete("couchdb", "max_attachment_size"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
attachment_too_large_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Attachment size too large replication tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "1000", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
+ ok = replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc_with_attachment(Source, <<"doc">>, 1000),
config:set("couchdb", "max_attachment_size", "999", _Persist = false),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
-
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+ ok = replicate(Source, Target),
+ ?assertError({not_found, <<"doc">>}, compare(Source, Target)).
create_doc_with_attachment(DbName, DocId, AttSize) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = #doc{id = DocId, atts = att(AttSize)},
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db),
- ok.
+ {ok, {1, _Rev}} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
att(Size) when is_integer(Size), Size >= 1 ->
[
@@ -90,10 +58,11 @@ att(Size) when is_integer(Size), Size >= 1 ->
])
].
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
+
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
[couchdb] 09/18: Update couch_replicator_id_too_long_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 065b212e5929d088abb9f38787c9bfeefc0c0b19
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:23:47 2022 -0400
Update couch_replicator_id_too_long_tests
Use common setup and teardown helpers along with some local replicate/2 and db_url/2 functions.
Remove foreachx goop and use TDEF_FE for consistency with other tests.
---
.../eunit/couch_replicator_id_too_long_tests.erl | 82 +++++++---------------
1 file changed, 26 insertions(+), 56 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
index 9ed415a29..08454bd71 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_id_too_long_tests.erl
@@ -14,73 +14,43 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
-setup(_) ->
- Ctx = test_util:start_couch([couch_replicator]),
- Source = create_db(),
- create_doc(Source),
- Target = create_db(),
- {Ctx, {Source, Target}}.
-
-teardown(_, {Ctx, {Source, Target}}) ->
- delete_db(Source),
- delete_db(Target),
- config:set("replicator", "max_document_id_length", "infinity"),
- ok = test_util:stop_couch(Ctx).
+-include("couch_replicator_test.hrl").
id_too_long_replication_test_() ->
- Pairs = [{remote, remote}],
{
"Doc id too long tests",
{
- foreachx,
- fun setup/1,
- fun teardown/2,
- [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
- [{Pair, fun should_fail/2} || Pair <- Pairs]
+ foreach,
+ fun couch_replicator_test_helper:test_setup/0,
+ fun couch_replicator_test_helper:test_teardown/1,
+ [
+ ?TDEF_FE(should_succeed),
+ ?TDEF_FE(should_fail)
+ ]
}
}.
-should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "5"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
-
-should_fail({From, To}, {_Ctx, {Source, Target}}) ->
- RepObject =
- {[
- {<<"source">>, db_url(From, Source)},
- {<<"target">>, db_url(To, Target)}
- ]},
- config:set("replicator", "max_document_id_length", "4"),
- {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
- ?_assertError(
- {badmatch, {not_found, missing}},
- couch_replicator_test_helper:compare_dbs(Source, Target)
- ).
+should_succeed({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "5", _Persist = false),
+ replicate(Source, Target),
+ ?assertEqual(ok, compare(Source, Target)).
-create_db() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- DbName.
+should_fail({_Ctx, {Source, Target}}) ->
+ create_doc(Source),
+ config:set("replicator", "max_document_id_length", "4", _Persist = false),
+ replicate(Source, Target),
+ ?assertError({not_found, <<"12345">>}, compare(Source, Target)).
create_doc(DbName) ->
- {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
Doc = couch_doc:from_json_obj({[{<<"_id">>, <<"12345">>}]}),
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
+ {ok, _} = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]).
+
+db_url(DbName) ->
+ couch_replicator_test_helper:cluster_db_url(DbName).
-delete_db(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+compare(Source, Target) ->
+ couch_replicator_test_helper:cluster_compare_dbs(Source, Target).
-db_url(remote, DbName) ->
- Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = mochiweb_socket_server:get(couch_httpd, port),
- ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
+replicate(Source, Target) ->
+ couch_replicator_test_helper:replicate(db_url(Source), db_url(Target)).
[couchdb] 08/18: Update couch_replicator_httpc_pool_tests
Posted by va...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vatamane pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 75e73da04df4b110c3c395b43be30a724def875e
Author: Nick Vatamaniuc <va...@gmail.com>
AuthorDate: Thu Aug 18 21:22:30 2022 -0400
Update couch_replicator_httpc_pool_tests
Use the TDEF_FE macro and remove the ?_test(begin...end) construct.
---
.../eunit/couch_replicator_httpc_pool_tests.erl | 96 +++++++++++-----------
1 file changed, 46 insertions(+), 50 deletions(-)
diff --git a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
index 31f1da48e..5fce5e886 100644
--- a/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
+++ b/src/couch_replicator/test/eunit/couch_replicator_httpc_pool_tests.erl
@@ -13,7 +13,7 @@
-module(couch_replicator_httpc_pool_tests).
-include_lib("couch/include/couch_eunit.hrl").
--include_lib("couch/include/couch_db.hrl").
+-include("couch_replicator_test.hrl").
-define(TIMEOUT, 1000).
@@ -35,71 +35,67 @@ httpc_pool_test_() ->
fun setup/0,
fun teardown/1,
[
- fun should_block_new_clients_when_full/1,
- fun should_replace_worker_on_death/1
+ ?TDEF_FE(should_block_new_clients_when_full),
+ ?TDEF_FE(should_replace_worker_on_death)
]
}
}
}.
should_block_new_clients_when_full(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
+ Client1 = spawn_client(Pool),
+ Client2 = spawn_client(Pool),
+ Client3 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- ?assertEqual(ok, ping_client(Client2)),
- ?assertEqual(ok, ping_client(Client3)),
+ ?assertEqual(ok, ping_client(Client1)),
+ ?assertEqual(ok, ping_client(Client2)),
+ ?assertEqual(ok, ping_client(Client3)),
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
+ Worker1 = get_client_worker(Client1, "1"),
+ Worker2 = get_client_worker(Client2, "2"),
+ Worker3 = get_client_worker(Client3, "3"),
- ?assert(is_process_alive(Worker1)),
- ?assert(is_process_alive(Worker2)),
- ?assert(is_process_alive(Worker3)),
+ ?assert(is_process_alive(Worker1)),
+ ?assert(is_process_alive(Worker2)),
+ ?assert(is_process_alive(Worker3)),
- ?assertNotEqual(Worker1, Worker2),
- ?assertNotEqual(Worker2, Worker3),
- ?assertNotEqual(Worker3, Worker1),
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertNotEqual(Worker2, Worker3),
+ ?assertNotEqual(Worker3, Worker1),
- Client4 = spawn_client(Pool),
- ?assertEqual(timeout, ping_client(Client4)),
+ Client4 = spawn_client(Pool),
+ ?assertEqual(timeout, ping_client(Client4)),
- ?assertEqual(ok, stop_client(Client1)),
- ?assertEqual(ok, ping_client(Client4)),
+ ?assertEqual(ok, stop_client(Client1)),
+ ?assertEqual(ok, ping_client(Client4)),
- Worker4 = get_client_worker(Client4, "4"),
- ?assertEqual(Worker1, Worker4),
+ Worker4 = get_client_worker(Client4, "4"),
+ ?assertEqual(Worker1, Worker4),
- lists:foreach(
- fun(C) ->
- ?assertEqual(ok, stop_client(C))
- end,
- [Client2, Client3, Client4]
- )
- end).
+ lists:foreach(
+ fun(C) ->
+ ?assertEqual(ok, stop_client(C))
+ end,
+ [Client2, Client3, Client4]
+ ).
should_replace_worker_on_death(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- Worker1 = get_client_worker(Client1, "1"),
- ?assert(is_process_alive(Worker1)),
-
- ?assertEqual(ok, kill_client_worker(Client1)),
- ?assertNot(is_process_alive(Worker1)),
- ?assertEqual(ok, stop_client(Client1)),
-
- Client2 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client2)),
- Worker2 = get_client_worker(Client2, "2"),
- ?assert(is_process_alive(Worker2)),
-
- ?assertNotEqual(Worker1, Worker2),
- ?assertEqual(ok, stop_client(Client2))
- end).
+ Client1 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client1)),
+ Worker1 = get_client_worker(Client1, "1"),
+ ?assert(is_process_alive(Worker1)),
+
+ ?assertEqual(ok, kill_client_worker(Client1)),
+ ?assertNot(is_process_alive(Worker1)),
+ ?assertEqual(ok, stop_client(Client1)),
+
+ Client2 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client2)),
+ Worker2 = get_client_worker(Client2, "2"),
+ ?assert(is_process_alive(Worker2)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertEqual(ok, stop_client(Client2)).
spawn_client(Pool) ->
Parent = self(),