You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2014/09/05 23:10:48 UTC
[01/14] Move files out of test/couchdb into top level test/ folder
Repository: couchdb-couch-replicator
Updated Branches:
refs/heads/master aafb5f9d9 -> 3cf0b131b
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
new file mode 100644
index 0000000..ed7ec50
--- /dev/null
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -0,0 +1,218 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_large_atts_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(ATT_SIZE_1, 2 * 1024 * 1024).
+-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
+-define(DOCS_COUNT, 11).
+-define(TIMEOUT_EUNIT, 30).
+-define(TIMEOUT_STOP, 1000).
+
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ couch_config:set("attachments", "compressible_types", "text/*", false),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+large_atts_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate docs with large attachments",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target)]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ Docs = lists:foldl(
+ fun(DocIdCounter, Acc) ->
+ Doc = #doc{
+ id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ body = {[]},
+ atts = [
+ att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
+ att(<<"att2">>, ?ATT_SIZE_2, <<"app/binary">>)
+ ]
+ },
+ [Doc | Acc]
+ end,
+ [], lists:seq(1, DocCount)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []),
+ couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
+ Id = DocSource#doc.id,
+
+ {ok, DocTarget} = couch_db:open_doc(TargetDb, Id),
+ ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+ #doc{atts = SourceAtts} = DocSource,
+ #doc{atts = TargetAtts} = DocTarget,
+ ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+ lists:sort([N || #att{name = N} <- TargetAtts])),
+
+ FunCompareAtts = fun(#att{name = AttName} = Att) ->
+ {ok, AttTarget} = find_att(TargetAtts, AttName),
+ SourceMd5 = att_md5(Att),
+ TargetMd5 = att_md5(AttTarget),
+ case AttName of
+ <<"att1">> ->
+ ?assertEqual(gzip, Att#att.encoding),
+ ?assertEqual(gzip, AttTarget#att.encoding),
+ DecSourceMd5 = att_decoded_md5(Att),
+ DecTargetMd5 = att_decoded_md5(AttTarget),
+ ?assertEqual(DecSourceMd5, DecTargetMd5);
+ _ ->
+ ?assertEqual(identity, Att#att.encoding),
+ ?assertEqual(identity, AttTarget#att.encoding)
+ end,
+ ?assertEqual(SourceMd5, TargetMd5),
+ ?assert(is_integer(Att#att.disk_len)),
+ ?assert(is_integer(Att#att.att_len)),
+ ?assert(is_integer(AttTarget#att.disk_len)),
+ ?assert(is_integer(AttTarget#att.att_len)),
+ ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+ ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+ ?assertEqual(Att#att.type, AttTarget#att.type),
+ ?assertEqual(Att#att.md5, AttTarget#att.md5)
+ end,
+
+ lists:foreach(FunCompareAtts, SourceAtts),
+
+ {ok, Acc}
+ end,
+
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+att(Name, Size, Type) ->
+ #att{
+ name = Name,
+ type = Type,
+ att_len = Size,
+ data = fun(Count) -> crypto:rand_bytes(Count) end
+ }.
+
+find_att([], _Name) ->
+ nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+ {ok, Att};
+find_att([_ | Rest], Name) ->
+ find_att(Rest, Name).
+
+att_md5(Att) ->
+ Md50 = couch_doc:att_foldl(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+ Md50 = couch_doc:att_foldl_decode(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[{<<"source">>, Source}, {<<"target">>, Target}]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
new file mode 100644
index 0000000..7d0de9e
--- /dev/null
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -0,0 +1,232 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_many_leaves_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_CONFLICTS, [
+ {<<"doc1">>, 10},
+ {<<"doc2">>, 100},
+ % a number > MaxURLlength (7000) / length(DocRevisionString)
+ {<<"doc3">>, 210}
+]).
+-define(NUM_ATTS, 2).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 60).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+docs_with_many_leaves_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate documents with many leaves",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_verify_target({remote, Source}, Target) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, {remote, Target}) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb)
+ end)}.
+
+should_add_attachments_to_source({remote, Source}) ->
+ should_add_attachments_to_source(Source);
+should_add_attachments_to_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb)
+ end)}.
+
+populate_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ lists:foreach(
+ fun({DocId, NumConflicts}) ->
+ Value = <<"0">>,
+ Doc = #doc{
+ id = DocId,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ end, ?DOCS_CONFLICTS),
+ couch_db:close(Db).
+
+add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
+ add_doc_siblings(Db, DocId, NumLeaves, [], []).
+
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
+ {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
+ {ok, AccRevs};
+
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
+ Value = ?l2b(?i2l(NumLeaves)),
+ Rev = couch_util:md5(Value),
+ Doc = #doc{
+ id = DocId,
+ revs = {1, [Rev]},
+ body = {[ {<<"value">>, Value} ]}
+ },
+ add_doc_siblings(Db, DocId, NumLeaves - 1,
+ [Doc | AccDocs], [{1, Rev} | AccRevs]).
+
+verify_target(_SourceDb, _TargetDb, []) ->
+ ok;
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(
+ SourceDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ {ok, TargetLookups} = couch_db:open_doc_revs(
+ TargetDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(TargetDocs)),
+ lists:foreach(
+ fun({SourceDoc, TargetDoc}) ->
+ SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]),
+ TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]),
+ ?assertEqual(SourceJson, TargetJson)
+ end,
+ lists:zip(SourceDocs, TargetDocs)),
+ verify_target(SourceDb, TargetDb, Rest).
+
+add_attachments(_SourceDb, _NumAtts, []) ->
+ ok;
+add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(SourceDocs)),
+ NewDocs = lists:foldl(
+ fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+ NewAtts = lists:foldl(fun(I, AttAcc) ->
+ AttData = crypto:rand_bytes(100),
+ NewAtt = #att{
+ name = ?io2b(["att_", ?i2l(I), "_",
+ couch_doc:rev_to_str({Pos, Rev})]),
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ },
+ [NewAtt | AttAcc]
+ end, [], lists:seq(1, NumAtts)),
+ [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+ end,
+ [], SourceDocs),
+ {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ NewRevs = [R || {ok, R} <- UpdateResults],
+ ?assertEqual(length(NewDocs), length(NewRevs)),
+ add_attachments(SourceDb, NumAtts, Rest).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
new file mode 100644
index 0000000..0c2c30d
--- /dev/null
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -0,0 +1,260 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_missing_stubs_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(REVS_LIMIT, 3).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+missing_stubs_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate docs with missing stubs (COUCHDB-1365)",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_replicate_docs_with_missed_att_stubs/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_replicate_docs_with_missed_att_stubs({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source),
+ should_set_target_revs_limit(Target, ?REVS_LIMIT),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target),
+ should_update_source_docs(Source, ?REVS_LIMIT * 2),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_set_target_revs_limit({remote, Target}, RevsLimit) ->
+ should_set_target_revs_limit(Target, RevsLimit);
+should_set_target_revs_limit(Target, RevsLimit) ->
+ ?_test(begin
+ {ok, Db} = couch_db:open_int(Target, [?ADMIN_USER]),
+ ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
+ ok = couch_db:close(Db)
+ end).
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+should_update_source_docs({remote, Source}, Times) ->
+ should_update_source_docs(Source, Times);
+should_update_source_docs(Source, Times) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+
+
+populate_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ AttData = crypto:rand_bytes(6000),
+ Doc = #doc{
+ id = <<"doc1">>,
+ atts = [
+ #att{
+ name = <<"doc1_att1">>,
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ }
+ ]
+ },
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ couch_db:close(Db).
+
+update_db_docs(DbName, Times) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, _, _} = couch_db:enum_docs(
+ Db,
+ fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
+ {DbName, Times},
+ []),
+ ok = couch_db:close(Db).
+
+db_fold_fun(FullDocInfo, {DbName, Times}) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+ lists:foldl(
+ fun(_, {Pos, RevId}) ->
+ {ok, Db2} = couch_db:reopen(Db),
+ NewDocVersion = Doc#doc{
+ revs = {Pos, [RevId]},
+ body = {[{<<"value">>, base64:encode(crypto:rand_bytes(100))}]}
+ },
+ {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+ NewRev
+ end,
+ {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+ lists:seq(1, Times)),
+ ok = couch_db:close(Db),
+ {ok, {DbName, Times}}.
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo,
+ [conflicts, deleted_conflicts]),
+ Id = DocSource#doc.id,
+
+ {ok, DocTarget} = couch_db:open_doc(TargetDb, Id,
+ [conflicts, deleted_conflicts]),
+ ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+ ?assertEqual(couch_doc:to_json_obj(DocSource, []),
+ couch_doc:to_json_obj(DocTarget, [])),
+
+ #doc{atts = SourceAtts} = DocSource,
+ #doc{atts = TargetAtts} = DocTarget,
+ ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+ lists:sort([N || #att{name = N} <- TargetAtts])),
+
+ lists:foreach(
+ fun(#att{name = AttName} = Att) ->
+ {ok, AttTarget} = find_att(TargetAtts, AttName),
+ SourceMd5 = att_md5(Att),
+ TargetMd5 = att_md5(AttTarget),
+ case AttName of
+ <<"att1">> ->
+ ?assertEqual(gzip, Att#att.encoding),
+ ?assertEqual(gzip, AttTarget#att.encoding),
+ DecSourceMd5 = att_decoded_md5(Att),
+ DecTargetMd5 = att_decoded_md5(AttTarget),
+ ?assertEqual(DecSourceMd5, DecTargetMd5);
+ _ ->
+ ?assertEqual(identity, Att#att.encoding),
+ ?assertEqual(identity, AttTarget#att.encoding)
+ end,
+ ?assertEqual(SourceMd5, TargetMd5),
+ ?assert(is_integer(Att#att.disk_len)),
+ ?assert(is_integer(Att#att.att_len)),
+ ?assert(is_integer(AttTarget#att.disk_len)),
+ ?assert(is_integer(AttTarget#att.att_len)),
+ ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+ ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+ ?assertEqual(Att#att.type, AttTarget#att.type),
+ ?assertEqual(Att#att.md5, AttTarget#att.md5)
+ end,
+ SourceAtts),
+ {ok, Acc}
+ end,
+
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+find_att([], _Name) ->
+ nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+ {ok, Att};
+find_att([_ | Rest], Name) ->
+ find_att(Rest, Name).
+
+att_md5(Att) ->
+ Md50 = couch_doc:att_foldl(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+ Md50 = couch_doc:att_foldl_decode(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_modules_load_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_modules_load_tests.erl b/test/couch_replicator_modules_load_tests.erl
new file mode 100644
index 0000000..cea4cc2
--- /dev/null
+++ b/test/couch_replicator_modules_load_tests.erl
@@ -0,0 +1,40 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_modules_load_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+
+
+modules_load_test_() ->
+ {
+ "Verify that all modules loads",
+ should_load_modules()
+ }.
+
+
+should_load_modules() ->
+ Modules = [
+ couch_replicator_api_wrap,
+ couch_replicator_httpc,
+ couch_replicator_httpd,
+ couch_replicator_manager,
+ couch_replicator_notifier,
+ couch_replicator,
+ couch_replicator_worker,
+ couch_replicator_utils,
+ couch_replicator_job_sup
+ ],
+ [should_load_module(Mod) || Mod <- Modules].
+
+should_load_module(Mod) ->
+ {atom_to_list(Mod), ?_assertMatch({module, _}, code:load_file(Mod))}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
new file mode 100644
index 0000000..f09a235
--- /dev/null
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -0,0 +1,200 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_use_checkpoints_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_COUNT, 100).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+
+start(false) ->
+ fun
+ ({finished, _, {CheckpointHistory}}) ->
+ ?assertEqual([{<<"use_checkpoints">>,false}], CheckpointHistory);
+ (_) ->
+ ok
+ end;
+start(true) ->
+ fun
+ ({finished, _, {CheckpointHistory}}) ->
+ ?assertNotEqual(false, lists:keyfind(<<"session_id">>,
+ 1, CheckpointHistory));
+ (_) ->
+ ok
+ end.
+
+stop(_, _) ->
+ ok.
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({_, Fun, {A, B}}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target, Listener}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target, Listener}) ->
+ teardown(Source),
+ teardown(Target),
+
+ couch_replicator_notifier:stop(Listener),
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+use_checkpoints_test_() ->
+ {
+ "Replication use_checkpoints feature tests",
+ {
+ foreachx,
+ fun start/1, fun stop/2,
+ [{UseCheckpoints, fun use_checkpoints_tests/2}
+ || UseCheckpoints <- [false, true]]
+ }
+ }.
+
+use_checkpoints_tests(UseCheckpoints, Fun) ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+should_test_checkpoints({UseCheckpoints, _, {From, To}}, {Source, Target, _}) ->
+ should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
+should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source, ?DOCS_COUNT),
+ should_replicate(Source, Target, UseCheckpoints),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}, DocCount) ->
+ should_populate_source(Source, DocCount);
+should_populate_source(Source, DocCount) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
+
+should_replicate({remote, Source}, Target, UseCheckpoints) ->
+ should_replicate(db_url(Source), Target, UseCheckpoints);
+should_replicate(Source, {remote, Target}, UseCheckpoints) ->
+ should_replicate(Source, db_url(Target), UseCheckpoints);
+should_replicate(Source, Target, UseCheckpoints) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ Docs = lists:foldl(
+ fun(DocIdCounter, Acc) ->
+ Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
+ Value = ?io2b(["val", ?i2l(DocIdCounter)]),
+ Doc = #doc{
+ id = Id,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ [Doc | Acc]
+ end,
+ [], lists:seq(1, DocCount)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []),
+ ok = couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Error opening document '",
+ ?b2l(DocId), "' from target: ",
+ couch_util:to_list(Error)])}]})
+ end,
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target, UseCheckpoints) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target},
+ {<<"use_checkpoints">>, UseCheckpoints}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
[11/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Switch to using test_util:{start,stop}_couch
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/270bdc19
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/270bdc19
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/270bdc19
Branch: refs/heads/master
Commit: 270bdc19b5a378a55b7963dc6c78241698de6f16
Parents: 62a3db9
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 16:40:51 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:41 2014 -0700
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 13 ++-----------
test/couch_replicator_httpc_pool_tests.erl | 16 +---------------
test/couch_replicator_large_atts_tests.erl | 13 ++-----------
test/couch_replicator_many_leaves_tests.erl | 13 ++-----------
test/couch_replicator_missing_stubs_tests.erl | 13 ++-----------
test/couch_replicator_use_checkpoints_tests.erl | 13 ++-----------
6 files changed, 11 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 407695e..09140ed 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -34,7 +34,7 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ ok = test_util:start_couch(),
Source = setup(A),
Target = setup(B),
{Source, Target}.
@@ -49,16 +49,7 @@ teardown(_, {Source, Target}) ->
teardown(Source),
teardown(Target),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
+ ok = test_util:stop_couch().
compact_test_() ->
Pairs = [{local, local}, {local, remote},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_httpc_pool_tests.erl b/test/couch_replicator_httpc_pool_tests.erl
index b65d174..d067f94 100644
--- a/test/couch_replicator_httpc_pool_tests.erl
+++ b/test/couch_replicator_httpc_pool_tests.erl
@@ -18,20 +18,6 @@
-define(TIMEOUT, 1000).
-start() ->
- {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
- Pid.
-
-stop(Pid) ->
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT ->
- throw({timeout, server_stop})
- end.
-
setup() ->
spawn_pool().
@@ -44,7 +30,7 @@ httpc_pool_test_() ->
"httpc pool tests",
{
setup,
- fun start/0, fun stop/1,
+ fun test_util:start_couch/0, fun test_util:stop_couch/1,
{
foreach,
fun setup/0, fun teardown/1,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
index b821a18..71293d8 100644
--- a/test/couch_replicator_large_atts_tests.erl
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -33,8 +33,8 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
couch_config:set("attachments", "compressible_types", "text/*", false),
+ ok = test_util:start_couch(),
Source = setup(A),
Target = setup(B),
{Source, Target}.
@@ -49,16 +49,7 @@ teardown(_, {Source, Target}) ->
teardown(Source),
teardown(Target),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
+ ok = test_util:stop_couch().
large_atts_test_() ->
Pairs = [{local, local}, {local, remote},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
index 0a75da7..cf56440 100644
--- a/test/couch_replicator_many_leaves_tests.erl
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -38,7 +38,7 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ ok = test_util:start_couch(),
Source = setup(A),
Target = setup(B),
{Source, Target}.
@@ -53,16 +53,7 @@ teardown(_, {Source, Target}) ->
teardown(Source),
teardown(Target),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
+ ok = test_util:stop_couch().
docs_with_many_leaves_test_() ->
Pairs = [{local, local}, {local, remote},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index 713ab01..df8685e 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -31,7 +31,7 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ ok = test_util:start_couch(),
Source = setup(A),
Target = setup(B),
{Source, Target}.
@@ -46,16 +46,7 @@ teardown(_, {Source, Target}) ->
teardown(Source),
teardown(Target),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
+ ok = test_util:stop_couch().
missing_stubs_test_() ->
Pairs = [{local, local}, {local, remote},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/270bdc19/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
index 69ac661..edb8ec1 100644
--- a/test/couch_replicator_use_checkpoints_tests.erl
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -52,7 +52,7 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({_, Fun, {A, B}}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ ok = test_util:start_couch(),
{ok, Listener} = couch_replicator_notifier:start_link(Fun),
Source = setup(A),
Target = setup(B),
@@ -69,16 +69,7 @@ teardown(_, {Source, Target, Listener}) ->
teardown(Target),
couch_replicator_notifier:stop(Listener),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
+ ok = test_util:stop_couch().
use_checkpoints_test_() ->
{
[09/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/05-replication-many-leaves.t etap test to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9a2c8388
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9a2c8388
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9a2c8388
Branch: refs/heads/master
Commit: 9a2c83881c3644a6a6363f84534b2c7f260aa486
Parents: b010033
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 16 19:28:50 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_many_leaves_tests.erl | 232 +++++++++++++++
test/05-replication-many-leaves.t | 283 -------------------
2 files changed, 232 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9a2c8388/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
new file mode 100644
index 0000000..7d0de9e
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
@@ -0,0 +1,232 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_many_leaves_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_CONFLICTS, [
+ {<<"doc1">>, 10},
+ {<<"doc2">>, 100},
+ % a number > MaxURLlength (7000) / length(DocRevisionString)
+ {<<"doc3">>, 210}
+]).
+-define(NUM_ATTS, 2).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 60).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+docs_with_many_leaves_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate documents with many leaves",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_verify_target({remote, Source}, Target) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, {remote, Target}) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb)
+ end)}.
+
+should_add_attachments_to_source({remote, Source}) ->
+ should_add_attachments_to_source(Source);
+should_add_attachments_to_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb)
+ end)}.
+
+populate_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ lists:foreach(
+ fun({DocId, NumConflicts}) ->
+ Value = <<"0">>,
+ Doc = #doc{
+ id = DocId,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ end, ?DOCS_CONFLICTS),
+ couch_db:close(Db).
+
+add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
+ add_doc_siblings(Db, DocId, NumLeaves, [], []).
+
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
+ {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
+ {ok, AccRevs};
+
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
+ Value = ?l2b(?i2l(NumLeaves)),
+ Rev = couch_util:md5(Value),
+ Doc = #doc{
+ id = DocId,
+ revs = {1, [Rev]},
+ body = {[ {<<"value">>, Value} ]}
+ },
+ add_doc_siblings(Db, DocId, NumLeaves - 1,
+ [Doc | AccDocs], [{1, Rev} | AccRevs]).
+
+verify_target(_SourceDb, _TargetDb, []) ->
+ ok;
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(
+ SourceDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ {ok, TargetLookups} = couch_db:open_doc_revs(
+ TargetDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(TargetDocs)),
+ lists:foreach(
+ fun({SourceDoc, TargetDoc}) ->
+ SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]),
+ TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]),
+ ?assertEqual(SourceJson, TargetJson)
+ end,
+ lists:zip(SourceDocs, TargetDocs)),
+ verify_target(SourceDb, TargetDb, Rest).
+
+add_attachments(_SourceDb, _NumAtts, []) ->
+ ok;
+add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(SourceDocs)),
+ NewDocs = lists:foldl(
+ fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+ NewAtts = lists:foldl(fun(I, AttAcc) ->
+ AttData = crypto:rand_bytes(100),
+ NewAtt = #att{
+ name = ?io2b(["att_", ?i2l(I), "_",
+ couch_doc:rev_to_str({Pos, Rev})]),
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ },
+ [NewAtt | AttAcc]
+ end, [], lists:seq(1, NumAtts)),
+ [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+ end,
+ [], SourceDocs),
+ {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ NewRevs = [R || {ok, R} <- UpdateResults],
+ ?assertEqual(length(NewDocs), length(NewRevs)),
+ add_attachments(SourceDb, NumAtts, Rest).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9a2c8388/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
deleted file mode 100755
index bd795ec..0000000
--- a/test/05-replication-many-leaves.t
+++ /dev/null
@@ -1,283 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Test replication of documents with many leaf revisions.
-% Motivated by COUCHDB-1340 and other similar issues where a document
-% GET with a too long ?open_revs revision list doesn't work due to
-% maximum web server limits for the HTTP request path.
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(doc, {
- id = <<"">>,
- revs = {0, []},
- body = {[]},
- atts = [],
- deleted = false,
- meta = []
-}).
-
--record(att, {
- name,
- type,
- att_len,
- disk_len,
- md5= <<>>,
- revpos=0,
- data,
- encoding=identity
-}).
-
--define(b2l(B), binary_to_list(B)).
--define(l2b(L), list_to_binary(L)).
--define(i2l(I), integer_to_list(I)).
-
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-doc_ids() ->
- [<<"doc1">>, <<"doc2">>, <<"doc3">>].
-
-doc_num_conflicts(<<"doc1">>) -> 10;
-doc_num_conflicts(<<"doc2">>) -> 100;
-% a number > MaxURLlength (7000) / length(DocRevisionString)
-doc_num_conflicts(<<"doc3">>) -> 210.
-
-
-main(_) ->
- test_util:run(56, fun() -> test() end).
-
-
-test() ->
- test_util:start_couch(),
- ibrowse:start(),
- crypto:start(),
- config:set("replicator", "connection_timeout", "90000", false),
-
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- lists:foreach(
- fun({Source, Target}) ->
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- {ok, DocRevs} = populate_db(SourceDb),
- ok = couch_db:close(SourceDb),
- etap:diag("Creating target database"),
- {ok, TargetDb} = create_db(target_db_name()),
-
- ok = couch_db:close(TargetDb),
- etap:diag("Triggering replication"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- {ok, SourceDb2} = couch_db:open_int(source_db_name(), []),
- {ok, TargetDb2} = couch_db:open_int(target_db_name(), []),
- verify_target(SourceDb2, TargetDb2, DocRevs),
- ok = couch_db:close(SourceDb2),
- ok = couch_db:close(TargetDb2),
-
- {ok, SourceDb3} = couch_db:open_int(source_db_name(), []),
- {ok, DocRevs2} = add_attachments(SourceDb3, DocRevs, 2),
- ok = couch_db:close(SourceDb3),
- etap:diag("Triggering replication again"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- {ok, SourceDb4} = couch_db:open_int(source_db_name(), []),
- {ok, TargetDb4} = couch_db:open_int(target_db_name(), []),
- verify_target(SourceDb4, TargetDb4, DocRevs2),
- ok = couch_db:close(SourceDb4),
- ok = couch_db:close(TargetDb4),
-
- etap:diag("Deleting source and target databases"),
- delete_db(TargetDb),
- delete_db(SourceDb),
- ok = timer:sleep(1000)
- end,
- Pairs),
-
- test_util:stop_couch().
-
-
-populate_db(Db) ->
- DocRevsDict = lists:foldl(
- fun(DocId, Acc) ->
- Value = <<"0">>,
- Doc = #doc{
- id = DocId,
- body = {[ {<<"value">>, Value} ]}
- },
- {ok, Rev} = couch_db:update_doc(Db, Doc, []),
- {ok, DocRevs} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
- dict:store(DocId, [Rev | DocRevs], Acc)
- end,
- dict:new(), doc_ids()),
- {ok, dict:to_list(DocRevsDict)}.
-
-
-add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
- add_doc_siblings(Db, DocId, NumLeaves, [], []).
-
-
-add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
- {ok, AccRevs};
-
-add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
- Value = list_to_binary(integer_to_list(NumLeaves)),
- Rev = couch_util:md5(Value),
- Doc = #doc{
- id = DocId,
- revs = {1, [Rev]},
- body = {[ {<<"value">>, Value} ]}
- },
- add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], [{1, Rev} | AccRevs]).
-
-
-verify_target(_SourceDb, _TargetDb, []) ->
- ok;
-
-verify_target(SourceDb, TargetDb, [{DocId, RevList} | Rest]) ->
- {ok, Lookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- RevList,
- [conflicts, deleted_conflicts]),
- Docs = [Doc || {ok, Doc} <- Lookups],
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- RevList,
- [conflicts, deleted_conflicts]),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- Total = doc_num_conflicts(DocId) + 1,
- etap:is(
- length(Docs),
- Total,
- "Target has " ++ ?i2l(Total) ++ " leaf revisions of document " ++ ?b2l(DocId)),
- etap:diag("Verifying all revisions of document " ++ ?b2l(DocId)),
- lists:foreach(
- fun({#doc{id = Id, revs = Revs} = TgtDoc, #doc{id = Id, revs = Revs} = SrcDoc}) ->
- SourceJson = couch_doc:to_json_obj(SrcDoc, [attachments]),
- TargetJson = couch_doc:to_json_obj(TgtDoc, [attachments]),
- case TargetJson of
- SourceJson ->
- ok;
- _ ->
- {Pos, [Rev | _]} = Revs,
- etap:bail("Wrong value for revision " ++
- ?b2l(couch_doc:rev_to_str({Pos, Rev})) ++
- " of document " ++ ?b2l(DocId))
- end
- end,
- lists:zip(Docs, SourceDocs)),
- verify_target(SourceDb, TargetDb, Rest).
-
-
-add_attachments(Source, DocIdRevs, NumAtts) ->
- add_attachments(Source, DocIdRevs, NumAtts, []).
-
-add_attachments(_SourceDb, [], _NumAtts, Acc) ->
- {ok, Acc};
-
-add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- RevList,
- []),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- Total = doc_num_conflicts(DocId) + 1,
- etap:is(
- length(SourceDocs),
- Total,
- "Source still has " ++ ?i2l(Total) ++
- " leaf revisions of document " ++ ?b2l(DocId)),
- etap:diag("Adding " ++ ?i2l(NumAtts) ++
- " attachments to each revision of the document " ++ ?b2l(DocId)),
- NewDocs = lists:foldl(
- fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
- NewAtts = lists:foldl(
- fun(I, AttAcc) ->
- AttData = crypto:rand_bytes(100),
- NewAtt = #att{
- name = iolist_to_binary(
- ["att_", ?i2l(I), "_", couch_doc:rev_to_str({Pos, Rev})]),
- type = <<"application/foobar">>,
- att_len = byte_size(AttData),
- data = AttData
- },
- [NewAtt | AttAcc]
- end,
- [], lists:seq(1, NumAtts)),
- [Doc#doc{atts = Atts ++ NewAtts} | Acc]
- end,
- [], SourceDocs),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
- NewRevs = [R || {ok, R} <- UpdateResults],
- etap:is(
- length(NewRevs),
- length(NewDocs),
- "Document revisions updated with " ++ ?i2l(NumAtts) ++ " attachments"),
- add_attachments(SourceDb, Rest, NumAtts, [{DocId, NewRevs} | IdRevsAcc]).
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]).
-
-
-delete_db(Db) ->
- ok = couch_server:delete(
- couch_db:name(Db), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).
-
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, Reason} ->
- etap:is(Reason, normal, "Replication finished successfully")
- after 900000 ->
- etap:bail("Timeout waiting for replication to finish")
- end.
[13/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Switch to using config instead of couch_config
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/6d34c4ed
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/6d34c4ed
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/6d34c4ed
Branch: refs/heads/master
Commit: 6d34c4ed32cfdec274a8aab27b7c94c818110c7d
Parents: 270bdc1
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 16:41:46 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:41 2014 -0700
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 2 +-
test/couch_replicator_httpc_pool_tests.erl | 4 ++--
test/couch_replicator_large_atts_tests.erl | 4 ++--
test/couch_replicator_many_leaves_tests.erl | 2 +-
test/couch_replicator_missing_stubs_tests.erl | 2 +-
test/couch_replicator_use_checkpoints_tests.erl | 2 +-
6 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 09140ed..f9529d6 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -292,7 +292,7 @@ check_ref_counter(Type, #db{name = Name, fd_ref_counter = OldRefCounter}) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_httpc_pool_tests.erl b/test/couch_replicator_httpc_pool_tests.erl
index d067f94..ea36f7f 100644
--- a/test/couch_replicator_httpc_pool_tests.erl
+++ b/test/couch_replicator_httpc_pool_tests.erl
@@ -164,8 +164,8 @@ loop(Parent, Ref, Worker, Pool) ->
end.
spawn_pool() ->
- Host = couch_config:get("httpd", "bind_address", "127.0.0.1"),
- Port = couch_config:get("httpd", "port", "5984"),
+ Host = config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = config:get("httpd", "port", "5984"),
{ok, Pool} = couch_replicator_httpc_pool:start_link(
"http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}]),
Pool.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
index 71293d8..e8c0d1a 100644
--- a/test/couch_replicator_large_atts_tests.erl
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -33,8 +33,8 @@ setup(local) ->
setup(remote) ->
{remote, setup()};
setup({A, B}) ->
- couch_config:set("attachments", "compressible_types", "text/*", false),
ok = test_util:start_couch(),
+ config:set("attachments", "compressible_types", "text/*", false),
Source = setup(A),
Target = setup(B),
{Source, Target}.
@@ -191,7 +191,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
index cf56440..04d21ac 100644
--- a/test/couch_replicator_many_leaves_tests.erl
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -202,7 +202,7 @@ add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index df8685e..db95451 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -230,7 +230,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d34c4ed/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
index edb8ec1..5b5668d 100644
--- a/test/couch_replicator_use_checkpoints_tests.erl
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -169,7 +169,7 @@ compare_dbs(Source, Target) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
[02/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Move files out of test/couchdb into top level test/ folder
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/2850b946
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/2850b946
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/2850b946
Branch: refs/heads/master
Commit: 2850b94675d15974cad8bf98ac6cf4df63577f5c
Parents: ba8c397
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 15:31:54 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_compact_tests.erl | 448 -------------------
.../test/couch_replicator_httpc_pool_tests.erl | 189 --------
.../test/couch_replicator_large_atts_tests.erl | 218 ---------
.../test/couch_replicator_many_leaves_tests.erl | 232 ----------
.../couch_replicator_missing_stubs_tests.erl | 260 -----------
.../couch_replicator_modules_load_tests.erl | 40 --
.../couch_replicator_use_checkpoints_tests.erl | 200 ---------
test/couch_replicator_compact_tests.erl | 448 +++++++++++++++++++
test/couch_replicator_httpc_pool_tests.erl | 189 ++++++++
test/couch_replicator_large_atts_tests.erl | 218 +++++++++
test/couch_replicator_many_leaves_tests.erl | 232 ++++++++++
test/couch_replicator_missing_stubs_tests.erl | 260 +++++++++++
test/couch_replicator_modules_load_tests.erl | 40 ++
test/couch_replicator_use_checkpoints_tests.erl | 200 +++++++++
14 files changed, 1587 insertions(+), 1587 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_compact_tests.erl b/src/couch_replicator/test/couch_replicator_compact_tests.erl
deleted file mode 100644
index 4e019fe..0000000
--- a/src/couch_replicator/test/couch_replicator_compact_tests.erl
+++ /dev/null
@@ -1,448 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_compact_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
--include_lib("couch_replicator/src/couch_replicator.hrl").
-
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
--define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
--define(DELAY, 100).
--define(TIMEOUT, 30000).
--define(TIMEOUT_STOP, 1000).
--define(TIMEOUT_WRITER, 3000).
--define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 5).
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
- Source = setup(A),
- Target = setup(B),
- {Source, Target}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_USER]),
- ok.
-
-teardown(_, {Source, Target}) ->
- teardown(Source),
- teardown(Target),
-
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
-
-compact_test_() ->
- Pairs = [{local, local}, {local, remote},
- {remote, local}, {remote, remote}],
- {
- "Compaction during replication tests",
- {
- foreachx,
- fun setup/1, fun teardown/2,
- [{Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs]
- }
- }.
-
-
-should_populate_replicate_compact({From, To}, {Source, Target}) ->
- {ok, RepPid, RepId} = replicate(Source, Target),
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_run_replication(RepPid, RepId, Source, Target),
- should_all_processes_be_alive(RepPid, Source, Target),
- should_populate_and_compact(RepPid, Source, Target, 50, 5),
- should_wait_target_in_sync(Source, Target),
- should_ensure_replication_still_running(RepPid, RepId, Source, Target),
- should_cancel_replication(RepId, RepPid),
- should_compare_databases(Source, Target)
- ]}}.
-
-should_all_processes_be_alive(RepPid, Source, Target) ->
- ?_test(begin
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb#db.main_pid)),
- ?assert(is_process_alive(TargetDb#db.main_pid))
- end).
-
-should_run_replication(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
-
-should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
- ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
-
-check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
- Source = case Src of
- {remote, NameSrc} ->
- <<(db_url(NameSrc))/binary, $/>>;
- _ ->
- Src
- end,
- Target = case Tgt of
- {remote, NameTgt} ->
- <<(db_url(NameTgt))/binary, $/>>;
- _ ->
- Tgt
- end,
- FullRepId = ?l2b(BaseId ++ Ext),
- Pid = ?l2b(pid_to_list(RepPid)),
- [RepTask] = couch_task_status:all(),
- ?assertEqual(Pid, couch_util:get_value(pid, RepTask)),
- ?assertEqual(FullRepId, couch_util:get_value(replication_id, RepTask)),
- ?assertEqual(true, couch_util:get_value(continuous, RepTask)),
- ?assertEqual(Source, couch_util:get_value(source, RepTask)),
- ?assertEqual(Target, couch_util:get_value(target, RepTask)),
- ?assert(is_integer(couch_util:get_value(docs_read, RepTask))),
- ?assert(is_integer(couch_util:get_value(docs_written, RepTask))),
- ?assert(is_integer(couch_util:get_value(doc_write_failures, RepTask))),
- ?assert(is_integer(couch_util:get_value(revisions_checked, RepTask))),
- ?assert(is_integer(couch_util:get_value(missing_revisions_found, RepTask))),
- ?assert(is_integer(couch_util:get_value(checkpointed_source_seq, RepTask))),
- ?assert(is_integer(couch_util:get_value(source_seq, RepTask))),
- Progress = couch_util:get_value(progress, RepTask),
- ?assert(is_integer(Progress)),
- ?assert(Progress =< 100).
-
-should_cancel_replication(RepId, RepPid) ->
- ?_assertNot(begin
- {ok, _} = couch_replicator:cancel_replication(RepId),
- is_process_alive(RepPid)
- end).
-
-should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(begin
- {ok, SourceDb0} = reopen_db(Source),
- Writer = spawn_writer(SourceDb0),
- lists:foreach(
- fun(N) ->
- {ok, SourceDb} = reopen_db(Source),
- {ok, TargetDb} = reopen_db(Target),
- pause_writer(Writer),
-
- compact_db("source", SourceDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb#db.main_pid)),
- check_ref_counter("source", SourceDb),
-
- compact_db("target", TargetDb),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(TargetDb#db.main_pid)),
- check_ref_counter("target", TargetDb),
-
- {ok, SourceDb2} = reopen_db(SourceDb),
- {ok, TargetDb2} = reopen_db(TargetDb),
-
- resume_writer(Writer),
- wait_writer(Writer, BatchSize * N),
-
- compact_db("source", SourceDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb2#db.main_pid)),
- pause_writer(Writer),
- check_ref_counter("source", SourceDb2),
- resume_writer(Writer),
-
- compact_db("target", TargetDb2),
- ?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(TargetDb2#db.main_pid)),
- pause_writer(Writer),
- check_ref_counter("target", TargetDb2),
- resume_writer(Writer)
- end, lists:seq(1, Rounds)),
- stop_writer(Writer)
- end)}.
-
-should_wait_target_in_sync({remote, Source}, Target) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, {remote, Target}) ->
- should_wait_target_in_sync(Source, Target);
-should_wait_target_in_sync(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_assert(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
- ok = couch_db:close(SourceDb),
- SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
- wait_target_in_sync_loop(SourceDocCount, Target, 300)
- end)}.
-
-wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, "Could not get source and target databases in sync"}]});
-wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
-wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
- case TargetDocCount == DocCount of
- true ->
- true;
- false ->
- ok = timer:sleep(?DELAY),
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
- end.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, 35, ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget = case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, lists:concat(["Error opening document '",
- ?b2l(DocId), "' from target: ",
- couch_util:to_list(Error)])}]})
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
- end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
-
-
-reopen_db({remote, Db}) ->
- reopen_db(Db);
-reopen_db(#db{name=DbName}) ->
- reopen_db(DbName);
-reopen_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- ok = couch_db:close(Db),
- {ok, Db}.
-
-compact_db(Type, #db{name = Name}) ->
- {ok, Db} = couch_db:open_int(Name, []),
- {ok, CompactPid} = couch_db:start_compact(Db),
- MonRef = erlang:monitor(process, CompactPid),
- receive
- {'DOWN', MonRef, process, CompactPid, normal} ->
- ok;
- {'DOWN', MonRef, process, CompactPid, Reason} ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason,
- lists:concat(["Error compacting ", Type, " database ",
- ?b2l(Name), ": ",
- couch_util:to_list(Reason)])}]})
- after ?TIMEOUT ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, lists:concat(["Compaction for ", Type, " database ",
- ?b2l(Name), " didn't finish"])}]})
- end,
- ok = couch_db:close(Db).
-
-check_ref_counter(Type, #db{name = Name, fd_ref_counter = OldRefCounter}) ->
- MonRef = erlang:monitor(process, OldRefCounter),
- receive
- {'DOWN', MonRef, process, OldRefCounter, _} ->
- ok
- after ?TIMEOUT ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, lists:concat(["Old ", Type,
- " database ref counter didn't"
- " terminate"])}]})
- end,
- {ok, #db{fd_ref_counter = NewRefCounter} = Db} = couch_db:open_int(Name, []),
- ok = couch_db:close(Db),
- ?assertNotEqual(OldRefCounter, NewRefCounter).
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target},
- {<<"continuous">>, true}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- {ok, Pid, Rep#rep.id}.
-
-
-wait_writer(Pid, NumDocs) ->
- case get_writer_num_docs_written(Pid) of
- N when N >= NumDocs ->
- ok;
- _ ->
- wait_writer(Pid, NumDocs)
- end.
-
-spawn_writer(Db) ->
- Parent = self(),
- Pid = spawn(fun() -> writer_loop(Db, Parent, 0) end),
- Pid.
-
-
-pause_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {pause, Ref},
- receive
- {paused, Ref} ->
- ok
- after ?TIMEOUT_WRITER ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Failed to pause source database writer"}]})
- end.
-
-resume_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {continue, Ref},
- receive
- {ok, Ref} ->
- ok
- after ?TIMEOUT_WRITER ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Failed to pause source database writer"}]})
- end.
-
-get_writer_num_docs_written(Pid) ->
- Ref = make_ref(),
- Pid ! {get_count, Ref},
- receive
- {count, Ref, Count} ->
- Count
- after ?TIMEOUT_WRITER ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Timeout getting number of documents written"
- " from source database writer"}]})
- end.
-
-stop_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {stop, Ref},
- receive
- {stopped, Ref, DocsWritten} ->
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _Reason} ->
- DocsWritten
- after ?TIMEOUT ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Timeout stopping source database writer"}]})
- end
- after ?TIMEOUT_WRITER ->
- erlang:error({assertion_failed,
- [{module, ?MODULE},
- {line, ?LINE},
- {reason, "Timeout stopping source database writer"}]})
- end.
-
-writer_loop(#db{name = DbName}, Parent, Counter) ->
- {ok, Data} = file:read_file(?ATTFILE),
- maybe_pause(Parent, Counter),
- Doc = couch_doc:from_json_obj({[
- {<<"_id">>, ?l2b(integer_to_list(Counter + 1))},
- {<<"value">>, Counter + 1},
- {<<"_attachments">>, {[
- {<<"icon1.png">>, {[
- {<<"data">>, base64:encode(Data)},
- {<<"content_type">>, <<"image/png">>}
- ]}},
- {<<"icon2.png">>, {[
- {<<"data">>, base64:encode(iolist_to_binary([Data, Data]))},
- {<<"content_type">>, <<"image/png">>}
- ]}}
- ]}}
- ]}),
- maybe_pause(Parent, Counter),
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- ok = couch_db:close(Db),
- receive
- {get_count, Ref} ->
- Parent ! {count, Ref, Counter + 1},
- writer_loop(Db, Parent, Counter + 1);
- {stop, Ref} ->
- Parent ! {stopped, Ref, Counter + 1}
- after 0 ->
- timer:sleep(?DELAY),
- writer_loop(Db, Parent, Counter + 1)
- end.
-
-maybe_pause(Parent, Counter) ->
- receive
- {get_count, Ref} ->
- Parent ! {count, Ref, Counter};
- {pause, Ref} ->
- Parent ! {paused, Ref},
- receive
- {continue, Ref2} ->
- Parent ! {ok, Ref2}
- end
- after 0 ->
- ok
- end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
deleted file mode 100644
index 6bede4c..0000000
--- a/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
+++ /dev/null
@@ -1,189 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_httpc_pool_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
-
--define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
--define(TIMEOUT, 1000).
-
-
-start() ->
- {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
- Pid.
-
-stop(Pid) ->
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT ->
- throw({timeout, server_stop})
- end.
-
-setup() ->
- spawn_pool().
-
-teardown(Pool) ->
- stop_pool(Pool).
-
-
-httpc_pool_test_() ->
- {
- "httpc pool tests",
- {
- setup,
- fun start/0, fun stop/1,
- {
- foreach,
- fun setup/0, fun teardown/1,
- [
- fun should_block_new_clients_when_full/1,
- fun should_replace_worker_on_death/1
- ]
- }
- }
- }.
-
-
-should_block_new_clients_when_full(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
-
- ?assertEqual(ok, ping_client(Client1)),
- ?assertEqual(ok, ping_client(Client2)),
- ?assertEqual(ok, ping_client(Client3)),
-
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
-
- ?assert(is_process_alive(Worker1)),
- ?assert(is_process_alive(Worker2)),
- ?assert(is_process_alive(Worker3)),
-
- ?assertNotEqual(Worker1, Worker2),
- ?assertNotEqual(Worker2, Worker3),
- ?assertNotEqual(Worker3, Worker1),
-
- Client4 = spawn_client(Pool),
- ?assertEqual(timeout, ping_client(Client4)),
-
- ?assertEqual(ok, stop_client(Client1)),
- ?assertEqual(ok, ping_client(Client4)),
-
- Worker4 = get_client_worker(Client4, "4"),
- ?assertEqual(Worker1, Worker4),
-
- lists:foreach(
- fun(C) ->
- ?assertEqual(ok, stop_client(C))
- end, [Client2, Client3, Client4])
- end).
-
-should_replace_worker_on_death(Pool) ->
- ?_test(begin
- Client1 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client1)),
- Worker1 = get_client_worker(Client1, "1"),
- ?assert(is_process_alive(Worker1)),
-
- ?assertEqual(ok, kill_client_worker(Client1)),
- ?assertNot(is_process_alive(Worker1)),
- ?assertEqual(ok, stop_client(Client1)),
-
- Client2 = spawn_client(Pool),
- ?assertEqual(ok, ping_client(Client2)),
- Worker2 = get_client_worker(Client2, "2"),
- ?assert(is_process_alive(Worker2)),
-
- ?assertNotEqual(Worker1, Worker2),
- ?assertEqual(ok, stop_client(Client2))
- end).
-
-
-spawn_client(Pool) ->
- Parent = self(),
- Ref = make_ref(),
- Pid = spawn(fun() ->
- {ok, Worker} = couch_replicator_httpc_pool:get_worker(Pool),
- loop(Parent, Ref, Worker, Pool)
- end),
- {Pid, Ref}.
-
-ping_client({Pid, Ref}) ->
- Pid ! ping,
- receive
- {pong, Ref} ->
- ok
- after ?TIMEOUT ->
- timeout
- end.
-
-get_client_worker({Pid, Ref}, ClientName) ->
- Pid ! get_worker,
- receive
- {worker, Ref, Worker} ->
- Worker
- after ?TIMEOUT ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, "Timeout getting client " ++ ClientName ++ " worker"}]})
- end.
-
-stop_client({Pid, Ref}) ->
- Pid ! stop,
- receive
- {stop, Ref} ->
- ok
- after ?TIMEOUT ->
- timeout
- end.
-
-kill_client_worker({Pid, Ref}) ->
- Pid ! get_worker,
- receive
- {worker, Ref, Worker} ->
- exit(Worker, kill),
- ok
- after ?TIMEOUT ->
- timeout
- end.
-
-loop(Parent, Ref, Worker, Pool) ->
- receive
- ping ->
- Parent ! {pong, Ref},
- loop(Parent, Ref, Worker, Pool);
- get_worker ->
- Parent ! {worker, Ref, Worker},
- loop(Parent, Ref, Worker, Pool);
- stop ->
- couch_replicator_httpc_pool:release_worker(Pool, Worker),
- Parent ! {stop, Ref}
- end.
-
-spawn_pool() ->
- Host = couch_config:get("httpd", "bind_address", "127.0.0.1"),
- Port = couch_config:get("httpd", "port", "5984"),
- {ok, Pool} = couch_replicator_httpc_pool:start_link(
- "http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}]),
- Pool.
-
-stop_pool(Pool) ->
- ok = couch_replicator_httpc_pool:stop(Pool).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
deleted file mode 100644
index ed7ec50..0000000
--- a/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
+++ /dev/null
@@ -1,218 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_large_atts_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
-
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
--define(ATT_SIZE_1, 2 * 1024 * 1024).
--define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
--define(DOCS_COUNT, 11).
--define(TIMEOUT_EUNIT, 30).
--define(TIMEOUT_STOP, 1000).
-
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
- couch_config:set("attachments", "compressible_types", "text/*", false),
- Source = setup(A),
- Target = setup(B),
- {Source, Target}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_USER]),
- ok.
-
-teardown(_, {Source, Target}) ->
- teardown(Source),
- teardown(Target),
-
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
-
-large_atts_test_() ->
- Pairs = [{local, local}, {local, remote},
- {remote, local}, {remote, remote}],
- {
- "Replicate docs with large attachments",
- {
- foreachx,
- fun setup/1, fun teardown/2,
- [{Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs]
- }
- }.
-
-
-should_populate_replicate_compact({From, To}, {Source, Target}) ->
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [should_populate_source(Source),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)]}}.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-
-populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- Docs = lists:foldl(
- fun(DocIdCounter, Acc) ->
- Doc = #doc{
- id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
- body = {[]},
- atts = [
- att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
- att(<<"att2">>, ?ATT_SIZE_2, <<"app/binary">>)
- ]
- },
- [Doc | Acc]
- end,
- [], lists:seq(1, DocCount)),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- couch_db:close(Db).
-
-compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
-
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
- Id = DocSource#doc.id,
-
- {ok, DocTarget} = couch_db:open_doc(TargetDb, Id),
- ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
-
- #doc{atts = SourceAtts} = DocSource,
- #doc{atts = TargetAtts} = DocTarget,
- ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
- lists:sort([N || #att{name = N} <- TargetAtts])),
-
- FunCompareAtts = fun(#att{name = AttName} = Att) ->
- {ok, AttTarget} = find_att(TargetAtts, AttName),
- SourceMd5 = att_md5(Att),
- TargetMd5 = att_md5(AttTarget),
- case AttName of
- <<"att1">> ->
- ?assertEqual(gzip, Att#att.encoding),
- ?assertEqual(gzip, AttTarget#att.encoding),
- DecSourceMd5 = att_decoded_md5(Att),
- DecTargetMd5 = att_decoded_md5(AttTarget),
- ?assertEqual(DecSourceMd5, DecTargetMd5);
- _ ->
- ?assertEqual(identity, Att#att.encoding),
- ?assertEqual(identity, AttTarget#att.encoding)
- end,
- ?assertEqual(SourceMd5, TargetMd5),
- ?assert(is_integer(Att#att.disk_len)),
- ?assert(is_integer(Att#att.att_len)),
- ?assert(is_integer(AttTarget#att.disk_len)),
- ?assert(is_integer(AttTarget#att.att_len)),
- ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
- ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
- ?assertEqual(Att#att.type, AttTarget#att.type),
- ?assertEqual(Att#att.md5, AttTarget#att.md5)
- end,
-
- lists:foreach(FunCompareAtts, SourceAtts),
-
- {ok, Acc}
- end,
-
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-att(Name, Size, Type) ->
- #att{
- name = Name,
- type = Type,
- att_len = Size,
- data = fun(Count) -> crypto:rand_bytes(Count) end
- }.
-
-find_att([], _Name) ->
- nil;
-find_att([#att{name = Name} = Att | _], Name) ->
- {ok, Att};
-find_att([_ | Rest], Name) ->
- find_att(Rest, Name).
-
-att_md5(Att) ->
- Md50 = couch_doc:att_foldl(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-att_decoded_md5(Att) ->
- Md50 = couch_doc:att_foldl_decode(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-replicate(Source, Target) ->
- RepObject = {[{<<"source">>, Source}, {<<"target">>, Target}]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _} ->
- ok
- end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
deleted file mode 100644
index 7d0de9e..0000000
--- a/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
+++ /dev/null
@@ -1,232 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_many_leaves_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
-
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
--define(DOCS_CONFLICTS, [
- {<<"doc1">>, 10},
- {<<"doc2">>, 100},
- % a number > MaxURLlength (7000) / length(DocRevisionString)
- {<<"doc3">>, 210}
-]).
--define(NUM_ATTS, 2).
--define(TIMEOUT_STOP, 1000).
--define(TIMEOUT_EUNIT, 60).
--define(i2l(I), integer_to_list(I)).
--define(io2b(Io), iolist_to_binary(Io)).
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
- Source = setup(A),
- Target = setup(B),
- {Source, Target}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_USER]),
- ok.
-
-teardown(_, {Source, Target}) ->
- teardown(Source),
- teardown(Target),
-
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
-
-docs_with_many_leaves_test_() ->
- Pairs = [{local, local}, {local, remote},
- {remote, local}, {remote, remote}],
- {
- "Replicate documents with many leaves",
- {
- foreachx,
- fun setup/1, fun teardown/2,
- [{Pair, fun should_populate_replicate_compact/2}
- || Pair <- Pairs]
- }
- }.
-
-
-should_populate_replicate_compact({From, To}, {Source, Target}) ->
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target),
- should_add_attachments_to_source(Source),
- should_replicate(Source, Target),
- should_verify_target(Source, Target)
- ]}}.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_verify_target({remote, Source}, Target) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, {remote, Target}) ->
- should_verify_target(Source, Target);
-should_verify_target(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb)
- end)}.
-
-should_add_attachments_to_source({remote, Source}) ->
- should_add_attachments_to_source(Source);
-should_add_attachments_to_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(begin
- {ok, SourceDb} = couch_db:open_int(Source, []),
- add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
- ok = couch_db:close(SourceDb)
- end)}.
-
-populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- lists:foreach(
- fun({DocId, NumConflicts}) ->
- Value = <<"0">>,
- Doc = #doc{
- id = DocId,
- body = {[ {<<"value">>, Value} ]}
- },
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
- end, ?DOCS_CONFLICTS),
- couch_db:close(Db).
-
-add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
- add_doc_siblings(Db, DocId, NumLeaves, [], []).
-
-add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
- {ok, AccRevs};
-
-add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
- Value = ?l2b(?i2l(NumLeaves)),
- Rev = couch_util:md5(Value),
- Doc = #doc{
- id = DocId,
- revs = {1, [Rev]},
- body = {[ {<<"value">>, Value} ]}
- },
- add_doc_siblings(Db, DocId, NumLeaves - 1,
- [Doc | AccDocs], [{1, Rev} | AccRevs]).
-
-verify_target(_SourceDb, _TargetDb, []) ->
- ok;
-verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]),
- {ok, TargetLookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- all,
- [conflicts, deleted_conflicts]),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
- Total = NumConflicts + 1,
- ?assertEqual(Total, length(TargetDocs)),
- lists:foreach(
- fun({SourceDoc, TargetDoc}) ->
- SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]),
- TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]),
- ?assertEqual(SourceJson, TargetJson)
- end,
- lists:zip(SourceDocs, TargetDocs)),
- verify_target(SourceDb, TargetDb, Rest).
-
-add_attachments(_SourceDb, _NumAtts, []) ->
- ok;
-add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- Total = NumConflicts + 1,
- ?assertEqual(Total, length(SourceDocs)),
- NewDocs = lists:foldl(
- fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
- NewAtts = lists:foldl(fun(I, AttAcc) ->
- AttData = crypto:rand_bytes(100),
- NewAtt = #att{
- name = ?io2b(["att_", ?i2l(I), "_",
- couch_doc:rev_to_str({Pos, Rev})]),
- type = <<"application/foobar">>,
- att_len = byte_size(AttData),
- data = AttData
- },
- [NewAtt | AttAcc]
- end, [], lists:seq(1, NumAtts)),
- [Doc#doc{atts = Atts ++ NewAtts} | Acc]
- end,
- [], SourceDocs),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
- NewRevs = [R || {ok, R} <- UpdateResults],
- ?assertEqual(length(NewDocs), length(NewRevs)),
- add_attachments(SourceDb, NumAtts, Rest).
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _} ->
- ok
- end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
deleted file mode 100644
index 0c2c30d..0000000
--- a/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
+++ /dev/null
@@ -1,260 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_missing_stubs_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
-
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
--define(REVS_LIMIT, 3).
--define(TIMEOUT_STOP, 1000).
--define(TIMEOUT_EUNIT, 30).
-
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
- Source = setup(A),
- Target = setup(B),
- {Source, Target}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_USER]),
- ok.
-
-teardown(_, {Source, Target}) ->
- teardown(Source),
- teardown(Target),
-
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
-
-missing_stubs_test_() ->
- Pairs = [{local, local}, {local, remote},
- {remote, local}, {remote, remote}],
- {
- "Replicate docs with missing stubs (COUCHDB-1365)",
- {
- foreachx,
- fun setup/1, fun teardown/2,
- [{Pair, fun should_replicate_docs_with_missed_att_stubs/2}
- || Pair <- Pairs]
- }
- }.
-
-
-should_replicate_docs_with_missed_att_stubs({From, To}, {Source, Target}) ->
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source),
- should_set_target_revs_limit(Target, ?REVS_LIMIT),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target),
- should_update_source_docs(Source, ?REVS_LIMIT * 2),
- should_replicate(Source, Target),
- should_compare_databases(Source, Target)
- ]}}.
-
-should_populate_source({remote, Source}) ->
- should_populate_source(Source);
-should_populate_source(Source) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
-
-should_replicate({remote, Source}, Target) ->
- should_replicate(db_url(Source), Target);
-should_replicate(Source, {remote, Target}) ->
- should_replicate(Source, db_url(Target));
-should_replicate(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
-
-should_set_target_revs_limit({remote, Target}, RevsLimit) ->
- should_set_target_revs_limit(Target, RevsLimit);
-should_set_target_revs_limit(Target, RevsLimit) ->
- ?_test(begin
- {ok, Db} = couch_db:open_int(Target, [?ADMIN_USER]),
- ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
- ok = couch_db:close(Db)
- end).
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-should_update_source_docs({remote, Source}, Times) ->
- should_update_source_docs(Source, Times);
-should_update_source_docs(Source, Times) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
-
-
-populate_db(DbName) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- AttData = crypto:rand_bytes(6000),
- Doc = #doc{
- id = <<"doc1">>,
- atts = [
- #att{
- name = <<"doc1_att1">>,
- type = <<"application/foobar">>,
- att_len = byte_size(AttData),
- data = AttData
- }
- ]
- },
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- couch_db:close(Db).
-
-update_db_docs(DbName, Times) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _, _} = couch_db:enum_docs(
- Db,
- fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
- {DbName, Times},
- []),
- ok = couch_db:close(Db).
-
-db_fold_fun(FullDocInfo, {DbName, Times}) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
- lists:foldl(
- fun(_, {Pos, RevId}) ->
- {ok, Db2} = couch_db:reopen(Db),
- NewDocVersion = Doc#doc{
- revs = {Pos, [RevId]},
- body = {[{<<"value">>, base64:encode(crypto:rand_bytes(100))}]}
- },
- {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
- NewRev
- end,
- {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
- lists:seq(1, Times)),
- ok = couch_db:close(Db),
- {ok, {DbName, Times}}.
-
-compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
-
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo,
- [conflicts, deleted_conflicts]),
- Id = DocSource#doc.id,
-
- {ok, DocTarget} = couch_db:open_doc(TargetDb, Id,
- [conflicts, deleted_conflicts]),
- ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
-
- ?assertEqual(couch_doc:to_json_obj(DocSource, []),
- couch_doc:to_json_obj(DocTarget, [])),
-
- #doc{atts = SourceAtts} = DocSource,
- #doc{atts = TargetAtts} = DocTarget,
- ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
- lists:sort([N || #att{name = N} <- TargetAtts])),
-
- lists:foreach(
- fun(#att{name = AttName} = Att) ->
- {ok, AttTarget} = find_att(TargetAtts, AttName),
- SourceMd5 = att_md5(Att),
- TargetMd5 = att_md5(AttTarget),
- case AttName of
- <<"att1">> ->
- ?assertEqual(gzip, Att#att.encoding),
- ?assertEqual(gzip, AttTarget#att.encoding),
- DecSourceMd5 = att_decoded_md5(Att),
- DecTargetMd5 = att_decoded_md5(AttTarget),
- ?assertEqual(DecSourceMd5, DecTargetMd5);
- _ ->
- ?assertEqual(identity, Att#att.encoding),
- ?assertEqual(identity, AttTarget#att.encoding)
- end,
- ?assertEqual(SourceMd5, TargetMd5),
- ?assert(is_integer(Att#att.disk_len)),
- ?assert(is_integer(Att#att.att_len)),
- ?assert(is_integer(AttTarget#att.disk_len)),
- ?assert(is_integer(AttTarget#att.att_len)),
- ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
- ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
- ?assertEqual(Att#att.type, AttTarget#att.type),
- ?assertEqual(Att#att.md5, AttTarget#att.md5)
- end,
- SourceAtts),
- {ok, Acc}
- end,
-
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-find_att([], _Name) ->
- nil;
-find_att([#att{name = Name} = Att | _], Name) ->
- {ok, Att};
-find_att([_ | Rest], Name) ->
- find_att(Rest, Name).
-
-att_md5(Att) ->
- Md50 = couch_doc:att_foldl(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-att_decoded_md5(Att) ->
- Md50 = couch_doc:att_foldl_decode(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _} ->
- ok
- end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_modules_load_tests.erl b/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
deleted file mode 100644
index cea4cc2..0000000
--- a/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
+++ /dev/null
@@ -1,40 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_modules_load_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
-
-
-modules_load_test_() ->
- {
- "Verify that all modules loads",
- should_load_modules()
- }.
-
-
-should_load_modules() ->
- Modules = [
- couch_replicator_api_wrap,
- couch_replicator_httpc,
- couch_replicator_httpd,
- couch_replicator_manager,
- couch_replicator_notifier,
- couch_replicator,
- couch_replicator_worker,
- couch_replicator_utils,
- couch_replicator_job_sup
- ],
- [should_load_module(Mod) || Mod <- Modules].
-
-should_load_module(Mod) ->
- {atom_to_list(Mod), ?_assertMatch({module, _}, code:load_file(Mod))}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
deleted file mode 100644
index f09a235..0000000
--- a/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
+++ /dev/null
@@ -1,200 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
--module(couch_replicator_use_checkpoints_tests).
-
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
-
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
--define(DOCS_COUNT, 100).
--define(TIMEOUT_STOP, 1000).
--define(TIMEOUT_EUNIT, 30).
--define(i2l(I), integer_to_list(I)).
--define(io2b(Io), iolist_to_binary(Io)).
-
-
-start(false) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertEqual([{<<"use_checkpoints">>,false}], CheckpointHistory);
- (_) ->
- ok
- end;
-start(true) ->
- fun
- ({finished, _, {CheckpointHistory}}) ->
- ?assertNotEqual(false, lists:keyfind(<<"session_id">>,
- 1, CheckpointHistory));
- (_) ->
- ok
- end.
-
-stop(_, _) ->
- ok.
-
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
- ok = couch_db:close(Db),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({_, Fun, {A, B}}) ->
- {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
- {ok, Listener} = couch_replicator_notifier:start_link(Fun),
- Source = setup(A),
- Target = setup(B),
- {Source, Target, Listener}.
-
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_USER]),
- ok.
-
-teardown(_, {Source, Target, Listener}) ->
- teardown(Source),
- teardown(Target),
-
- couch_replicator_notifier:stop(Listener),
- Pid = whereis(couch_server_sup),
- erlang:monitor(process, Pid),
- couch_server_sup:stop(),
- receive
- {'DOWN', _, _, Pid, _} ->
- ok
- after ?TIMEOUT_STOP ->
- throw({timeout, server_stop})
- end.
-
-
-use_checkpoints_test_() ->
- {
- "Replication use_checkpoints feature tests",
- {
- foreachx,
- fun start/1, fun stop/2,
- [{UseCheckpoints, fun use_checkpoints_tests/2}
- || UseCheckpoints <- [false, true]]
- }
- }.
-
-use_checkpoints_tests(UseCheckpoints, Fun) ->
- Pairs = [{local, local}, {local, remote},
- {remote, local}, {remote, remote}],
- {
- "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
- {
- foreachx,
- fun setup/1, fun teardown/2,
- [{{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
- || Pair <- Pairs]
- }
- }.
-
-should_test_checkpoints({UseCheckpoints, _, {From, To}}, {Source, Target, _}) ->
- should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
-should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_populate_source(Source, ?DOCS_COUNT),
- should_replicate(Source, Target, UseCheckpoints),
- should_compare_databases(Source, Target)
- ]}}.
-
-should_populate_source({remote, Source}, DocCount) ->
- should_populate_source(Source, DocCount);
-should_populate_source(Source, DocCount) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
-
-should_replicate({remote, Source}, Target, UseCheckpoints) ->
- should_replicate(db_url(Source), Target, UseCheckpoints);
-should_replicate(Source, {remote, Target}, UseCheckpoints) ->
- should_replicate(Source, db_url(Target), UseCheckpoints);
-should_replicate(Source, Target, UseCheckpoints) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
-
-should_compare_databases({remote, Source}, Target) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, {remote, Target}) ->
- should_compare_databases(Source, Target);
-should_compare_databases(Source, Target) ->
- {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
-
-
-populate_db(DbName, DocCount) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- Docs = lists:foldl(
- fun(DocIdCounter, Acc) ->
- Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
- Value = ?io2b(["val", ?i2l(DocIdCounter)]),
- Doc = #doc{
- id = Id,
- body = {[ {<<"value">>, Value} ]}
- },
- [Doc | Acc]
- end,
- [], lists:seq(1, DocCount)),
- {ok, _} = couch_db:update_docs(Db, Docs, []),
- ok = couch_db:close(Db).
-
-compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(Source, []),
- {ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget = case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, ?LINE},
- {reason, lists:concat(["Error opening document '",
- ?b2l(DocId), "' from target: ",
- couch_util:to_list(Error)])}]})
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- ?assertEqual(DocJson, DocTargetJson),
- {ok, Acc}
- end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-replicate(Source, Target, UseCheckpoints) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target},
- {<<"use_checkpoints">>, UseCheckpoints}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _} ->
- ok
- end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
new file mode 100644
index 0000000..4e019fe
--- /dev/null
+++ b/test/couch_replicator_compact_tests.erl
@@ -0,0 +1,448 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_compact_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+-include_lib("couch_replicator/src/couch_replicator.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
+-define(DELAY, 100).
+-define(TIMEOUT, 30000).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_WRITER, 3000).
+-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 5).
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+compact_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Compaction during replication tests",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {ok, RepPid, RepId} = replicate(Source, Target),
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_run_replication(RepPid, RepId, Source, Target),
+ should_all_processes_be_alive(RepPid, Source, Target),
+ should_populate_and_compact(RepPid, Source, Target, 50, 5),
+ should_wait_target_in_sync(Source, Target),
+ should_ensure_replication_still_running(RepPid, RepId, Source, Target),
+ should_cancel_replication(RepId, RepPid),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_all_processes_be_alive(RepPid, Source, Target) ->
+ ?_test(begin
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb#db.main_pid)),
+ ?assert(is_process_alive(TargetDb#db.main_pid))
+ end).
+
+should_run_replication(RepPid, RepId, Source, Target) ->
+ ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+
+should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
+ ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+
+check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
+ Source = case Src of
+ {remote, NameSrc} ->
+ <<(db_url(NameSrc))/binary, $/>>;
+ _ ->
+ Src
+ end,
+ Target = case Tgt of
+ {remote, NameTgt} ->
+ <<(db_url(NameTgt))/binary, $/>>;
+ _ ->
+ Tgt
+ end,
+ FullRepId = ?l2b(BaseId ++ Ext),
+ Pid = ?l2b(pid_to_list(RepPid)),
+ [RepTask] = couch_task_status:all(),
+ ?assertEqual(Pid, couch_util:get_value(pid, RepTask)),
+ ?assertEqual(FullRepId, couch_util:get_value(replication_id, RepTask)),
+ ?assertEqual(true, couch_util:get_value(continuous, RepTask)),
+ ?assertEqual(Source, couch_util:get_value(source, RepTask)),
+ ?assertEqual(Target, couch_util:get_value(target, RepTask)),
+ ?assert(is_integer(couch_util:get_value(docs_read, RepTask))),
+ ?assert(is_integer(couch_util:get_value(docs_written, RepTask))),
+ ?assert(is_integer(couch_util:get_value(doc_write_failures, RepTask))),
+ ?assert(is_integer(couch_util:get_value(revisions_checked, RepTask))),
+ ?assert(is_integer(couch_util:get_value(missing_revisions_found, RepTask))),
+ ?assert(is_integer(couch_util:get_value(checkpointed_source_seq, RepTask))),
+ ?assert(is_integer(couch_util:get_value(source_seq, RepTask))),
+ Progress = couch_util:get_value(progress, RepTask),
+ ?assert(is_integer(Progress)),
+ ?assert(Progress =< 100).
+
+should_cancel_replication(RepId, RepPid) ->
+ ?_assertNot(begin
+ {ok, _} = couch_replicator:cancel_replication(RepId),
+ is_process_alive(RepPid)
+ end).
+
+should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb0} = reopen_db(Source),
+ Writer = spawn_writer(SourceDb0),
+ lists:foreach(
+ fun(N) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ pause_writer(Writer),
+
+ compact_db("source", SourceDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb#db.main_pid)),
+ check_ref_counter("source", SourceDb),
+
+ compact_db("target", TargetDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(TargetDb#db.main_pid)),
+ check_ref_counter("target", TargetDb),
+
+ {ok, SourceDb2} = reopen_db(SourceDb),
+ {ok, TargetDb2} = reopen_db(TargetDb),
+
+ resume_writer(Writer),
+ wait_writer(Writer, BatchSize * N),
+
+ compact_db("source", SourceDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb2#db.main_pid)),
+ pause_writer(Writer),
+ check_ref_counter("source", SourceDb2),
+ resume_writer(Writer),
+
+ compact_db("target", TargetDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(TargetDb2#db.main_pid)),
+ pause_writer(Writer),
+ check_ref_counter("target", TargetDb2),
+ resume_writer(Writer)
+ end, lists:seq(1, Rounds)),
+ stop_writer(Writer)
+ end)}.
+
+should_wait_target_in_sync({remote, Source}, Target) ->
+ should_wait_target_in_sync(Source, Target);
+should_wait_target_in_sync(Source, {remote, Target}) ->
+ should_wait_target_in_sync(Source, Target);
+should_wait_target_in_sync(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_assert(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
+ ok = couch_db:close(SourceDb),
+ SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ wait_target_in_sync_loop(SourceDocCount, Target, 300)
+ end)}.
+
+wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, "Could not get source and target databases in sync"}]});
+wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
+ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
+wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
+ {ok, Target} = couch_db:open_int(TargetName, []),
+ {ok, TargetInfo} = couch_db:get_db_info(Target),
+ ok = couch_db:close(Target),
+ TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ case TargetDocCount == DocCount of
+ true ->
+ true;
+ false ->
+ ok = timer:sleep(?DELAY),
+ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
+ end.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, 35, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Error opening document '",
+ ?b2l(DocId), "' from target: ",
+ couch_util:to_list(Error)])}]})
+ end,
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb)
+ end)}.
+
+
+reopen_db({remote, Db}) ->
+ reopen_db(Db);
+reopen_db(#db{name=DbName}) ->
+ reopen_db(DbName);
+reopen_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ ok = couch_db:close(Db),
+ {ok, Db}.
+
+compact_db(Type, #db{name = Name}) ->
+ {ok, Db} = couch_db:open_int(Name, []),
+ {ok, CompactPid} = couch_db:start_compact(Db),
+ MonRef = erlang:monitor(process, CompactPid),
+ receive
+ {'DOWN', MonRef, process, CompactPid, normal} ->
+ ok;
+ {'DOWN', MonRef, process, CompactPid, Reason} ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason,
+ lists:concat(["Error compacting ", Type, " database ",
+ ?b2l(Name), ": ",
+ couch_util:to_list(Reason)])}]})
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Compaction for ", Type, " database ",
+ ?b2l(Name), " didn't finish"])}]})
+ end,
+ ok = couch_db:close(Db).
+
+check_ref_counter(Type, #db{name = Name, fd_ref_counter = OldRefCounter}) ->
+ MonRef = erlang:monitor(process, OldRefCounter),
+ receive
+ {'DOWN', MonRef, process, OldRefCounter, _} ->
+ ok
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Old ", Type,
+ " database ref counter didn't"
+ " terminate"])}]})
+ end,
+ {ok, #db{fd_ref_counter = NewRefCounter} = Db} = couch_db:open_int(Name, []),
+ ok = couch_db:close(Db),
+ ?assertNotEqual(OldRefCounter, NewRefCounter).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate({remote, Db}, Target) ->
+ replicate(db_url(Db), Target);
+
+replicate(Source, {remote, Db}) ->
+ replicate(Source, db_url(Db));
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target},
+ {<<"continuous">>, true}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ {ok, Pid, Rep#rep.id}.
+
+
+wait_writer(Pid, NumDocs) ->
+ case get_writer_num_docs_written(Pid) of
+ N when N >= NumDocs ->
+ ok;
+ _ ->
+ wait_writer(Pid, NumDocs)
+ end.
+
+spawn_writer(Db) ->
+ Parent = self(),
+ Pid = spawn(fun() -> writer_loop(Db, Parent, 0) end),
+ Pid.
+
+
+pause_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {pause, Ref},
+ receive
+ {paused, Ref} ->
+ ok
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Failed to pause source database writer"}]})
+ end.
+
+resume_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {continue, Ref},
+ receive
+ {ok, Ref} ->
+ ok
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Failed to pause source database writer"}]})
+ end.
+
+get_writer_num_docs_written(Pid) ->
+ Ref = make_ref(),
+ Pid ! {get_count, Ref},
+ receive
+ {count, Ref, Count} ->
+ Count
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout getting number of documents written"
+ " from source database writer"}]})
+ end.
+
+stop_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {stop, Ref},
+ receive
+ {stopped, Ref, DocsWritten} ->
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _Reason} ->
+ DocsWritten
+ after ?TIMEOUT ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout stopping source database writer"}]})
+ end
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout stopping source database writer"}]})
+ end.
+
+writer_loop(#db{name = DbName}, Parent, Counter) ->
+ {ok, Data} = file:read_file(?ATTFILE),
+ maybe_pause(Parent, Counter),
+ Doc = couch_doc:from_json_obj({[
+ {<<"_id">>, ?l2b(integer_to_list(Counter + 1))},
+ {<<"value">>, Counter + 1},
+ {<<"_attachments">>, {[
+ {<<"icon1.png">>, {[
+ {<<"data">>, base64:encode(Data)},
+ {<<"content_type">>, <<"image/png">>}
+ ]}},
+ {<<"icon2.png">>, {[
+ {<<"data">>, base64:encode(iolist_to_binary([Data, Data]))},
+ {<<"content_type">>, <<"image/png">>}
+ ]}}
+ ]}}
+ ]}),
+ maybe_pause(Parent, Counter),
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ ok = couch_db:close(Db),
+ receive
+ {get_count, Ref} ->
+ Parent ! {count, Ref, Counter + 1},
+ writer_loop(Db, Parent, Counter + 1);
+ {stop, Ref} ->
+ Parent ! {stopped, Ref, Counter + 1}
+ after 0 ->
+ timer:sleep(?DELAY),
+ writer_loop(Db, Parent, Counter + 1)
+ end.
+
+maybe_pause(Parent, Counter) ->
+ receive
+ {get_count, Ref} ->
+ Parent ! {count, Ref, Counter};
+ {pause, Ref} ->
+ Parent ! {paused, Ref},
+ receive
+ {continue, Ref2} ->
+ Parent ! {ok, Ref2}
+ end
+ after 0 ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2850b946/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_httpc_pool_tests.erl b/test/couch_replicator_httpc_pool_tests.erl
new file mode 100644
index 0000000..6bede4c
--- /dev/null
+++ b/test/couch_replicator_httpc_pool_tests.erl
@@ -0,0 +1,189 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_httpc_pool_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
+-define(TIMEOUT, 1000).
+
+
+start() ->
+ {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Pid.
+
+stop(Pid) ->
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT ->
+ throw({timeout, server_stop})
+ end.
+
+setup() ->
+ spawn_pool().
+
+teardown(Pool) ->
+ stop_pool(Pool).
+
+
+httpc_pool_test_() ->
+ {
+ "httpc pool tests",
+ {
+ setup,
+ fun start/0, fun stop/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun should_block_new_clients_when_full/1,
+ fun should_replace_worker_on_death/1
+ ]
+ }
+ }
+ }.
+
+
+should_block_new_clients_when_full(Pool) ->
+ ?_test(begin
+ Client1 = spawn_client(Pool),
+ Client2 = spawn_client(Pool),
+ Client3 = spawn_client(Pool),
+
+ ?assertEqual(ok, ping_client(Client1)),
+ ?assertEqual(ok, ping_client(Client2)),
+ ?assertEqual(ok, ping_client(Client3)),
+
+ Worker1 = get_client_worker(Client1, "1"),
+ Worker2 = get_client_worker(Client2, "2"),
+ Worker3 = get_client_worker(Client3, "3"),
+
+ ?assert(is_process_alive(Worker1)),
+ ?assert(is_process_alive(Worker2)),
+ ?assert(is_process_alive(Worker3)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertNotEqual(Worker2, Worker3),
+ ?assertNotEqual(Worker3, Worker1),
+
+ Client4 = spawn_client(Pool),
+ ?assertEqual(timeout, ping_client(Client4)),
+
+ ?assertEqual(ok, stop_client(Client1)),
+ ?assertEqual(ok, ping_client(Client4)),
+
+ Worker4 = get_client_worker(Client4, "4"),
+ ?assertEqual(Worker1, Worker4),
+
+ lists:foreach(
+ fun(C) ->
+ ?assertEqual(ok, stop_client(C))
+ end, [Client2, Client3, Client4])
+ end).
+
+should_replace_worker_on_death(Pool) ->
+ ?_test(begin
+ Client1 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client1)),
+ Worker1 = get_client_worker(Client1, "1"),
+ ?assert(is_process_alive(Worker1)),
+
+ ?assertEqual(ok, kill_client_worker(Client1)),
+ ?assertNot(is_process_alive(Worker1)),
+ ?assertEqual(ok, stop_client(Client1)),
+
+ Client2 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client2)),
+ Worker2 = get_client_worker(Client2, "2"),
+ ?assert(is_process_alive(Worker2)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertEqual(ok, stop_client(Client2))
+ end).
+
+
+spawn_client(Pool) ->
+ Parent = self(),
+ Ref = make_ref(),
+ Pid = spawn(fun() ->
+ {ok, Worker} = couch_replicator_httpc_pool:get_worker(Pool),
+ loop(Parent, Ref, Worker, Pool)
+ end),
+ {Pid, Ref}.
+
+ping_client({Pid, Ref}) ->
+ Pid ! ping,
+ receive
+ {pong, Ref} ->
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+get_client_worker({Pid, Ref}, ClientName) ->
+ Pid ! get_worker,
+ receive
+ {worker, Ref, Worker} ->
+ Worker
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, "Timeout getting client " ++ ClientName ++ " worker"}]})
+ end.
+
+stop_client({Pid, Ref}) ->
+ Pid ! stop,
+ receive
+ {stop, Ref} ->
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+kill_client_worker({Pid, Ref}) ->
+ Pid ! get_worker,
+ receive
+ {worker, Ref, Worker} ->
+ exit(Worker, kill),
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+loop(Parent, Ref, Worker, Pool) ->
+ receive
+ ping ->
+ Parent ! {pong, Ref},
+ loop(Parent, Ref, Worker, Pool);
+ get_worker ->
+ Parent ! {worker, Ref, Worker},
+ loop(Parent, Ref, Worker, Pool);
+ stop ->
+ couch_replicator_httpc_pool:release_worker(Pool, Worker),
+ Parent ! {stop, Ref}
+ end.
+
+spawn_pool() ->
+ Host = couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = couch_config:get("httpd", "port", "5984"),
+ {ok, Pool} = couch_replicator_httpc_pool:start_link(
+ "http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}]),
+ Pool.
+
+stop_pool(Pool) ->
+ ok = couch_replicator_httpc_pool:stop(Pool).
[12/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Update include paths
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/3233c26d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/3233c26d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/3233c26d
Branch: refs/heads/master
Commit: 3233c26d1aeebfacbf6ead000c236bc64e8d7e4e
Parents: 2850b94
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 15:36:07 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:41 2014 -0700
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 4 ++--
test/couch_replicator_httpc_pool_tests.erl | 4 ++--
test/couch_replicator_large_atts_tests.erl | 4 ++--
test/couch_replicator_many_leaves_tests.erl | 4 ++--
test/couch_replicator_missing_stubs_tests.erl | 4 ++--
test/couch_replicator_modules_load_tests.erl | 2 +-
test/couch_replicator_use_checkpoints_tests.erl | 4 ++--
7 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 4e019fe..899bc15 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_compact_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_httpc_pool_tests.erl b/test/couch_replicator_httpc_pool_tests.erl
index 6bede4c..c6c6ad1 100644
--- a/test/couch_replicator_httpc_pool_tests.erl
+++ b/test/couch_replicator_httpc_pool_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_httpc_pool_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
-define(TIMEOUT, 1000).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
index ed7ec50..23cd3ea 100644
--- a/test/couch_replicator_large_atts_tests.erl
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_large_atts_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
index 7d0de9e..215bc55 100644
--- a/test/couch_replicator_many_leaves_tests.erl
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_many_leaves_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index 0c2c30d..3371089 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_missing_stubs_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_modules_load_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_modules_load_tests.erl b/test/couch_replicator_modules_load_tests.erl
index cea4cc2..96a9346 100644
--- a/test/couch_replicator_modules_load_tests.erl
+++ b/test/couch_replicator_modules_load_tests.erl
@@ -12,7 +12,7 @@
-module(couch_replicator_modules_load_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
modules_load_test_() ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3233c26d/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
index f09a235..6381366 100644
--- a/test/couch_replicator_use_checkpoints_tests.erl
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -12,8 +12,8 @@
-module(couch_replicator_use_checkpoints_tests).
--include("../../../test/couchdb/couch_eunit.hrl").
--include_lib("couchdb/couch_db.hrl").
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
[14/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Use ?ADMIN macros from couch_eunit.hrl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/62a3db93
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/62a3db93
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/62a3db93
Branch: refs/heads/master
Commit: 62a3db9397eba9dd5984bb67365fab2d75513e1d
Parents: 3233c26
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 16:38:40 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:41 2014 -0700
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 2 --
test/couch_replicator_httpc_pool_tests.erl | 1 -
test/couch_replicator_large_atts_tests.erl | 2 --
test/couch_replicator_many_leaves_tests.erl | 2 --
test/couch_replicator_missing_stubs_tests.erl | 2 --
test/couch_replicator_use_checkpoints_tests.erl | 2 --
6 files changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 899bc15..407695e 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -16,8 +16,6 @@
-include_lib("couch/include/couch_db.hrl").
-include_lib("couch_replicator/src/couch_replicator.hrl").
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
-define(DELAY, 100).
-define(TIMEOUT, 30000).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_httpc_pool_tests.erl b/test/couch_replicator_httpc_pool_tests.erl
index c6c6ad1..b65d174 100644
--- a/test/couch_replicator_httpc_pool_tests.erl
+++ b/test/couch_replicator_httpc_pool_tests.erl
@@ -15,7 +15,6 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
-define(TIMEOUT, 1000).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_large_atts_tests.erl b/test/couch_replicator_large_atts_tests.erl
index 23cd3ea..b821a18 100644
--- a/test/couch_replicator_large_atts_tests.erl
+++ b/test/couch_replicator_large_atts_tests.erl
@@ -15,8 +15,6 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
-define(ATT_SIZE_1, 2 * 1024 * 1024).
-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
-define(DOCS_COUNT, 11).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_many_leaves_tests.erl b/test/couch_replicator_many_leaves_tests.erl
index 215bc55..0a75da7 100644
--- a/test/couch_replicator_many_leaves_tests.erl
+++ b/test/couch_replicator_many_leaves_tests.erl
@@ -15,8 +15,6 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
-define(DOCS_CONFLICTS, [
{<<"doc1">>, 10},
{<<"doc2">>, 100},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index 3371089..713ab01 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -15,8 +15,6 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
-define(REVS_LIMIT, 3).
-define(TIMEOUT_STOP, 1000).
-define(TIMEOUT_EUNIT, 30).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62a3db93/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
index 6381366..69ac661 100644
--- a/test/couch_replicator_use_checkpoints_tests.erl
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -15,8 +15,6 @@
-include_lib("couch/include/couch_eunit.hrl").
-include_lib("couch/include/couch_db.hrl").
--define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
--define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
-define(DOCS_COUNT, 100).
-define(TIMEOUT_STOP, 1000).
-define(TIMEOUT_EUNIT, 30).
[06/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/03-replication-compact.t etap test suite to eunit
Split big test fun into smaller steps, optimize timeouts.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/191af63b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/191af63b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/191af63b
Branch: refs/heads/master
Commit: 191af63bb497b813a2bd2d06850cacc4e9428485
Parents: 450edc0
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 16 00:29:29 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_compact_tests.erl | 448 +++++++++++++++++
test/03-replication-compact.t | 493 -------------------
2 files changed, 448 insertions(+), 493 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/191af63b/src/couch_replicator/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_compact_tests.erl b/src/couch_replicator/test/couch_replicator_compact_tests.erl
new file mode 100644
index 0000000..4e019fe
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_compact_tests.erl
@@ -0,0 +1,448 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_compact_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+-include_lib("couch_replicator/src/couch_replicator.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(ATTFILE, filename:join([?FIXTURESDIR, "logo.png"])).
+-define(DELAY, 100).
+-define(TIMEOUT, 30000).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_WRITER, 3000).
+-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 5).
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+compact_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Compaction during replication tests",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {ok, RepPid, RepId} = replicate(Source, Target),
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_run_replication(RepPid, RepId, Source, Target),
+ should_all_processes_be_alive(RepPid, Source, Target),
+ should_populate_and_compact(RepPid, Source, Target, 50, 5),
+ should_wait_target_in_sync(Source, Target),
+ should_ensure_replication_still_running(RepPid, RepId, Source, Target),
+ should_cancel_replication(RepId, RepPid),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_all_processes_be_alive(RepPid, Source, Target) ->
+ ?_test(begin
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb#db.main_pid)),
+ ?assert(is_process_alive(TargetDb#db.main_pid))
+ end).
+
+should_run_replication(RepPid, RepId, Source, Target) ->
+ ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+
+should_ensure_replication_still_running(RepPid, RepId, Source, Target) ->
+ ?_test(check_active_tasks(RepPid, RepId, Source, Target)).
+
+check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
+ Source = case Src of
+ {remote, NameSrc} ->
+ <<(db_url(NameSrc))/binary, $/>>;
+ _ ->
+ Src
+ end,
+ Target = case Tgt of
+ {remote, NameTgt} ->
+ <<(db_url(NameTgt))/binary, $/>>;
+ _ ->
+ Tgt
+ end,
+ FullRepId = ?l2b(BaseId ++ Ext),
+ Pid = ?l2b(pid_to_list(RepPid)),
+ [RepTask] = couch_task_status:all(),
+ ?assertEqual(Pid, couch_util:get_value(pid, RepTask)),
+ ?assertEqual(FullRepId, couch_util:get_value(replication_id, RepTask)),
+ ?assertEqual(true, couch_util:get_value(continuous, RepTask)),
+ ?assertEqual(Source, couch_util:get_value(source, RepTask)),
+ ?assertEqual(Target, couch_util:get_value(target, RepTask)),
+ ?assert(is_integer(couch_util:get_value(docs_read, RepTask))),
+ ?assert(is_integer(couch_util:get_value(docs_written, RepTask))),
+ ?assert(is_integer(couch_util:get_value(doc_write_failures, RepTask))),
+ ?assert(is_integer(couch_util:get_value(revisions_checked, RepTask))),
+ ?assert(is_integer(couch_util:get_value(missing_revisions_found, RepTask))),
+ ?assert(is_integer(couch_util:get_value(checkpointed_source_seq, RepTask))),
+ ?assert(is_integer(couch_util:get_value(source_seq, RepTask))),
+ Progress = couch_util:get_value(progress, RepTask),
+ ?assert(is_integer(Progress)),
+ ?assert(Progress =< 100).
+
+should_cancel_replication(RepId, RepPid) ->
+ ?_assertNot(begin
+ {ok, _} = couch_replicator:cancel_replication(RepId),
+ is_process_alive(RepPid)
+ end).
+
+should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb0} = reopen_db(Source),
+ Writer = spawn_writer(SourceDb0),
+ lists:foreach(
+ fun(N) ->
+ {ok, SourceDb} = reopen_db(Source),
+ {ok, TargetDb} = reopen_db(Target),
+ pause_writer(Writer),
+
+ compact_db("source", SourceDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb#db.main_pid)),
+ check_ref_counter("source", SourceDb),
+
+ compact_db("target", TargetDb),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(TargetDb#db.main_pid)),
+ check_ref_counter("target", TargetDb),
+
+ {ok, SourceDb2} = reopen_db(SourceDb),
+ {ok, TargetDb2} = reopen_db(TargetDb),
+
+ resume_writer(Writer),
+ wait_writer(Writer, BatchSize * N),
+
+ compact_db("source", SourceDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(SourceDb2#db.main_pid)),
+ pause_writer(Writer),
+ check_ref_counter("source", SourceDb2),
+ resume_writer(Writer),
+
+ compact_db("target", TargetDb2),
+ ?assert(is_process_alive(RepPid)),
+ ?assert(is_process_alive(TargetDb2#db.main_pid)),
+ pause_writer(Writer),
+ check_ref_counter("target", TargetDb2),
+ resume_writer(Writer)
+ end, lists:seq(1, Rounds)),
+ stop_writer(Writer)
+ end)}.
+
+should_wait_target_in_sync({remote, Source}, Target) ->
+ should_wait_target_in_sync(Source, Target);
+should_wait_target_in_sync(Source, {remote, Target}) ->
+ should_wait_target_in_sync(Source, Target);
+should_wait_target_in_sync(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_assert(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, SourceInfo} = couch_db:get_db_info(SourceDb),
+ ok = couch_db:close(SourceDb),
+ SourceDocCount = couch_util:get_value(doc_count, SourceInfo),
+ wait_target_in_sync_loop(SourceDocCount, Target, 300)
+ end)}.
+
+wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, "Could not get source and target databases in sync"}]});
+wait_target_in_sync_loop(DocCount, {remote, TargetName}, RetriesLeft) ->
+ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft);
+wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
+ {ok, Target} = couch_db:open_int(TargetName, []),
+ {ok, TargetInfo} = couch_db:get_db_info(Target),
+ ok = couch_db:close(Target),
+ TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
+ case TargetDocCount == DocCount of
+ true ->
+ true;
+ false ->
+ ok = timer:sleep(?DELAY),
+ wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
+ end.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, 35, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Error opening document '",
+ ?b2l(DocId), "' from target: ",
+ couch_util:to_list(Error)])}]})
+ end,
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb)
+ end)}.
+
+
+reopen_db({remote, Db}) ->
+ reopen_db(Db);
+reopen_db(#db{name=DbName}) ->
+ reopen_db(DbName);
+reopen_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ ok = couch_db:close(Db),
+ {ok, Db}.
+
+compact_db(Type, #db{name = Name}) ->
+ {ok, Db} = couch_db:open_int(Name, []),
+ {ok, CompactPid} = couch_db:start_compact(Db),
+ MonRef = erlang:monitor(process, CompactPid),
+ receive
+ {'DOWN', MonRef, process, CompactPid, normal} ->
+ ok;
+ {'DOWN', MonRef, process, CompactPid, Reason} ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason,
+ lists:concat(["Error compacting ", Type, " database ",
+ ?b2l(Name), ": ",
+ couch_util:to_list(Reason)])}]})
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Compaction for ", Type, " database ",
+ ?b2l(Name), " didn't finish"])}]})
+ end,
+ ok = couch_db:close(Db).
+
+check_ref_counter(Type, #db{name = Name, fd_ref_counter = OldRefCounter}) ->
+ MonRef = erlang:monitor(process, OldRefCounter),
+ receive
+ {'DOWN', MonRef, process, OldRefCounter, _} ->
+ ok
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Old ", Type,
+ " database ref counter didn't"
+ " terminate"])}]})
+ end,
+ {ok, #db{fd_ref_counter = NewRefCounter} = Db} = couch_db:open_int(Name, []),
+ ok = couch_db:close(Db),
+ ?assertNotEqual(OldRefCounter, NewRefCounter).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate({remote, Db}, Target) ->
+ replicate(db_url(Db), Target);
+
+replicate(Source, {remote, Db}) ->
+ replicate(Source, db_url(Db));
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target},
+ {<<"continuous">>, true}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ {ok, Pid, Rep#rep.id}.
+
+
+wait_writer(Pid, NumDocs) ->
+ case get_writer_num_docs_written(Pid) of
+ N when N >= NumDocs ->
+ ok;
+ _ ->
+ wait_writer(Pid, NumDocs)
+ end.
+
+spawn_writer(Db) ->
+ Parent = self(),
+ Pid = spawn(fun() -> writer_loop(Db, Parent, 0) end),
+ Pid.
+
+
+pause_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {pause, Ref},
+ receive
+ {paused, Ref} ->
+ ok
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Failed to pause source database writer"}]})
+ end.
+
+resume_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {continue, Ref},
+ receive
+ {ok, Ref} ->
+ ok
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Failed to pause source database writer"}]})
+ end.
+
+get_writer_num_docs_written(Pid) ->
+ Ref = make_ref(),
+ Pid ! {get_count, Ref},
+ receive
+ {count, Ref, Count} ->
+ Count
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout getting number of documents written"
+ " from source database writer"}]})
+ end.
+
+stop_writer(Pid) ->
+ Ref = make_ref(),
+ Pid ! {stop, Ref},
+ receive
+ {stopped, Ref, DocsWritten} ->
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _Reason} ->
+ DocsWritten
+ after ?TIMEOUT ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout stopping source database writer"}]})
+ end
+ after ?TIMEOUT_WRITER ->
+ erlang:error({assertion_failed,
+ [{module, ?MODULE},
+ {line, ?LINE},
+ {reason, "Timeout stopping source database writer"}]})
+ end.
+
+writer_loop(#db{name = DbName}, Parent, Counter) ->
+ {ok, Data} = file:read_file(?ATTFILE),
+ maybe_pause(Parent, Counter),
+ Doc = couch_doc:from_json_obj({[
+ {<<"_id">>, ?l2b(integer_to_list(Counter + 1))},
+ {<<"value">>, Counter + 1},
+ {<<"_attachments">>, {[
+ {<<"icon1.png">>, {[
+ {<<"data">>, base64:encode(Data)},
+ {<<"content_type">>, <<"image/png">>}
+ ]}},
+ {<<"icon2.png">>, {[
+ {<<"data">>, base64:encode(iolist_to_binary([Data, Data]))},
+ {<<"content_type">>, <<"image/png">>}
+ ]}}
+ ]}}
+ ]}),
+ maybe_pause(Parent, Counter),
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ ok = couch_db:close(Db),
+ receive
+ {get_count, Ref} ->
+ Parent ! {count, Ref, Counter + 1},
+ writer_loop(Db, Parent, Counter + 1);
+ {stop, Ref} ->
+ Parent ! {stopped, Ref, Counter + 1}
+ after 0 ->
+ timer:sleep(?DELAY),
+ writer_loop(Db, Parent, Counter + 1)
+ end.
+
+maybe_pause(Parent, Counter) ->
+ receive
+ {get_count, Ref} ->
+ Parent ! {count, Ref, Counter};
+ {pause, Ref} ->
+ Parent ! {paused, Ref},
+ receive
+ {continue, Ref2} ->
+ Parent ! {ok, Ref2}
+ end
+ after 0 ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/191af63b/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
deleted file mode 100755
index 888d123..0000000
--- a/test/03-replication-compact.t
+++ /dev/null
@@ -1,493 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Verify that compacting databases that are being used as the source or
-% target of a replication doesn't affect the replication and that the
-% replication doesn't hold their reference counters forever.
-
--mode(compile).
-
--define(b2l(B), binary_to_list(B)).
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(db, {
- main_pid = nil,
- compactor_pid = nil,
- instance_start_time, % number of microsecs since jan 1 1970 as a binary string
- fd,
- fd_monitor,
- header = nil,
- committed_update_seq,
- id_tree,
- seq_tree,
- local_tree,
- update_seq,
- name,
- filepath,
- validate_doc_funs = [],
- security = [],
- security_ptr = nil,
- user_ctx = #user_ctx{},
- waiting_delayed_commit = nil,
- revs_limit = 1000,
- fsync_options = [],
- options = [],
- compression,
- before_doc_update,
- after_doc_read
-}).
-
--record(rep, {
- id,
- source,
- target,
- options,
- user_ctx,
- doc_id
-}).
-
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-
-main(_) ->
- test_util:run(376, fun() -> test() end).
-
-
-test() ->
- test_util:start_couch(),
- ibrowse:start(),
-
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- lists:foreach(
- fun({Source, Target}) ->
- {ok, SourceDb} = create_db(source_db_name()),
- etap:is(couch_db:is_idle(SourceDb), true,
- "Source database is idle before starting replication"),
-
- {ok, TargetDb} = create_db(target_db_name()),
- etap:is(couch_db:is_idle(TargetDb), true,
- "Target database is idle before starting replication"),
-
- {ok, RepPid, RepId} = replicate(Source, Target),
- check_active_tasks(RepPid, RepId, Source, Target),
- {ok, DocsWritten} = populate_and_compact_test(
- RepPid, SourceDb, TargetDb),
-
- wait_target_in_sync(DocsWritten, TargetDb),
- check_active_tasks(RepPid, RepId, Source, Target),
- cancel_replication(RepId, RepPid),
- compare_dbs(SourceDb, TargetDb),
-
- delete_db(SourceDb),
- delete_db(TargetDb),
- ok = test_util:stop_couch(),
- ok = timer:sleep(1000),
- ok = test_util:start_couch()
- end,
- Pairs),
-
- ok = test_util:stop_couch(),
- ok.
-
-
-populate_and_compact_test(RepPid, SourceDb0, TargetDb0) ->
- etap:is(is_process_alive(RepPid), true, "Replication process is alive"),
- check_db_alive("source", SourceDb0),
- check_db_alive("target", TargetDb0),
-
- Writer = spawn_writer(SourceDb0),
-
- lists:foldl(
- fun(_, {SourceDb, TargetDb, DocCount}) ->
- pause_writer(Writer),
-
- compact_db("source", SourceDb),
- etap:is(is_process_alive(RepPid), true,
- "Replication process is alive after source database compaction"),
- check_db_alive("source", SourceDb),
- check_fd("source", SourceDb),
-
- compact_db("target", TargetDb),
- etap:is(is_process_alive(RepPid), true,
- "Replication process is alive after target database compaction"),
- check_db_alive("target", TargetDb),
- check_fd("target", TargetDb),
-
- {ok, SourceDb2} = reopen_db(SourceDb),
- {ok, TargetDb2} = reopen_db(TargetDb),
-
- resume_writer(Writer),
- wait_writer(Writer, DocCount),
-
- compact_db("source", SourceDb2),
- etap:is(is_process_alive(RepPid), true,
- "Replication process is alive after source database compaction"),
- check_db_alive("source", SourceDb2),
- pause_writer(Writer),
- check_fd("source", SourceDb2),
- resume_writer(Writer),
-
- compact_db("target", TargetDb2),
- etap:is(is_process_alive(RepPid), true,
- "Replication process is alive after target database compaction"),
- check_db_alive("target", TargetDb2),
- pause_writer(Writer),
- check_fd("target", TargetDb2),
- resume_writer(Writer),
-
- {ok, SourceDb3} = reopen_db(SourceDb2),
- {ok, TargetDb3} = reopen_db(TargetDb2),
- {SourceDb3, TargetDb3, DocCount + 50}
- end,
- {SourceDb0, TargetDb0, 50}, lists:seq(1, 5)),
-
- DocsWritten = stop_writer(Writer),
- {ok, DocsWritten}.
-
-
-check_db_alive(Type, #db{main_pid = Pid}) ->
- etap:is(is_process_alive(Pid), true,
- "Local " ++ Type ++ " database main pid is alive").
-
-
-compact_db(Type, #db{name = Name}) ->
- {ok, Db} = couch_db:open_int(Name, []),
- {ok, CompactPid} = couch_db:start_compact(Db),
- MonRef = erlang:monitor(process, CompactPid),
- receive
- {'DOWN', MonRef, process, CompactPid, normal} ->
- ok;
- {'DOWN', MonRef, process, CompactPid, Reason} ->
- etap:bail("Error compacting " ++ Type ++ " database " ++ ?b2l(Name) ++
- ": " ++ couch_util:to_list(Reason))
- after 30000 ->
- etap:bail("Compaction for " ++ Type ++ " database " ++ ?b2l(Name) ++
- " didn't finish")
- end,
- ok = couch_db:close(Db).
-
-
-check_fd(Type, #db{name = Name, fd = Fd, fd_monitor=OldMonRef}) ->
- {_, MonRef} = spawn_monitor(fun() ->
- MC = fun(F) ->
- % Speed up the close after the switch
- Fd ! maybe_close,
- {ok, Db} = couch_db:open_int(Name, []),
- couch_db:close(Db),
- case Db#db.fd of
- Fd ->
- receive _ -> ok after 1000 -> ok end,
- F(F);
- _ ->
- ok
- end
- end,
- MC(MC)
- end),
- receive {'DOWN', MonRef, process, _, _} ->
- etap:diag("Old " ++ Type ++ " database fd terminated")
- after 30000 ->
- etap:bail("Old " ++ Type ++ " database fd didn't terminate")
- end,
- {ok, #db{fd_monitor = NewMonRef} = Db} = couch_db:open_int(Name, []),
- ok = couch_db:close(Db),
- etap:isnt(
- NewMonRef, OldMonRef, Type ++ " database has new fd monitor").
-
-
-reopen_db(#db{name = Name}) ->
- {ok, Db} = couch_db:open_int(Name, []),
- ok = couch_db:close(Db),
- {ok, Db}.
-
-
-wait_target_in_sync(DocCount, #db{name = TargetName}) ->
- wait_target_in_sync_loop(DocCount, TargetName, 300).
-
-
-wait_target_in_sync_loop(_DocCount, _TargetName, 0) ->
- etap:bail("Could not get source and target databases in sync");
-wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft) ->
- {ok, Target} = couch_db:open_int(TargetName, []),
- {ok, TargetInfo} = couch_db:get_db_info(Target),
- ok = couch_db:close(Target),
- TargetDocCount = couch_util:get_value(doc_count, TargetInfo),
- case TargetDocCount == DocCount of
- true ->
- etap:diag("Source and target databases are in sync");
- false ->
- ok = timer:sleep(100),
- wait_target_in_sync_loop(DocCount, TargetName, RetriesLeft - 1)
- end.
-
-
-compare_dbs(#db{name = SourceName}, #db{name = TargetName}) ->
- {ok, SourceDb} = couch_db:open_int(SourceName, []),
- {ok, TargetDb} = couch_db:open_int(TargetName, []),
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget = case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- etap:bail("Error opening document '" ++ ?b2l(DocId) ++
- "' from target: " ++ couch_util:to_list(Error))
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- case DocTargetJson of
- DocJson ->
- ok;
- _ ->
- etap:bail("Content from document '" ++ ?b2l(DocId) ++
- "' differs in target database")
- end,
- {ok, Acc}
- end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- etap:diag("Target database has the same documents as the source database"),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-
-check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
- timer:sleep(1000),
- Source = case Src of
- {remote, NameSrc} ->
- <<(db_url(NameSrc))/binary, $/>>;
- _ ->
- Src
- end,
- Target = case Tgt of
- {remote, NameTgt} ->
- <<(db_url(NameTgt))/binary, $/>>;
- _ ->
- Tgt
- end,
- FullRepId = list_to_binary(BaseId ++ Ext),
- Pid = list_to_binary(pid_to_list(RepPid)),
- [RepTask] = couch_task_status:all(),
- etap:is(couch_util:get_value(pid, RepTask), Pid,
- "_active_tasks entry has correct pid property"),
- etap:is(couch_util:get_value(replication_id, RepTask), FullRepId,
- "_active_tasks entry has right replication id"),
- etap:is(couch_util:get_value(continuous, RepTask), true,
- "_active_tasks entry has continuous property set to true"),
- etap:is(couch_util:get_value(source, RepTask), Source,
- "_active_tasks entry has correct source property"),
- etap:is(couch_util:get_value(target, RepTask), Target,
- "_active_tasks entry has correct target property"),
- etap:is(is_integer(couch_util:get_value(docs_read, RepTask)), true,
- "_active_tasks entry has integer docs_read property"),
- etap:is(is_integer(couch_util:get_value(docs_written, RepTask)), true,
- "_active_tasks entry has integer docs_written property"),
- etap:is(is_integer(couch_util:get_value(doc_write_failures, RepTask)), true,
- "_active_tasks entry has integer doc_write_failures property"),
- etap:is(is_integer(couch_util:get_value(revisions_checked, RepTask)), true,
- "_active_tasks entry has integer revisions_checked property"),
- etap:is(is_integer(couch_util:get_value(missing_revisions_found, RepTask)), true,
- "_active_tasks entry has integer missing_revisions_found property"),
- etap:is(is_integer(couch_util:get_value(checkpointed_source_seq, RepTask)), true,
- "_active_tasks entry has integer checkpointed_source_seq property"),
- etap:is(is_integer(couch_util:get_value(source_seq, RepTask)), true,
- "_active_tasks entry has integer source_seq property"),
- Progress = couch_util:get_value(progress, RepTask),
- etap:is(is_integer(Progress), true,
- "_active_tasks entry has an integer progress property"),
- etap:is(Progress =< 100, true, "Progress is not greater than 100%").
-
-
-wait_writer(Pid, NumDocs) ->
- case get_writer_num_docs_written(Pid) of
- N when N >= NumDocs ->
- ok;
- _ ->
- wait_writer(Pid, NumDocs)
- end.
-
-
-spawn_writer(Db) ->
- Parent = self(),
- Pid = spawn(fun() -> writer_loop(Db, Parent, 0) end),
- etap:diag("Started source database writer"),
- Pid.
-
-
-pause_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {pause, Ref},
- receive
- {paused, Ref} ->
- ok
- after 30000 ->
- etap:bail("Failed to pause source database writer")
- end.
-
-
-resume_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {continue, Ref},
- receive
- {ok, Ref} ->
- ok
- after 30000 ->
- etap:bail("Failed to unpause source database writer")
- end.
-
-
-get_writer_num_docs_written(Pid) ->
- Ref = make_ref(),
- Pid ! {get_count, Ref},
- receive
- {count, Ref, Count} ->
- Count
- after 30000 ->
- etap:bail("Timeout getting number of documents written from "
- "source database writer")
- end.
-
-
-stop_writer(Pid) ->
- Ref = make_ref(),
- Pid ! {stop, Ref},
- receive
- {stopped, Ref, DocsWritten} ->
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _Reason} ->
- etap:diag("Stopped source database writer"),
- DocsWritten
- after 30000 ->
- etap:bail("Timeout stopping source database writer")
- end
- after 30000 ->
- etap:bail("Timeout stopping source database writer")
- end.
-
-
-writer_loop(#db{name = DbName}, Parent, Counter) ->
- maybe_pause(Parent, Counter),
- Doc = couch_doc:from_json_obj({[
- {<<"_id">>, list_to_binary(integer_to_list(Counter + 1))},
- {<<"value">>, Counter + 1},
- {<<"_attachments">>, {[
- {<<"icon1.png">>, {[
- {<<"data">>, base64:encode(att_data())},
- {<<"content_type">>, <<"image/png">>}
- ]}},
- {<<"icon2.png">>, {[
- {<<"data">>, base64:encode(iolist_to_binary(
- [att_data(), att_data()]))},
- {<<"content_type">>, <<"image/png">>}
- ]}}
- ]}}
- ]}),
- maybe_pause(Parent, Counter),
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _} = couch_db:update_doc(Db, Doc, []),
- ok = couch_db:close(Db),
- receive
- {get_count, Ref} ->
- Parent ! {count, Ref, Counter + 1},
- writer_loop(Db, Parent, Counter + 1);
- {stop, Ref} ->
- Parent ! {stopped, Ref, Counter + 1}
- after 0 ->
- ok = timer:sleep(500),
- writer_loop(Db, Parent, Counter + 1)
- end.
-
-
-maybe_pause(Parent, Counter) ->
- receive
- {get_count, Ref} ->
- Parent ! {count, Ref, Counter};
- {pause, Ref} ->
- Parent ! {paused, Ref},
- receive {continue, Ref2} -> Parent ! {ok, Ref2} end
- after 0 ->
- ok
- end.
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- {ok, Db} = couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]),
- couch_db:close(Db),
- {ok, Db}.
-
-
-delete_db(#db{name = DbName, main_pid = Pid}) ->
- ok = couch_server:delete(
- DbName, [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _Reason} ->
- ok
- after 30000 ->
- etap:bail("Timeout deleting database")
- end.
-
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target},
- {<<"continuous">>, true}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- {ok, Pid, Rep#rep.id}.
-
-
-cancel_replication(RepId, RepPid) ->
- {ok, _} = couch_replicator:cancel_replication(RepId),
- etap:is(is_process_alive(RepPid), false,
- "Replication process is no longer alive after cancel").
-
-
-att_data() ->
- {ok, Data} = file:read_file(
- test_util:source_file("share/www/image/logo.png")),
- Data.
[10/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Disable problematic tests
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/3cf0b131
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/3cf0b131
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/3cf0b131
Branch: refs/heads/master
Commit: 3cf0b131b5ca3269cdd90a4e6760cf84a71d0147
Parents: 6d34c4e
Author: Russell Branca <ch...@apache.org>
Authored: Wed Aug 27 16:42:09 2014 -0700
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:41 2014 -0700
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 4 ++++
test/couch_replicator_missing_stubs_tests.erl | 5 +++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3cf0b131/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index f9529d6..8378567 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -23,6 +23,8 @@
-define(TIMEOUT_WRITER, 3000).
-define(TIMEOUT_EUNIT, ?TIMEOUT div 1000 + 5).
+-ifdef(run_broken_tests).
+
setup() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
@@ -435,3 +437,5 @@ maybe_pause(Parent, Counter) ->
after 0 ->
ok
end.
+
+-endif.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3cf0b131/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index db95451..0bf2f6a 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -20,6 +20,8 @@
-define(TIMEOUT_EUNIT, 30).
+-ifdef(run_broken_tests).
+
setup() ->
DbName = ?tempdb(),
{ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
@@ -247,3 +249,6 @@ replicate(Source, Target) ->
{'DOWN', MonRef, process, Pid, _} ->
ok
end.
+
+-endif.
+
[08/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/04-replication-large_atts.t etap test to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b010033c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b010033c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b010033c
Branch: refs/heads/master
Commit: b010033c0fb232863e54242088b22b4ceda54c0c
Parents: 191af63
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 16 13:45:04 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_large_atts_tests.erl | 218 ++++++++++++++++
test/04-replication-large-atts.t | 256 -------------------
2 files changed, 218 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b010033c/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_large_atts_tests.erl b/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
new file mode 100644
index 0000000..ed7ec50
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_large_atts_tests.erl
@@ -0,0 +1,218 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_large_atts_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(ATT_SIZE_1, 2 * 1024 * 1024).
+-define(ATT_SIZE_2, round(6.6 * 1024 * 1024)).
+-define(DOCS_COUNT, 11).
+-define(TIMEOUT_EUNIT, 30).
+-define(TIMEOUT_STOP, 1000).
+
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ couch_config:set("attachments", "compressible_types", "text/*", false),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+large_atts_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate docs with large attachments",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target)]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, ?DOCS_COUNT))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ Docs = lists:foldl(
+ fun(DocIdCounter, Acc) ->
+ Doc = #doc{
+ id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ body = {[]},
+ atts = [
+ att(<<"att1">>, ?ATT_SIZE_1, <<"text/plain">>),
+ att(<<"att2">>, ?ATT_SIZE_2, <<"app/binary">>)
+ ]
+ },
+ [Doc | Acc]
+ end,
+ [], lists:seq(1, DocCount)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []),
+ couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
+ Id = DocSource#doc.id,
+
+ {ok, DocTarget} = couch_db:open_doc(TargetDb, Id),
+ ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+ #doc{atts = SourceAtts} = DocSource,
+ #doc{atts = TargetAtts} = DocTarget,
+ ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+ lists:sort([N || #att{name = N} <- TargetAtts])),
+
+ FunCompareAtts = fun(#att{name = AttName} = Att) ->
+ {ok, AttTarget} = find_att(TargetAtts, AttName),
+ SourceMd5 = att_md5(Att),
+ TargetMd5 = att_md5(AttTarget),
+ case AttName of
+ <<"att1">> ->
+ ?assertEqual(gzip, Att#att.encoding),
+ ?assertEqual(gzip, AttTarget#att.encoding),
+ DecSourceMd5 = att_decoded_md5(Att),
+ DecTargetMd5 = att_decoded_md5(AttTarget),
+ ?assertEqual(DecSourceMd5, DecTargetMd5);
+ _ ->
+ ?assertEqual(identity, Att#att.encoding),
+ ?assertEqual(identity, AttTarget#att.encoding)
+ end,
+ ?assertEqual(SourceMd5, TargetMd5),
+ ?assert(is_integer(Att#att.disk_len)),
+ ?assert(is_integer(Att#att.att_len)),
+ ?assert(is_integer(AttTarget#att.disk_len)),
+ ?assert(is_integer(AttTarget#att.att_len)),
+ ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+ ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+ ?assertEqual(Att#att.type, AttTarget#att.type),
+ ?assertEqual(Att#att.md5, AttTarget#att.md5)
+ end,
+
+ lists:foreach(FunCompareAtts, SourceAtts),
+
+ {ok, Acc}
+ end,
+
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+att(Name, Size, Type) ->
+ #att{
+ name = Name,
+ type = Type,
+ att_len = Size,
+ data = fun(Count) -> crypto:rand_bytes(Count) end
+ }.
+
+find_att([], _Name) ->
+ nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+ {ok, Att};
+find_att([_ | Rest], Name) ->
+ find_att(Rest, Name).
+
+att_md5(Att) ->
+ Md50 = couch_doc:att_foldl(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+ Md50 = couch_doc:att_foldl_decode(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[{<<"source">>, Source}, {<<"target">>, Target}]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b010033c/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
deleted file mode 100755
index adc3101..0000000
--- a/test/04-replication-large-atts.t
+++ /dev/null
@@ -1,256 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Test replication of large attachments. Verify that both source and
-% target have the same attachment data and metadata.
-
--define(b2l(Bin), binary_to_list(Bin)).
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(doc, {
- id = <<"">>,
- revs = {0, []},
- body = {[]},
- atts = [],
- deleted = false,
- meta = []
-}).
-
--record(att, {
- name,
- type,
- att_len,
- disk_len,
- md5= <<>>,
- revpos=0,
- data,
- encoding=identity
-}).
-
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-
-main(_) ->
- test_util:run(1192, fun() -> test() end).
-
-
-test() ->
- test_util:start_couch(),
- application:start(ibrowse),
- application:start(crypto),
- config:set("attachments", "compressible_types", "text/*", false),
-
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- populate_db(SourceDb, 11),
- ok = couch_db:close(SourceDb),
-
- lists:foreach(
- fun({Source, Target}) ->
- etap:diag("Creating target database"),
- {ok, TargetDb} = create_db(target_db_name()),
-
- ok = couch_db:close(TargetDb),
- etap:diag("Triggering replication"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- compare_dbs(SourceDb, TargetDb),
-
- etap:diag("Deleting target database"),
- delete_db(TargetDb),
- ok = timer:sleep(1000)
- end,
- Pairs),
-
- delete_db(SourceDb),
- test_util:stop_couch().
-
-
-populate_db(Db, DocCount) ->
- Docs = lists:foldl(
- fun(DocIdCounter, Acc) ->
- Doc = #doc{
- id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
- body = {[]},
- atts = [
- att(<<"att1">>, 2 * 1024 * 1024, <<"text/plain">>),
- att(<<"att2">>, round(6.6 * 1024 * 1024), <<"app/binary">>)
- ]
- },
- [Doc | Acc]
- end,
- [], lists:seq(1, DocCount)),
- {ok, _} = couch_db:update_docs(Db, Docs, []).
-
-
-att(Name, Size, Type) ->
- #att{
- name = Name,
- type = Type,
- att_len = Size,
- data = fun(Count) -> crypto:rand_bytes(Count) end
- }.
-
-
-compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(couch_db:name(Source), []),
- {ok, TargetDb} = couch_db:open_int(couch_db:name(Target), []),
-
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
- Id = DocSource#doc.id,
-
- etap:diag("Verifying document " ++ ?b2l(Id)),
-
- {ok, DocTarget} = couch_db:open_doc(TargetDb, Id),
- etap:is(DocTarget#doc.body, DocSource#doc.body,
- "Same body in source and target databases"),
-
- #doc{atts = SourceAtts} = DocSource,
- #doc{atts = TargetAtts} = DocTarget,
- etap:is(
- lists:sort([N || #att{name = N} <- SourceAtts]),
- lists:sort([N || #att{name = N} <- TargetAtts]),
- "Document has same number (and names) of attachments in "
- "source and target databases"),
-
- lists:foreach(
- fun(#att{name = AttName} = Att) ->
- etap:diag("Verifying attachment " ++ ?b2l(AttName)),
-
- {ok, AttTarget} = find_att(TargetAtts, AttName),
- SourceMd5 = att_md5(Att),
- TargetMd5 = att_md5(AttTarget),
- case AttName of
- <<"att1">> ->
- etap:is(Att#att.encoding, gzip,
- "Attachment is gzip encoded in source database"),
- etap:is(AttTarget#att.encoding, gzip,
- "Attachment is gzip encoded in target database"),
- DecSourceMd5 = att_decoded_md5(Att),
- DecTargetMd5 = att_decoded_md5(AttTarget),
- etap:is(DecTargetMd5, DecSourceMd5,
- "Same identity content in source and target databases");
- _ ->
- etap:is(Att#att.encoding, identity,
- "Attachment is not encoded in source database"),
- etap:is(AttTarget#att.encoding, identity,
- "Attachment is not encoded in target database")
- end,
- etap:is(TargetMd5, SourceMd5,
- "Same content in source and target databases"),
- etap:is(is_integer(Att#att.disk_len), true,
- "#att.disk_len is an integer in source database"),
- etap:is(is_integer(Att#att.att_len), true,
- "#att.att_len is an integer in source database"),
- etap:is(is_integer(AttTarget#att.disk_len), true,
- "#att.disk_len is an integer in target database"),
- etap:is(is_integer(AttTarget#att.att_len), true,
- "#att.att_len is an integer in target database"),
- etap:is(Att#att.disk_len, AttTarget#att.disk_len,
- "Same identity length in source and target databases"),
- etap:is(Att#att.att_len, AttTarget#att.att_len,
- "Same encoded length in source and target databases"),
- etap:is(Att#att.type, AttTarget#att.type,
- "Same type in source and target databases"),
- etap:is(Att#att.md5, SourceMd5, "Correct MD5 in source database"),
- etap:is(AttTarget#att.md5, SourceMd5, "Correct MD5 in target database")
- end,
- SourceAtts),
-
- {ok, Acc}
- end,
-
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-
-find_att([], _Name) ->
- nil;
-find_att([#att{name = Name} = Att | _], Name) ->
- {ok, Att};
-find_att([_ | Rest], Name) ->
- find_att(Rest, Name).
-
-
-att_md5(Att) ->
- Md50 = couch_doc:att_foldl(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-att_decoded_md5(Att) ->
- Md50 = couch_doc:att_foldl_decode(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]).
-
-
-delete_db(Db) ->
- ok = couch_server:delete(
- couch_db:name(Db), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).
-
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, Reason} ->
- etap:is(Reason, normal, "Replication finished successfully")
- after 300000 ->
- etap:bail("Timeout waiting for replication to finish")
- end.
[04/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/02-httpc-pool.t etap test suite to eunit
Test test_worker_dead_pool_full removed as it's redundant.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/450edc06
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/450edc06
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/450edc06
Branch: refs/heads/master
Commit: 450edc066aede7583c5f4864ecd6dd8cb71d0a6c
Parents: 86aba61
Author: Alexander Shorin <kx...@apache.org>
Authored: Thu Jun 12 11:03:38 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_httpc_pool_tests.erl | 189 +++++++++++++++
test/02-httpc-pool.t | 240 -------------------
2 files changed, 189 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/450edc06/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl b/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
new file mode 100644
index 0000000..6bede4c
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_httpc_pool_tests.erl
@@ -0,0 +1,189 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_httpc_pool_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_USER, {user_ctx, #user_ctx{roles=[<<"_admin">>]}}).
+-define(TIMEOUT, 1000).
+
+
+start() ->
+ {ok, Pid} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Pid.
+
+stop(Pid) ->
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT ->
+ throw({timeout, server_stop})
+ end.
+
+setup() ->
+ spawn_pool().
+
+teardown(Pool) ->
+ stop_pool(Pool).
+
+
+httpc_pool_test_() ->
+ {
+ "httpc pool tests",
+ {
+ setup,
+ fun start/0, fun stop/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun should_block_new_clients_when_full/1,
+ fun should_replace_worker_on_death/1
+ ]
+ }
+ }
+ }.
+
+
+should_block_new_clients_when_full(Pool) ->
+ ?_test(begin
+ Client1 = spawn_client(Pool),
+ Client2 = spawn_client(Pool),
+ Client3 = spawn_client(Pool),
+
+ ?assertEqual(ok, ping_client(Client1)),
+ ?assertEqual(ok, ping_client(Client2)),
+ ?assertEqual(ok, ping_client(Client3)),
+
+ Worker1 = get_client_worker(Client1, "1"),
+ Worker2 = get_client_worker(Client2, "2"),
+ Worker3 = get_client_worker(Client3, "3"),
+
+ ?assert(is_process_alive(Worker1)),
+ ?assert(is_process_alive(Worker2)),
+ ?assert(is_process_alive(Worker3)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertNotEqual(Worker2, Worker3),
+ ?assertNotEqual(Worker3, Worker1),
+
+ Client4 = spawn_client(Pool),
+ ?assertEqual(timeout, ping_client(Client4)),
+
+ ?assertEqual(ok, stop_client(Client1)),
+ ?assertEqual(ok, ping_client(Client4)),
+
+ Worker4 = get_client_worker(Client4, "4"),
+ ?assertEqual(Worker1, Worker4),
+
+ lists:foreach(
+ fun(C) ->
+ ?assertEqual(ok, stop_client(C))
+ end, [Client2, Client3, Client4])
+ end).
+
+should_replace_worker_on_death(Pool) ->
+ ?_test(begin
+ Client1 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client1)),
+ Worker1 = get_client_worker(Client1, "1"),
+ ?assert(is_process_alive(Worker1)),
+
+ ?assertEqual(ok, kill_client_worker(Client1)),
+ ?assertNot(is_process_alive(Worker1)),
+ ?assertEqual(ok, stop_client(Client1)),
+
+ Client2 = spawn_client(Pool),
+ ?assertEqual(ok, ping_client(Client2)),
+ Worker2 = get_client_worker(Client2, "2"),
+ ?assert(is_process_alive(Worker2)),
+
+ ?assertNotEqual(Worker1, Worker2),
+ ?assertEqual(ok, stop_client(Client2))
+ end).
+
+
+spawn_client(Pool) ->
+ Parent = self(),
+ Ref = make_ref(),
+ Pid = spawn(fun() ->
+ {ok, Worker} = couch_replicator_httpc_pool:get_worker(Pool),
+ loop(Parent, Ref, Worker, Pool)
+ end),
+ {Pid, Ref}.
+
+ping_client({Pid, Ref}) ->
+ Pid ! ping,
+ receive
+ {pong, Ref} ->
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+get_client_worker({Pid, Ref}, ClientName) ->
+ Pid ! get_worker,
+ receive
+ {worker, Ref, Worker} ->
+ Worker
+ after ?TIMEOUT ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, "Timeout getting client " ++ ClientName ++ " worker"}]})
+ end.
+
+stop_client({Pid, Ref}) ->
+ Pid ! stop,
+ receive
+ {stop, Ref} ->
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+kill_client_worker({Pid, Ref}) ->
+ Pid ! get_worker,
+ receive
+ {worker, Ref, Worker} ->
+ exit(Worker, kill),
+ ok
+ after ?TIMEOUT ->
+ timeout
+ end.
+
+loop(Parent, Ref, Worker, Pool) ->
+ receive
+ ping ->
+ Parent ! {pong, Ref},
+ loop(Parent, Ref, Worker, Pool);
+ get_worker ->
+ Parent ! {worker, Ref, Worker},
+ loop(Parent, Ref, Worker, Pool);
+ stop ->
+ couch_replicator_httpc_pool:release_worker(Pool, Worker),
+ Parent ! {stop, Ref}
+ end.
+
+spawn_pool() ->
+ Host = couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = couch_config:get("httpd", "port", "5984"),
+ {ok, Pool} = couch_replicator_httpc_pool:start_link(
+ "http://" ++ Host ++ ":" ++ Port, [{max_connections, 3}]),
+ Pool.
+
+stop_pool(Pool) ->
+ ok = couch_replicator_httpc_pool:stop(Pool).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/450edc06/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
deleted file mode 100755
index fc86cce..0000000
--- a/test/02-httpc-pool.t
+++ /dev/null
@@ -1,240 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-main(_) ->
- test_util:run(55, fun() -> test() end).
-
-
-test() ->
- test_util:start_couch(),
- ibrowse:start(),
-
- test_pool_full(),
- test_worker_dead_pool_non_full(),
- test_worker_dead_pool_full(),
-
- couch_server_sup:stop(),
- ok.
-
-
-test_pool_full() ->
- Pool = spawn_pool(),
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
-
- etap:diag("Check that we can spawn the max number of connections."),
- etap:is(ping_client(Client1), ok, "Client 1 started ok."),
- etap:is(ping_client(Client2), ok, "Client 2 started ok."),
- etap:is(ping_client(Client3), ok, "Client 3 started ok."),
-
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
- etap:is(is_process_alive(Worker1), true, "Client's 1 worker is alive."),
- etap:is(is_process_alive(Worker2), true, "Client's 2 worker is alive."),
- etap:is(is_process_alive(Worker3), true, "Client's 3 worker is alive."),
-
- etap:isnt(Worker1, Worker2, "Clients 1 and 2 got different workers."),
- etap:isnt(Worker2, Worker3, "Clients 2 and 3 got different workers."),
- etap:isnt(Worker1, Worker3, "Clients 1 and 3 got different workers."),
-
- etap:diag("Check that client 4 blocks waiting for a worker."),
- Client4 = spawn_client(Pool),
- etap:is(ping_client(Client4), timeout, "Client 4 blocked while waiting."),
-
- etap:diag("Check that stopping a client gives up its worker."),
- etap:is(stop_client(Client1), ok, "First client stopped."),
-
- etap:diag("And check that our blocked client has been unblocked."),
- etap:is(ping_client(Client4), ok, "Client 4 was unblocked."),
-
- Worker4 = get_client_worker(Client4, "4"),
- etap:is(is_process_alive(Worker4), true, "Client's 4 worker is alive."),
- etap:is(Worker4, Worker1, "Client 4 got worker that client 1 got before."),
-
- lists:foreach(fun(C) -> ok = stop_client(C) end, [Client2, Client3, Client4]),
- stop_pool(Pool).
-
-
-test_worker_dead_pool_non_full() ->
- Pool = spawn_pool(),
- Client1 = spawn_client(Pool),
-
- etap:is(ping_client(Client1), ok, "Client 1 started ok."),
- Worker1 = get_client_worker(Client1, "1"),
- etap:is(is_process_alive(Worker1), true, "Client's 1 worker is alive."),
-
- etap:diag("Kill client's 1 worker."),
- etap:is(kill_client_worker(Client1), ok, "Killed client's 1 worker."),
- etap:is(is_process_alive(Worker1), false, "Client's 1 worker process is dead."),
-
- etap:is(stop_client(Client1), ok, "First client stopped and released its worker."),
-
- Client2 = spawn_client(Pool),
- etap:is(ping_client(Client2), ok, "Client 2 started ok."),
- Worker2 = get_client_worker(Client2, "2"),
- etap:isnt(Worker2, Worker1, "Client 2 got a different worker from client 1"),
- etap:is(is_process_alive(Worker2), true, "Client's 2 worker is alive."),
-
- etap:is(stop_client(Client2), ok, "Second client stopped."),
- stop_pool(Pool).
-
-
-test_worker_dead_pool_full() ->
- Pool = spawn_pool(),
- Client1 = spawn_client(Pool),
- Client2 = spawn_client(Pool),
- Client3 = spawn_client(Pool),
-
- etap:diag("Check that we can spawn the max number of connections."),
- etap:is(ping_client(Client1), ok, "Client 1 started ok."),
- etap:is(ping_client(Client2), ok, "Client 2 started ok."),
- etap:is(ping_client(Client3), ok, "Client 3 started ok."),
-
- Worker1 = get_client_worker(Client1, "1"),
- Worker2 = get_client_worker(Client2, "2"),
- Worker3 = get_client_worker(Client3, "3"),
- etap:is(is_process_alive(Worker1), true, "Client's 1 worker is alive."),
- etap:is(is_process_alive(Worker2), true, "Client's 2 worker is alive."),
- etap:is(is_process_alive(Worker3), true, "Client's 3 worker is alive."),
-
- etap:isnt(Worker1, Worker2, "Clients 1 and 2 got different workers."),
- etap:isnt(Worker2, Worker3, "Clients 2 and 3 got different workers."),
- etap:isnt(Worker1, Worker3, "Clients 1 and 3 got different workers."),
-
- etap:diag("Check that client 4 blocks waiting for a worker."),
- Client4 = spawn_client(Pool),
- etap:is(ping_client(Client4), timeout, "Client 4 blocked while waiting."),
-
- etap:diag("Kill client's 1 worker."),
- etap:is(kill_client_worker(Client1), ok, "Killed client's 1 worker."),
- etap:is(is_process_alive(Worker1), false, "Client's 1 worker process is dead."),
-
- etap:diag("Check client 4 got unblocked after first worker's death"),
- etap:is(ping_client(Client4), ok, "Client 4 not blocked anymore."),
-
- Worker4 = get_client_worker(Client4, "4"),
- etap:is(is_process_alive(Worker4), true, "Client's 4 worker is alive."),
- etap:isnt(Worker4, Worker1, "Client 4 got a worker different from client 1."),
- etap:isnt(Worker4, Worker2, "Client 4 got a worker different from client 2."),
- etap:isnt(Worker4, Worker3, "Client 4 got a worker different from client 3."),
-
- etap:diag("Check that stopping client 1 is a noop."),
- etap:is(stop_client(Client1), ok, "First client stopped."),
-
- etap:is(is_process_alive(Worker2), true, "Client's 2 worker still alive."),
- etap:is(is_process_alive(Worker3), true, "Client's 3 worker still alive."),
- etap:is(is_process_alive(Worker4), true, "Client's 4 worker still alive."),
-
- etap:diag("Check that client 5 blocks waiting for a worker."),
- Client5 = spawn_client(Pool),
- etap:is(ping_client(Client5), timeout, "Client 5 blocked while waiting."),
-
- etap:diag("Check that stopping client 2 gives up its worker."),
- etap:is(stop_client(Client2), ok, "Second client stopped."),
-
- etap:diag("Now check that client 5 has been unblocked."),
- etap:is(ping_client(Client5), ok, "Client 5 was unblocked."),
-
- Worker5 = get_client_worker(Client5, "5"),
- etap:is(is_process_alive(Worker5), true, "Client's 5 worker is alive."),
- etap:isnt(Worker5, Worker1, "Client 5 got a worker different from client 1."),
- etap:is(Worker5, Worker2, "Client 5 got same worker as client 2."),
- etap:isnt(Worker5, Worker3, "Client 5 got a worker different from client 3."),
- etap:isnt(Worker5, Worker4, "Client 5 got a worker different from client 4."),
-
- etap:is(is_process_alive(Worker3), true, "Client's 3 worker still alive."),
- etap:is(is_process_alive(Worker4), true, "Client's 4 worker still alive."),
- etap:is(is_process_alive(Worker5), true, "Client's 5 worker still alive."),
-
- lists:foreach(fun(C) -> ok = stop_client(C) end, [Client3, Client4, Client5]),
- stop_pool(Pool).
-
-
-spawn_client(Pool) ->
- Parent = self(),
- Ref = make_ref(),
- Pid = spawn(fun() ->
- {ok, Worker} = couch_replicator_httpc_pool:get_worker(Pool),
- loop(Parent, Ref, Worker, Pool)
- end),
- {Pid, Ref}.
-
-
-ping_client({Pid, Ref}) ->
- Pid ! ping,
- receive
- {pong, Ref} ->
- ok
- after 3000 ->
- timeout
- end.
-
-
-get_client_worker({Pid, Ref}, ClientName) ->
- Pid ! get_worker,
- receive
- {worker, Ref, Worker} ->
- Worker
- after 3000 ->
- etap:bail("Timeout getting client " ++ ClientName ++ " worker.")
- end.
-
-
-stop_client({Pid, Ref}) ->
- Pid ! stop,
- receive
- {stop, Ref} ->
- ok
- after 3000 ->
- timeout
- end.
-
-
-kill_client_worker({Pid, Ref}) ->
- Pid ! get_worker,
- receive
- {worker, Ref, Worker} ->
- exit(Worker, kill),
- ok
- after 3000 ->
- timeout
- end.
-
-
-loop(Parent, Ref, Worker, Pool) ->
- receive
- ping ->
- Parent ! {pong, Ref},
- loop(Parent, Ref, Worker, Pool);
- get_worker ->
- Parent ! {worker, Ref, Worker},
- loop(Parent, Ref, Worker, Pool);
- stop ->
- couch_replicator_httpc_pool:release_worker(Pool, Worker),
- Parent ! {stop, Ref}
- end.
-
-
-spawn_pool() ->
- Host = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = config:get("httpd", "port", "5984"),
- {ok, Pool} = couch_replicator_httpc_pool:start_link(
- "http://" ++ Host ++ ":5984", [{max_connections, 3}]),
- Pool.
-
-
-stop_pool(Pool) ->
- ok = couch_replicator_httpc_pool:stop(Pool).
[03/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/07-use-checkpoints.t etap test to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ba8c3977
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ba8c3977
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ba8c3977
Branch: refs/heads/master
Commit: ba8c39775793af3f1bdf143a121ac899923d4753
Parents: b6033ab
Author: Alexander Shorin <kx...@apache.org>
Authored: Tue Jun 17 01:27:08 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../couch_replicator_use_checkpoints_tests.erl | 200 +++++++++++++++
test/07-use-checkpoints.t | 256 -------------------
2 files changed, 200 insertions(+), 256 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ba8c3977/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl b/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
new file mode 100644
index 0000000..f09a235
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_use_checkpoints_tests.erl
@@ -0,0 +1,200 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_use_checkpoints_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_COUNT, 100).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+
+start(false) ->
+ fun
+ ({finished, _, {CheckpointHistory}}) ->
+ ?assertEqual([{<<"use_checkpoints">>,false}], CheckpointHistory);
+ (_) ->
+ ok
+ end;
+start(true) ->
+ fun
+ ({finished, _, {CheckpointHistory}}) ->
+ ?assertNotEqual(false, lists:keyfind(<<"session_id">>,
+ 1, CheckpointHistory));
+ (_) ->
+ ok
+ end.
+
+stop(_, _) ->
+ ok.
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({_, Fun, {A, B}}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ {ok, Listener} = couch_replicator_notifier:start_link(Fun),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target, Listener}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target, Listener}) ->
+ teardown(Source),
+ teardown(Target),
+
+ couch_replicator_notifier:stop(Listener),
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+use_checkpoints_test_() ->
+ {
+ "Replication use_checkpoints feature tests",
+ {
+ foreachx,
+ fun start/1, fun stop/2,
+ [{UseCheckpoints, fun use_checkpoints_tests/2}
+ || UseCheckpoints <- [false, true]]
+ }
+ }.
+
+use_checkpoints_tests(UseCheckpoints, Fun) ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "use_checkpoints: " ++ atom_to_list(UseCheckpoints),
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{{UseCheckpoints, Fun, Pair}, fun should_test_checkpoints/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+should_test_checkpoints({UseCheckpoints, _, {From, To}}, {Source, Target, _}) ->
+ should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}).
+should_test_checkpoints(UseCheckpoints, {From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source, ?DOCS_COUNT),
+ should_replicate(Source, Target, UseCheckpoints),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}, DocCount) ->
+ should_populate_source(Source, DocCount);
+should_populate_source(Source, DocCount) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source, DocCount))}.
+
+should_replicate({remote, Source}, Target, UseCheckpoints) ->
+ should_replicate(db_url(Source), Target, UseCheckpoints);
+should_replicate(Source, {remote, Target}, UseCheckpoints) ->
+ should_replicate(Source, db_url(Target), UseCheckpoints);
+should_replicate(Source, Target, UseCheckpoints) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target, UseCheckpoints))}.
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+
+populate_db(DbName, DocCount) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ Docs = lists:foldl(
+ fun(DocIdCounter, Acc) ->
+ Id = ?io2b(["doc", ?i2l(DocIdCounter)]),
+ Value = ?io2b(["val", ?i2l(DocIdCounter)]),
+ Doc = #doc{
+ id = Id,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ [Doc | Acc]
+ end,
+ [], lists:seq(1, DocCount)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []),
+ ok = couch_db:close(Db).
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ erlang:error(
+ {assertion_failed,
+ [{module, ?MODULE}, {line, ?LINE},
+ {reason, lists:concat(["Error opening document '",
+ ?b2l(DocId), "' from target: ",
+ couch_util:to_list(Error)])}]})
+ end,
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ ?assertEqual(DocJson, DocTargetJson),
+ {ok, Acc}
+ end,
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target, UseCheckpoints) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target},
+ {<<"use_checkpoints">>, UseCheckpoints}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ba8c3977/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
deleted file mode 100755
index f2ed1de..0000000
--- a/test/07-use-checkpoints.t
+++ /dev/null
@@ -1,256 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Verify that compacting databases that are being used as the source or
-% target of a replication doesn't affect the replication and that the
-% replication doesn't hold their reference counters forever.
-
--define(b2l(B), binary_to_list(B)).
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(doc, {
- id = <<"">>,
- revs = {0, []},
- body = {[]},
- atts = [],
- deleted = false,
- meta = []
-}).
-
--record(db, {
- main_pid = nil,
- update_pid = nil,
- compactor_pid = nil,
- instance_start_time, % number of microsecs since jan 1 1970 as a binary string
- fd,
- updater_fd,
- fd_ref_counter,
- header = nil,
- committed_update_seq,
- fulldocinfo_by_id_btree,
- docinfo_by_seq_btree,
- local_docs_btree,
- update_seq,
- name,
- filepath,
- validate_doc_funs = [],
- security = [],
- security_ptr = nil,
- user_ctx = #user_ctx{},
- waiting_delayed_commit = nil,
- revs_limit = 1000,
- fsync_options = [],
- options = [],
- compression,
- before_doc_update,
- after_doc_read
-}).
-
--record(rep, {
- id,
- source,
- target,
- options,
- user_ctx,
- doc_id
-}).
-
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-
-main(_) ->
- test_util:init_code_path(),
-
- etap:plan(16),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
-
-
-test() ->
- couch_server_sup:start_link(test_util:config_files()),
- ibrowse:start(),
-
- test_use_checkpoints(false),
- test_use_checkpoints(true),
-
- couch_server_sup:stop(),
- ok.
-
-
-test_use_checkpoints(UseCheckpoints) ->
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- ListenerFun = case UseCheckpoints of
- false ->
- fun({finished, _, {CheckpointHistory}}) ->
- etap:is(CheckpointHistory,
- [{<<"use_checkpoints">>,false}],
- "No checkpoints found");
- (_) ->
- ok
- end;
- true ->
- fun({finished, _, {CheckpointHistory}}) ->
- SessionId = lists:keyfind(
- <<"session_id">>, 1, CheckpointHistory),
- etap:isnt(SessionId, false, "There's a checkpoint");
- (_) ->
- ok
- end
- end,
- {ok, Listener} = couch_replicator_notifier:start_link(ListenerFun),
-
- lists:foreach(
- fun({Source, Target}) ->
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- populate_db(SourceDb, 100),
- ok = couch_db:close(SourceDb),
-
- etap:diag("Creating target database"),
- {ok, TargetDb} = create_db(target_db_name()),
- ok = couch_db:close(TargetDb),
-
- etap:diag("Setup replicator notifier listener"),
-
- etap:diag("Triggering replication"),
- replicate(Source, Target, UseCheckpoints),
-
- etap:diag("Replication finished, comparing source and target databases"),
- compare_dbs(SourceDb, TargetDb),
-
- etap:diag("Deleting databases"),
- delete_db(TargetDb),
- delete_db(SourceDb),
-
- ok = timer:sleep(1000)
- end,
- Pairs),
-
- couch_replicator_notifier:stop(Listener).
-
-
-populate_db(Db, DocCount) ->
- Docs = lists:foldl(
- fun(DocIdCounter, Acc) ->
- Id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
- Value = iolist_to_binary(["val", integer_to_list(DocIdCounter)]),
- Doc = #doc{
- id = Id,
- body = {[ {<<"value">>, Value} ]}
- },
- [Doc | Acc]
- end,
- [], lists:seq(1, DocCount)),
- {ok, _} = couch_db:update_docs(Db, Docs, []).
-
-
-compare_dbs(#db{name = SourceName}, #db{name = TargetName}) ->
- {ok, SourceDb} = couch_db:open_int(SourceName, []),
- {ok, TargetDb} = couch_db:open_int(TargetName, []),
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
- {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
- DocId = couch_util:get_value(<<"_id">>, Props),
- DocTarget = case couch_db:open_doc(TargetDb, DocId) of
- {ok, DocT} ->
- DocT;
- Error ->
- etap:bail("Error opening document '" ++ ?b2l(DocId) ++
- "' from target: " ++ couch_util:to_list(Error))
- end,
- DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
- case DocTargetJson of
- DocJson ->
- ok;
- _ ->
- etap:bail("Content from document '" ++ ?b2l(DocId) ++
- "' differs in target database")
- end,
- {ok, Acc}
- end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- etap:diag("Target database has the same documents as the source database"),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- {ok, Db} = couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]),
- couch_db:close(Db),
- {ok, Db}.
-
-
-delete_db(#db{name = DbName, main_pid = Pid}) ->
- ok = couch_server:delete(
- DbName, [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, _Reason} ->
- ok
- after 30000 ->
- etap:bail("Timeout deleting database")
- end.
-
-
-replicate({remote, Db}, Target, UseCheckpoints) ->
- replicate(db_url(Db), Target, UseCheckpoints);
-
-replicate(Source, {remote, Db}, UseCheckpoints) ->
- replicate(Source, db_url(Db), UseCheckpoints);
-
-replicate(Source, Target, UseCheckpoints) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target},
- {<<"use_checkpoints">>, UseCheckpoints}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, Reason} ->
- etap:is(Reason, normal, "Replication finished successfully")
- after 300000 ->
- etap:bail("Timeout waiting for replication to finish")
- end.
[07/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/06-doc-missing-stubs.t etap test suite to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b6033abb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b6033abb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b6033abb
Branch: refs/heads/master
Commit: b6033abbe3559eeb706f61486963cafa5fb0863a
Parents: 9a2c838
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 16 22:51:09 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../couch_replicator_missing_stubs_tests.erl | 260 ++++++++++++++++
test/06-doc-missing-stubs.t | 293 -------------------
2 files changed, 260 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b6033abb/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl b/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
new file mode 100644
index 0000000..0c2c30d
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_missing_stubs_tests.erl
@@ -0,0 +1,260 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_missing_stubs_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(REVS_LIMIT, 3).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 30).
+
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+missing_stubs_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate docs with missing stubs (COUCHDB-1365)",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_replicate_docs_with_missed_att_stubs/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_replicate_docs_with_missed_att_stubs({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source),
+ should_set_target_revs_limit(Target, ?REVS_LIMIT),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target),
+ should_update_source_docs(Source, ?REVS_LIMIT * 2),
+ should_replicate(Source, Target),
+ should_compare_databases(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_set_target_revs_limit({remote, Target}, RevsLimit) ->
+ should_set_target_revs_limit(Target, RevsLimit);
+should_set_target_revs_limit(Target, RevsLimit) ->
+ ?_test(begin
+ {ok, Db} = couch_db:open_int(Target, [?ADMIN_USER]),
+ ?assertEqual(ok, couch_db:set_revs_limit(Db, RevsLimit)),
+ ok = couch_db:close(Db)
+ end).
+
+should_compare_databases({remote, Source}, Target) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, {remote, Target}) ->
+ should_compare_databases(Source, Target);
+should_compare_databases(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(compare_dbs(Source, Target))}.
+
+should_update_source_docs({remote, Source}, Times) ->
+ should_update_source_docs(Source, Times);
+should_update_source_docs(Source, Times) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(update_db_docs(Source, Times))}.
+
+
+populate_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ AttData = crypto:rand_bytes(6000),
+ Doc = #doc{
+ id = <<"doc1">>,
+ atts = [
+ #att{
+ name = <<"doc1_att1">>,
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ }
+ ]
+ },
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ couch_db:close(Db).
+
+update_db_docs(DbName, Times) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, _, _} = couch_db:enum_docs(
+ Db,
+ fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
+ {DbName, Times},
+ []),
+ ok = couch_db:close(Db).
+
+db_fold_fun(FullDocInfo, {DbName, Times}) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
+ lists:foldl(
+ fun(_, {Pos, RevId}) ->
+ {ok, Db2} = couch_db:reopen(Db),
+ NewDocVersion = Doc#doc{
+ revs = {Pos, [RevId]},
+ body = {[{<<"value">>, base64:encode(crypto:rand_bytes(100))}]}
+ },
+ {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
+ NewRev
+ end,
+ {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
+ lists:seq(1, Times)),
+ ok = couch_db:close(Db),
+ {ok, {DbName, Times}}.
+
+compare_dbs(Source, Target) ->
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo,
+ [conflicts, deleted_conflicts]),
+ Id = DocSource#doc.id,
+
+ {ok, DocTarget} = couch_db:open_doc(TargetDb, Id,
+ [conflicts, deleted_conflicts]),
+ ?assertEqual(DocSource#doc.body, DocTarget#doc.body),
+
+ ?assertEqual(couch_doc:to_json_obj(DocSource, []),
+ couch_doc:to_json_obj(DocTarget, [])),
+
+ #doc{atts = SourceAtts} = DocSource,
+ #doc{atts = TargetAtts} = DocTarget,
+ ?assertEqual(lists:sort([N || #att{name = N} <- SourceAtts]),
+ lists:sort([N || #att{name = N} <- TargetAtts])),
+
+ lists:foreach(
+ fun(#att{name = AttName} = Att) ->
+ {ok, AttTarget} = find_att(TargetAtts, AttName),
+ SourceMd5 = att_md5(Att),
+ TargetMd5 = att_md5(AttTarget),
+ case AttName of
+ <<"att1">> ->
+ ?assertEqual(gzip, Att#att.encoding),
+ ?assertEqual(gzip, AttTarget#att.encoding),
+ DecSourceMd5 = att_decoded_md5(Att),
+ DecTargetMd5 = att_decoded_md5(AttTarget),
+ ?assertEqual(DecSourceMd5, DecTargetMd5);
+ _ ->
+ ?assertEqual(identity, Att#att.encoding),
+ ?assertEqual(identity, AttTarget#att.encoding)
+ end,
+ ?assertEqual(SourceMd5, TargetMd5),
+ ?assert(is_integer(Att#att.disk_len)),
+ ?assert(is_integer(Att#att.att_len)),
+ ?assert(is_integer(AttTarget#att.disk_len)),
+ ?assert(is_integer(AttTarget#att.att_len)),
+ ?assertEqual(Att#att.disk_len, AttTarget#att.disk_len),
+ ?assertEqual(Att#att.att_len, AttTarget#att.att_len),
+ ?assertEqual(Att#att.type, AttTarget#att.type),
+ ?assertEqual(Att#att.md5, AttTarget#att.md5)
+ end,
+ SourceAtts),
+ {ok, Acc}
+ end,
+
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+find_att([], _Name) ->
+ nil;
+find_att([#att{name = Name} = Att | _], Name) ->
+ {ok, Att};
+find_att([_ | Rest], Name) ->
+ find_att(Rest, Name).
+
+att_md5(Att) ->
+ Md50 = couch_doc:att_foldl(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+att_decoded_md5(Att) ->
+ Md50 = couch_doc:att_foldl_decode(
+ Att,
+ fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
+ couch_util:md5_init()),
+ couch_util:md5_final(Md50).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b6033abb/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
deleted file mode 100755
index 116550c..0000000
--- a/test/06-doc-missing-stubs.t
+++ /dev/null
@@ -1,293 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Test replication of documents with many leaf revisions.
-% Motivated by COUCHDB-1340 and other similar issues where a document
-% GET with a too long ?open_revs revision list doesn't work due to
-% maximum web server limits for the HTTP request path.
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(doc, {
- id = <<"">>,
- revs = {0, []},
- body = {[]},
- atts = [],
- deleted = false,
- meta = []
-}).
-
--record(att, {
- name,
- type,
- att_len,
- disk_len,
- md5= <<>>,
- revpos=0,
- data,
- encoding=identity
-}).
-
--define(b2l(B), binary_to_list(B)).
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-target_revs_limit() -> 3.
-
-
-main(_) ->
- test_util:run(128, fun() -> test() end).
-
-
-% Test motivated by COUCHDB-1365.
-test() ->
- test_util:start_couch(),
- ibrowse:start(),
-
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- lists:foreach(
- fun({Source, Target}) ->
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- populate_db(SourceDb),
- ok = couch_db:close(SourceDb),
-
- etap:diag("Creating target database"),
- {ok, TargetDb} = create_db(target_db_name()),
- ok = couch_db:set_revs_limit(TargetDb, target_revs_limit()),
- ok = couch_db:close(TargetDb),
-
- etap:diag("Triggering replication"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- compare_dbs(SourceDb, TargetDb),
-
- etap:diag("Updating source database docs"),
- update_db_docs(couch_db:name(SourceDb), target_revs_limit() + 2),
-
- etap:diag("Triggering replication again"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- compare_dbs(SourceDb, TargetDb),
-
- etap:diag("Deleting databases"),
- delete_db(TargetDb),
- delete_db(SourceDb),
- ok = timer:sleep(1000)
- end,
- Pairs),
-
- test_util:stop_couch().
-
-
-populate_db(Db) ->
- AttData = crypto:rand_bytes(6000),
- Doc1 = #doc{
- id = <<"doc1">>,
- atts = [
- #att{
- name = <<"doc1_att1">>,
- type = <<"application/foobar">>,
- att_len = byte_size(AttData),
- data = AttData
- }
- ]
- },
- {ok, _} = couch_db:update_doc(Db, Doc1, []).
-
-
-update_db_docs(DbName, Times) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, _, _} = couch_db:enum_docs(
- Db,
- fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
- {DbName, Times},
- []),
- ok = couch_db:close(Db).
-
-
-db_fold_fun(FullDocInfo, {DbName, Times}) ->
- {ok, Db} = couch_db:open_int(DbName, []),
- {ok, Doc} = couch_db:open_doc(Db, FullDocInfo),
- lists:foldl(
- fun(_, {Pos, RevId}) ->
- {ok, Db2} = couch_db:reopen(Db),
- NewDocVersion = Doc#doc{
- revs = {Pos, [RevId]},
- body = {[{<<"value">>, base64:encode(crypto:rand_bytes(100))}]}
- },
- {ok, NewRev} = couch_db:update_doc(Db2, NewDocVersion, []),
- NewRev
- end,
- {element(1, Doc#doc.revs), hd(element(2, Doc#doc.revs))},
- lists:seq(1, Times)),
- ok = couch_db:close(Db),
- {ok, {DbName, Times}}.
-
-
-compare_dbs(Source, Target) ->
- {ok, SourceDb} = couch_db:open_int(couch_db:name(Source), []),
- {ok, TargetDb} = couch_db:open_int(couch_db:name(Target), []),
-
- Fun = fun(FullDocInfo, _, Acc) ->
- {ok, DocSource} = couch_db:open_doc(
- SourceDb, FullDocInfo, [conflicts, deleted_conflicts]),
- Id = DocSource#doc.id,
-
- etap:diag("Verifying document " ++ ?b2l(Id)),
-
- {ok, DocTarget} = couch_db:open_doc(
- TargetDb, Id, [conflicts, deleted_conflicts]),
- etap:is(DocTarget#doc.body, DocSource#doc.body,
- "Same body in source and target databases"),
-
- etap:is(
- couch_doc:to_json_obj(DocTarget, []),
- couch_doc:to_json_obj(DocSource, []),
- "Same doc body in source and target databases"),
-
- #doc{atts = SourceAtts} = DocSource,
- #doc{atts = TargetAtts} = DocTarget,
- etap:is(
- lists:sort([N || #att{name = N} <- SourceAtts]),
- lists:sort([N || #att{name = N} <- TargetAtts]),
- "Document has same number (and names) of attachments in "
- "source and target databases"),
-
- lists:foreach(
- fun(#att{name = AttName} = Att) ->
- etap:diag("Verifying attachment " ++ ?b2l(AttName)),
-
- {ok, AttTarget} = find_att(TargetAtts, AttName),
- SourceMd5 = att_md5(Att),
- TargetMd5 = att_md5(AttTarget),
- case AttName of
- <<"att1">> ->
- etap:is(Att#att.encoding, gzip,
- "Attachment is gzip encoded in source database"),
- etap:is(AttTarget#att.encoding, gzip,
- "Attachment is gzip encoded in target database"),
- DecSourceMd5 = att_decoded_md5(Att),
- DecTargetMd5 = att_decoded_md5(AttTarget),
- etap:is(DecTargetMd5, DecSourceMd5,
- "Same identity content in source and target databases");
- _ ->
- etap:is(Att#att.encoding, identity,
- "Attachment is not encoded in source database"),
- etap:is(AttTarget#att.encoding, identity,
- "Attachment is not encoded in target database")
- end,
- etap:is(TargetMd5, SourceMd5,
- "Same content in source and target databases"),
- etap:is(is_integer(Att#att.disk_len), true,
- "#att.disk_len is an integer in source database"),
- etap:is(is_integer(Att#att.att_len), true,
- "#att.att_len is an integer in source database"),
- etap:is(is_integer(AttTarget#att.disk_len), true,
- "#att.disk_len is an integer in target database"),
- etap:is(is_integer(AttTarget#att.att_len), true,
- "#att.att_len is an integer in target database"),
- etap:is(Att#att.disk_len, AttTarget#att.disk_len,
- "Same identity length in source and target databases"),
- etap:is(Att#att.att_len, AttTarget#att.att_len,
- "Same encoded length in source and target databases"),
- etap:is(Att#att.type, AttTarget#att.type,
- "Same type in source and target databases"),
- etap:is(Att#att.md5, SourceMd5, "Correct MD5 in source database"),
- etap:is(AttTarget#att.md5, SourceMd5, "Correct MD5 in target database")
- end,
- SourceAtts),
-
- {ok, Acc}
- end,
-
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
- ok = couch_db:close(SourceDb),
- ok = couch_db:close(TargetDb).
-
-
-find_att([], _Name) ->
- nil;
-find_att([#att{name = Name} = Att | _], Name) ->
- {ok, Att};
-find_att([_ | Rest], Name) ->
- find_att(Rest, Name).
-
-
-att_md5(Att) ->
- Md50 = couch_doc:att_foldl(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-att_decoded_md5(Att) ->
- Md50 = couch_doc:att_foldl_decode(
- Att,
- fun(Chunk, Acc) -> couch_util:md5_update(Acc, Chunk) end,
- couch_util:md5_init()),
- couch_util:md5_final(Md50).
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]).
-
-
-delete_db(Db) ->
- ok = couch_server:delete(
- couch_db:name(Db), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).
-
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, Reason} ->
- etap:is(Reason, normal, "Replication finished successfully")
- after 300000 ->
- etap:bail("Timeout waiting for replication to finish")
- end.
[05/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Posted by rn...@apache.org.
Port couch_replicator/01-load.t etap test suite to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/86aba61e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/86aba61e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/86aba61e
Branch: refs/heads/master
Commit: 86aba61edd46e19239207969fe430fa1d9b9261f
Parents: aafb5f9
Author: Alexander Shorin <kx...@apache.org>
Authored: Thu Jun 12 02:47:15 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../couch_replicator_modules_load_tests.erl | 40 ++++++++++++++++++++
test/01-load.t | 37 ------------------
2 files changed, 40 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/86aba61e/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_modules_load_tests.erl b/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
new file mode 100644
index 0000000..cea4cc2
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_modules_load_tests.erl
@@ -0,0 +1,40 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_modules_load_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+
+
+modules_load_test_() ->
+ {
+ "Verify that all modules loads",
+ should_load_modules()
+ }.
+
+
+should_load_modules() ->
+ Modules = [
+ couch_replicator_api_wrap,
+ couch_replicator_httpc,
+ couch_replicator_httpd,
+ couch_replicator_manager,
+ couch_replicator_notifier,
+ couch_replicator,
+ couch_replicator_worker,
+ couch_replicator_utils,
+ couch_replicator_job_sup
+ ],
+ [should_load_module(Mod) || Mod <- Modules].
+
+should_load_module(Mod) ->
+ {atom_to_list(Mod), ?_assertMatch({module, _}, code:load_file(Mod))}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/86aba61e/test/01-load.t
----------------------------------------------------------------------
diff --git a/test/01-load.t b/test/01-load.t
deleted file mode 100644
index 8bd82dd..0000000
--- a/test/01-load.t
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Test that we can load each module.
-
-main(_) ->
- test_util:init_code_path(),
- Modules = [
- couch_replicator_api_wrap,
- couch_replicator_httpc,
- couch_replicator_httpd,
- couch_replicator_manager,
- couch_replicator_notifier,
- couch_replicator,
- couch_replicator_worker,
- couch_replicator_utils,
- couch_replicator_job_sup
- ],
-
- etap:plan(length(Modules)),
- lists:foreach(
- fun(Module) ->
- etap:loaded_ok(Module, lists:concat(["Loaded: ", Module]))
- end, Modules),
- etap:end_tests().