You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2014/09/05 23:10:56 UTC
[09/14] couch-replicator commit: updated refs/heads/master to 3cf0b13
Port couch_replicator/05-replication-many-leaves.t etap test to eunit
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9a2c8388
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9a2c8388
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9a2c8388
Branch: refs/heads/master
Commit: 9a2c83881c3644a6a6363f84534b2c7f260aa486
Parents: b010033
Author: Alexander Shorin <kx...@apache.org>
Authored: Mon Jun 16 19:28:50 2014 +0400
Committer: Russell Branca <ch...@apache.org>
Committed: Thu Aug 28 10:29:40 2014 -0700
----------------------------------------------------------------------
.../test/couch_replicator_many_leaves_tests.erl | 232 +++++++++++++++
test/05-replication-many-leaves.t | 283 -------------------
2 files changed, 232 insertions(+), 283 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9a2c8388/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl b/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
new file mode 100644
index 0000000..7d0de9e
--- /dev/null
+++ b/src/couch_replicator/test/couch_replicator_many_leaves_tests.erl
@@ -0,0 +1,232 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_many_leaves_tests).
+
+-include("../../../test/couchdb/couch_eunit.hrl").
+-include_lib("couchdb/couch_db.hrl").
+
+-define(ADMIN_ROLE, #user_ctx{roles=[<<"_admin">>]}).
+-define(ADMIN_USER, {user_ctx, ?ADMIN_ROLE}).
+-define(DOCS_CONFLICTS, [
+ {<<"doc1">>, 10},
+ {<<"doc2">>, 100},
+ % a number > MaxURLlength (7000) / length(DocRevisionString)
+ {<<"doc3">>, 210}
+]).
+-define(NUM_ATTS, 2).
+-define(TIMEOUT_STOP, 1000).
+-define(TIMEOUT_EUNIT, 60).
+-define(i2l(I), integer_to_list(I)).
+-define(io2b(Io), iolist_to_binary(Io)).
+
+setup() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_USER]),
+ ok = couch_db:close(Db),
+ DbName.
+
+setup(local) ->
+ setup();
+setup(remote) ->
+ {remote, setup()};
+setup({A, B}) ->
+ {ok, _} = couch_server_sup:start_link(?CONFIG_CHAIN),
+ Source = setup(A),
+ Target = setup(B),
+ {Source, Target}.
+
+teardown({remote, DbName}) ->
+ teardown(DbName);
+teardown(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_USER]),
+ ok.
+
+teardown(_, {Source, Target}) ->
+ teardown(Source),
+ teardown(Target),
+
+ Pid = whereis(couch_server_sup),
+ erlang:monitor(process, Pid),
+ couch_server_sup:stop(),
+ receive
+ {'DOWN', _, _, Pid, _} ->
+ ok
+ after ?TIMEOUT_STOP ->
+ throw({timeout, server_stop})
+ end.
+
+
+docs_with_many_leaves_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Replicate documents with many leaves",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_populate_replicate_compact/2}
+ || Pair <- Pairs]
+ }
+ }.
+
+
+should_populate_replicate_compact({From, To}, {Source, Target}) ->
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
+ {inorder, [
+ should_populate_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target),
+ should_add_attachments_to_source(Source),
+ should_replicate(Source, Target),
+ should_verify_target(Source, Target)
+ ]}}.
+
+should_populate_source({remote, Source}) ->
+ should_populate_source(Source);
+should_populate_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(populate_db(Source))}.
+
+should_replicate({remote, Source}, Target) ->
+ should_replicate(db_url(Source), Target);
+should_replicate(Source, {remote, Target}) ->
+ should_replicate(Source, db_url(Target));
+should_replicate(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(replicate(Source, Target))}.
+
+should_verify_target({remote, Source}, Target) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, {remote, Target}) ->
+ should_verify_target(Source, Target);
+should_verify_target(Source, Target) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ {ok, TargetDb} = couch_db:open_int(Target, []),
+ verify_target(SourceDb, TargetDb, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb)
+ end)}.
+
+should_add_attachments_to_source({remote, Source}) ->
+ should_add_attachments_to_source(Source);
+should_add_attachments_to_source(Source) ->
+ {timeout, ?TIMEOUT_EUNIT, ?_test(begin
+ {ok, SourceDb} = couch_db:open_int(Source, []),
+ add_attachments(SourceDb, ?NUM_ATTS, ?DOCS_CONFLICTS),
+ ok = couch_db:close(SourceDb)
+ end)}.
+
+populate_db(DbName) ->
+ {ok, Db} = couch_db:open_int(DbName, []),
+ lists:foreach(
+ fun({DocId, NumConflicts}) ->
+ Value = <<"0">>,
+ Doc = #doc{
+ id = DocId,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ {ok, _} = add_doc_siblings(Db, DocId, NumConflicts)
+ end, ?DOCS_CONFLICTS),
+ couch_db:close(Db).
+
+add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
+ add_doc_siblings(Db, DocId, NumLeaves, [], []).
+
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
+ {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
+ {ok, AccRevs};
+
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
+ Value = ?l2b(?i2l(NumLeaves)),
+ Rev = couch_util:md5(Value),
+ Doc = #doc{
+ id = DocId,
+ revs = {1, [Rev]},
+ body = {[ {<<"value">>, Value} ]}
+ },
+ add_doc_siblings(Db, DocId, NumLeaves - 1,
+ [Doc | AccDocs], [{1, Rev} | AccRevs]).
+
+verify_target(_SourceDb, _TargetDb, []) ->
+ ok;
+verify_target(SourceDb, TargetDb, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(
+ SourceDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ {ok, TargetLookups} = couch_db:open_doc_revs(
+ TargetDb,
+ DocId,
+ all,
+ [conflicts, deleted_conflicts]),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ TargetDocs = [Doc || {ok, Doc} <- TargetLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(TargetDocs)),
+ lists:foreach(
+ fun({SourceDoc, TargetDoc}) ->
+ SourceJson = couch_doc:to_json_obj(SourceDoc, [attachments]),
+ TargetJson = couch_doc:to_json_obj(TargetDoc, [attachments]),
+ ?assertEqual(SourceJson, TargetJson)
+ end,
+ lists:zip(SourceDocs, TargetDocs)),
+ verify_target(SourceDb, TargetDb, Rest).
+
+add_attachments(_SourceDb, _NumAtts, []) ->
+ ok;
+add_attachments(SourceDb, NumAtts, [{DocId, NumConflicts} | Rest]) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(SourceDb, DocId, all, []),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ Total = NumConflicts + 1,
+ ?assertEqual(Total, length(SourceDocs)),
+ NewDocs = lists:foldl(
+ fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+ NewAtts = lists:foldl(fun(I, AttAcc) ->
+ AttData = crypto:rand_bytes(100),
+ NewAtt = #att{
+ name = ?io2b(["att_", ?i2l(I), "_",
+ couch_doc:rev_to_str({Pos, Rev})]),
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ },
+ [NewAtt | AttAcc]
+ end, [], lists:seq(1, NumAtts)),
+ [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+ end,
+ [], SourceDocs),
+ {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ NewRevs = [R || {ok, R} <- UpdateResults],
+ ?assertEqual(length(NewDocs), length(NewRevs)),
+ add_attachments(SourceDb, NumAtts, Rest).
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+replicate(Source, Target) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_ROLE),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _} ->
+ ok
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9a2c8388/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
deleted file mode 100755
index bd795ec..0000000
--- a/test/05-replication-many-leaves.t
+++ /dev/null
@@ -1,283 +0,0 @@
-#!/usr/bin/env escript
-%% -*- erlang -*-
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-% Test replication of documents with many leaf revisions.
-% Motivated by COUCHDB-1340 and other similar issues where a document
-% GET with a too long ?open_revs revision list doesn't work due to
-% maximum web server limits for the HTTP request path.
-
--record(user_ctx, {
- name = null,
- roles = [],
- handler
-}).
-
--record(doc, {
- id = <<"">>,
- revs = {0, []},
- body = {[]},
- atts = [],
- deleted = false,
- meta = []
-}).
-
--record(att, {
- name,
- type,
- att_len,
- disk_len,
- md5= <<>>,
- revpos=0,
- data,
- encoding=identity
-}).
-
--define(b2l(B), binary_to_list(B)).
--define(l2b(L), list_to_binary(L)).
--define(i2l(I), integer_to_list(I)).
-
-
-source_db_name() -> <<"couch_test_rep_db_a">>.
-target_db_name() -> <<"couch_test_rep_db_b">>.
-
-doc_ids() ->
- [<<"doc1">>, <<"doc2">>, <<"doc3">>].
-
-doc_num_conflicts(<<"doc1">>) -> 10;
-doc_num_conflicts(<<"doc2">>) -> 100;
-% a number > MaxURLlength (7000) / length(DocRevisionString)
-doc_num_conflicts(<<"doc3">>) -> 210.
-
-
-main(_) ->
- test_util:run(56, fun() -> test() end).
-
-
-test() ->
- test_util:start_couch(),
- ibrowse:start(),
- crypto:start(),
- config:set("replicator", "connection_timeout", "90000", false),
-
- Pairs = [
- {source_db_name(), target_db_name()},
- {{remote, source_db_name()}, target_db_name()},
- {source_db_name(), {remote, target_db_name()}},
- {{remote, source_db_name()}, {remote, (target_db_name())}}
- ],
-
- lists:foreach(
- fun({Source, Target}) ->
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- {ok, DocRevs} = populate_db(SourceDb),
- ok = couch_db:close(SourceDb),
- etap:diag("Creating target database"),
- {ok, TargetDb} = create_db(target_db_name()),
-
- ok = couch_db:close(TargetDb),
- etap:diag("Triggering replication"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- {ok, SourceDb2} = couch_db:open_int(source_db_name(), []),
- {ok, TargetDb2} = couch_db:open_int(target_db_name(), []),
- verify_target(SourceDb2, TargetDb2, DocRevs),
- ok = couch_db:close(SourceDb2),
- ok = couch_db:close(TargetDb2),
-
- {ok, SourceDb3} = couch_db:open_int(source_db_name(), []),
- {ok, DocRevs2} = add_attachments(SourceDb3, DocRevs, 2),
- ok = couch_db:close(SourceDb3),
- etap:diag("Triggering replication again"),
- replicate(Source, Target),
- etap:diag("Replication finished, comparing source and target databases"),
- {ok, SourceDb4} = couch_db:open_int(source_db_name(), []),
- {ok, TargetDb4} = couch_db:open_int(target_db_name(), []),
- verify_target(SourceDb4, TargetDb4, DocRevs2),
- ok = couch_db:close(SourceDb4),
- ok = couch_db:close(TargetDb4),
-
- etap:diag("Deleting source and target databases"),
- delete_db(TargetDb),
- delete_db(SourceDb),
- ok = timer:sleep(1000)
- end,
- Pairs),
-
- test_util:stop_couch().
-
-
-populate_db(Db) ->
- DocRevsDict = lists:foldl(
- fun(DocId, Acc) ->
- Value = <<"0">>,
- Doc = #doc{
- id = DocId,
- body = {[ {<<"value">>, Value} ]}
- },
- {ok, Rev} = couch_db:update_doc(Db, Doc, []),
- {ok, DocRevs} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
- dict:store(DocId, [Rev | DocRevs], Acc)
- end,
- dict:new(), doc_ids()),
- {ok, dict:to_list(DocRevsDict)}.
-
-
-add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
- add_doc_siblings(Db, DocId, NumLeaves, [], []).
-
-
-add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
- {ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
- {ok, AccRevs};
-
-add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
- Value = list_to_binary(integer_to_list(NumLeaves)),
- Rev = couch_util:md5(Value),
- Doc = #doc{
- id = DocId,
- revs = {1, [Rev]},
- body = {[ {<<"value">>, Value} ]}
- },
- add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], [{1, Rev} | AccRevs]).
-
-
-verify_target(_SourceDb, _TargetDb, []) ->
- ok;
-
-verify_target(SourceDb, TargetDb, [{DocId, RevList} | Rest]) ->
- {ok, Lookups} = couch_db:open_doc_revs(
- TargetDb,
- DocId,
- RevList,
- [conflicts, deleted_conflicts]),
- Docs = [Doc || {ok, Doc} <- Lookups],
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- RevList,
- [conflicts, deleted_conflicts]),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- Total = doc_num_conflicts(DocId) + 1,
- etap:is(
- length(Docs),
- Total,
- "Target has " ++ ?i2l(Total) ++ " leaf revisions of document " ++ ?b2l(DocId)),
- etap:diag("Verifying all revisions of document " ++ ?b2l(DocId)),
- lists:foreach(
- fun({#doc{id = Id, revs = Revs} = TgtDoc, #doc{id = Id, revs = Revs} = SrcDoc}) ->
- SourceJson = couch_doc:to_json_obj(SrcDoc, [attachments]),
- TargetJson = couch_doc:to_json_obj(TgtDoc, [attachments]),
- case TargetJson of
- SourceJson ->
- ok;
- _ ->
- {Pos, [Rev | _]} = Revs,
- etap:bail("Wrong value for revision " ++
- ?b2l(couch_doc:rev_to_str({Pos, Rev})) ++
- " of document " ++ ?b2l(DocId))
- end
- end,
- lists:zip(Docs, SourceDocs)),
- verify_target(SourceDb, TargetDb, Rest).
-
-
-add_attachments(Source, DocIdRevs, NumAtts) ->
- add_attachments(Source, DocIdRevs, NumAtts, []).
-
-add_attachments(_SourceDb, [], _NumAtts, Acc) ->
- {ok, Acc};
-
-add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
- {ok, SourceLookups} = couch_db:open_doc_revs(
- SourceDb,
- DocId,
- RevList,
- []),
- SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
- Total = doc_num_conflicts(DocId) + 1,
- etap:is(
- length(SourceDocs),
- Total,
- "Source still has " ++ ?i2l(Total) ++
- " leaf revisions of document " ++ ?b2l(DocId)),
- etap:diag("Adding " ++ ?i2l(NumAtts) ++
- " attachments to each revision of the document " ++ ?b2l(DocId)),
- NewDocs = lists:foldl(
- fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
- NewAtts = lists:foldl(
- fun(I, AttAcc) ->
- AttData = crypto:rand_bytes(100),
- NewAtt = #att{
- name = iolist_to_binary(
- ["att_", ?i2l(I), "_", couch_doc:rev_to_str({Pos, Rev})]),
- type = <<"application/foobar">>,
- att_len = byte_size(AttData),
- data = AttData
- },
- [NewAtt | AttAcc]
- end,
- [], lists:seq(1, NumAtts)),
- [Doc#doc{atts = Atts ++ NewAtts} | Acc]
- end,
- [], SourceDocs),
- {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
- NewRevs = [R || {ok, R} <- UpdateResults],
- etap:is(
- length(NewRevs),
- length(NewDocs),
- "Document revisions updated with " ++ ?i2l(NumAtts) ++ " attachments"),
- add_attachments(SourceDb, Rest, NumAtts, [{DocId, NewRevs} | IdRevsAcc]).
-
-
-db_url(DbName) ->
- iolist_to_binary([
- "http://", config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "/", DbName
- ]).
-
-
-create_db(DbName) ->
- couch_db:create(
- DbName,
- [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]).
-
-
-delete_db(Db) ->
- ok = couch_server:delete(
- couch_db:name(Db), [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]).
-
-
-replicate({remote, Db}, Target) ->
- replicate(db_url(Db), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, db_url(Db));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(
- RepObject, #user_ctx{roles = [<<"_admin">>]}),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- MonRef = erlang:monitor(process, Pid),
- receive
- {'DOWN', MonRef, process, Pid, Reason} ->
- etap:is(Reason, normal, "Replication finished successfully")
- after 900000 ->
- etap:bail("Timeout waiting for replication to finish")
- end.