You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by fd...@apache.org on 2012/01/04 15:46:17 UTC
git commit: Improve test 05-replication-many-leaves.t
Updated Branches:
refs/heads/master 32fb9f8e0 -> 75b6e09b5
Improve test 05-replication-many-leaves.t
The test now includes attachments to verify that the
incremental attachment replication code path succeeds
when there are many leaf revisions of a document with
attachments.
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/75b6e09b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/75b6e09b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/75b6e09b
Branch: refs/heads/master
Commit: 75b6e09b5bfa9b2f0022d69cf53d41b5757f1146
Parents: 32fb9f8
Author: Filipe David Borba Manana <fd...@apache.org>
Authored: Wed Jan 4 13:47:27 2012 +0000
Committer: Filipe David Borba Manana <fd...@apache.org>
Committed: Wed Jan 4 13:47:27 2012 +0000
----------------------------------------------------------------------
.../test/05-replication-many-leaves.t | 148 +++++++++++----
1 files changed, 112 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/75b6e09b/src/couch_replicator/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/src/couch_replicator/test/05-replication-many-leaves.t b/src/couch_replicator/test/05-replication-many-leaves.t
index 52d2023..22c7df8 100755
--- a/src/couch_replicator/test/05-replication-many-leaves.t
+++ b/src/couch_replicator/test/05-replication-many-leaves.t
@@ -32,6 +32,17 @@
meta = []
}).
+-record(att, {
+ name,
+ type,
+ att_len,
+ disk_len,
+ md5= <<>>,
+ revpos=0,
+ data,
+ encoding=identity
+}).
+
-define(b2l(B), binary_to_list(B)).
-define(l2b(L), list_to_binary(L)).
-define(i2l(I), integer_to_list(I)).
@@ -43,15 +54,15 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
doc_ids() ->
[<<"doc1">>, <<"doc2">>, <<"doc3">>].
-doc_num_conflicts(<<"doc1">>) -> 100;
-doc_num_conflicts(<<"doc2">>) -> 200;
-doc_num_conflicts(<<"doc3">>) -> 550.
+doc_num_conflicts(<<"doc1">>) -> 10;
+doc_num_conflicts(<<"doc2">>) -> 100;
+doc_num_conflicts(<<"doc3">>) -> 286.
main(_) ->
test_util:init_code_path(),
- etap:plan(16),
+ etap:plan(56),
case (catch test()) of
ok ->
etap:end_tests();
@@ -74,13 +85,12 @@ test() ->
{{remote, source_db_name()}, {remote, (target_db_name())}}
],
- {ok, SourceDb} = create_db(source_db_name()),
- etap:diag("Populating source database"),
- {ok, DocRevs} = populate_db(SourceDb),
- ok = couch_db:close(SourceDb),
-
lists:foreach(
fun({Source, Target}) ->
+ {ok, SourceDb} = create_db(source_db_name()),
+ etap:diag("Populating source database"),
+ {ok, DocRevs} = populate_db(SourceDb),
+ ok = couch_db:close(SourceDb),
etap:diag("Creating target database"),
{ok, TargetDb} = create_db(target_db_name()),
@@ -88,17 +98,31 @@ test() ->
etap:diag("Triggering replication"),
replicate(Source, Target),
etap:diag("Replication finished, comparing source and target databases"),
+ {ok, SourceDb2} = couch_db:open_int(source_db_name(), []),
{ok, TargetDb2} = couch_db:open_int(target_db_name(), []),
- verify_target(TargetDb2, DocRevs),
+ verify_target(SourceDb2, TargetDb2, DocRevs),
+ ok = couch_db:close(SourceDb2),
ok = couch_db:close(TargetDb2),
- etap:diag("Deleting target database"),
+ {ok, SourceDb3} = couch_db:open_int(source_db_name(), []),
+ {ok, DocRevs2} = add_attachments(SourceDb3, DocRevs, 2),
+ ok = couch_db:close(SourceDb3),
+ etap:diag("Triggering replication again"),
+ replicate(Source, Target),
+ etap:diag("Replication finished, comparing source and target databases"),
+ {ok, SourceDb4} = couch_db:open_int(source_db_name(), []),
+ {ok, TargetDb4} = couch_db:open_int(target_db_name(), []),
+ verify_target(SourceDb4, TargetDb4, DocRevs2),
+ ok = couch_db:close(SourceDb4),
+ ok = couch_db:close(TargetDb4),
+
+ etap:diag("Deleting source and target databases"),
delete_db(TargetDb),
+ delete_db(SourceDb),
ok = timer:sleep(1000)
end,
Pairs),
- delete_db(SourceDb),
couch_server_sup:stop(),
ok.
@@ -112,44 +136,48 @@ populate_db(Db) ->
body = {[ {<<"value">>, Value} ]}
},
{ok, Rev} = couch_db:update_doc(Db, Doc, []),
- {ok, RevsDict} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
- RevsDict2 = dict:store(Rev, Value, RevsDict),
- dict:store(DocId, RevsDict2, Acc)
+ {ok, DocRevs} = add_doc_siblings(Db, DocId, doc_num_conflicts(DocId)),
+ dict:store(DocId, [Rev | DocRevs], Acc)
end,
dict:new(), doc_ids()),
{ok, dict:to_list(DocRevsDict)}.
add_doc_siblings(Db, DocId, NumLeaves) when NumLeaves > 0 ->
- add_doc_siblings(Db, DocId, NumLeaves, [], dict:new()).
+ add_doc_siblings(Db, DocId, NumLeaves, [], []).
-add_doc_siblings(Db, _DocId, 0, AccDocs, RevsDict) ->
+add_doc_siblings(Db, _DocId, 0, AccDocs, AccRevs) ->
{ok, []} = couch_db:update_docs(Db, AccDocs, [], replicated_changes),
- {ok, RevsDict};
+ {ok, AccRevs};
-add_doc_siblings(Db, DocId, NumLeaves, AccDocs, RevsDict) ->
+add_doc_siblings(Db, DocId, NumLeaves, AccDocs, AccRevs) ->
Value = list_to_binary(integer_to_list(NumLeaves)),
Rev = couch_util:md5(Value),
- RevsDict2 = dict:store({1, Rev}, Value, RevsDict),
Doc = #doc{
id = DocId,
revs = {1, [Rev]},
body = {[ {<<"value">>, Value} ]}
},
- add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], RevsDict2).
+ add_doc_siblings(Db, DocId, NumLeaves - 1, [Doc | AccDocs], [{1, Rev} | AccRevs]).
-verify_target(_TargetDb, []) ->
+verify_target(_SourceDb, _TargetDb, []) ->
ok;
-verify_target(TargetDb, [{DocId, RevsDict} | Rest]) ->
+verify_target(SourceDb, TargetDb, [{DocId, RevList} | Rest]) ->
{ok, Lookups} = couch_db:open_doc_revs(
TargetDb,
DocId,
- [R || {R, _} <- dict:to_list(RevsDict)],
- [ejson_body]),
+ RevList,
+ []),
Docs = [Doc || {ok, Doc} <- Lookups],
+ {ok, SourceLookups} = couch_db:open_doc_revs(
+ SourceDb,
+ DocId,
+ RevList,
+ []),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
Total = doc_num_conflicts(DocId) + 1,
etap:is(
length(Docs),
@@ -157,21 +185,69 @@ verify_target(TargetDb, [{DocId, RevsDict} | Rest]) ->
"Target has " ++ ?i2l(Total) ++ " leaf revisions of document " ++ ?b2l(DocId)),
etap:diag("Verifying all revisions of document " ++ ?b2l(DocId)),
lists:foreach(
- fun(#doc{revs = {Pos, [RevId]}, body = {Body}}) ->
- Rev = {Pos, RevId},
- {ok, Value} = dict:find(Rev, RevsDict),
- case couch_util:get_value(<<"value">>, Body) of
- Value ->
+ fun({#doc{id = Id, revs = Revs} = TgtDoc, #doc{id = Id, revs = Revs} = SrcDoc}) ->
+ SourceJson = couch_doc:to_json_obj(SrcDoc, [attachments, conflicts]),
+ TargetJson = couch_doc:to_json_obj(TgtDoc, [attachments, conflicts]),
+ case TargetJson of
+ SourceJson ->
ok;
- Other ->
+ _ ->
+ {Pos, [Rev | _]} = Revs,
etap:bail("Wrong value for revision " ++
- ?b2l(couch_doc:rev_to_str(Rev)) ++ " of document " ++
- ?b2l(DocId) ++ ". Expected `" ++ couch_util:to_list(Value) ++
- "`, got `" ++ couch_util:to_list(Other) ++ "`")
+ ?b2l(couch_doc:rev_to_str({Pos, Rev})) ++
+ " of document " ++ ?b2l(DocId))
end
end,
- Docs),
- verify_target(TargetDb, Rest).
+ lists:zip(Docs, SourceDocs)),
+ verify_target(SourceDb, TargetDb, Rest).
+
+
+add_attachments(Source, DocIdRevs, NumAtts) ->
+ add_attachments(Source, DocIdRevs, NumAtts, []).
+
+add_attachments(_SourceDb, [], _NumAtts, Acc) ->
+ {ok, Acc};
+
+add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
+ {ok, SourceLookups} = couch_db:open_doc_revs(
+ SourceDb,
+ DocId,
+ RevList,
+ []),
+ SourceDocs = [Doc || {ok, Doc} <- SourceLookups],
+ Total = doc_num_conflicts(DocId) + 1,
+ etap:is(
+ length(SourceDocs),
+ Total,
+ "Source still has " ++ ?i2l(Total) ++
+ " leaf revisions of document " ++ ?b2l(DocId)),
+ etap:diag("Adding " ++ ?i2l(NumAtts) ++
+ " attachments to each revision of the document " ++ ?b2l(DocId)),
+ NewDocs = lists:foldl(
+ fun(#doc{atts = Atts, revs = {Pos, [Rev | _]}} = Doc, Acc) ->
+ NewAtts = lists:foldl(
+ fun(I, AttAcc) ->
+ AttData = crypto:rand_bytes(100),
+ NewAtt = #att{
+ name = iolist_to_binary(
+ ["att_", ?i2l(I), "_", couch_doc:rev_to_str({Pos, Rev})]),
+ type = <<"application/foobar">>,
+ att_len = byte_size(AttData),
+ data = AttData
+ },
+ [NewAtt | AttAcc]
+ end,
+ [], lists:seq(1, NumAtts)),
+ [Doc#doc{atts = Atts ++ NewAtts} | Acc]
+ end,
+ [], SourceDocs),
+ {ok, UpdateResults} = couch_db:update_docs(SourceDb, NewDocs, []),
+ NewRevs = [R || {ok, R} <- UpdateResults],
+ etap:is(
+ length(NewRevs),
+ length(NewDocs),
+ "Document revisions updated with " ++ ?i2l(NumAtts) ++ " attachments"),
+ add_attachments(SourceDb, Rest, NumAtts, [{DocId, NewRevs} | IdRevsAcc]).
db_url(DbName) ->