You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ei...@apache.org on 2016/02/11 13:54:47 UTC
[1/2] couch-replicator commit: updated refs/heads/master to 61c59b5
Repository: couchdb-couch-replicator
Updated Branches:
refs/heads/master ea408866c -> 61c59b5d7
Add filtered with query replication test
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/61c59b5d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/61c59b5d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/61c59b5d
Branch: refs/heads/master
Commit: 61c59b5d7349b222276639bd839d3817542583c8
Parents: b18ff40
Author: Eric Avdey <ei...@eiri.ca>
Authored: Fri Feb 5 13:15:33 2016 -0400
Committer: Eric Avdey <ei...@eiri.ca>
Committed: Thu Feb 11 08:28:07 2016 -0400
----------------------------------------------------------------------
test/couch_replicator_filtered_tests.erl | 57 ++++++++++++++++++++++++++-
1 file changed, 56 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/61c59b5d/test/couch_replicator_filtered_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_filtered_tests.erl b/test/couch_replicator_filtered_tests.erl
index 71aabfb..d39aabd 100644
--- a/test/couch_replicator_filtered_tests.erl
+++ b/test/couch_replicator_filtered_tests.erl
@@ -21,6 +21,16 @@
{<<"filters">>, {[
{<<"testfilter">>, <<"
function(doc, req){if (doc.class == 'mammal') return true;}
+ ">>},
+ {<<"queryfilter">>, <<"
+ function(doc, req) {
+ if (doc.class && req.query.starts) {
+ return doc.class.indexOf(req.query.starts) === 0;
+ }
+ else {
+ return false;
+ }
+ }
">>}
]}}
]}).
@@ -50,6 +60,18 @@ filtered_replication_test_() ->
}
}.
+query_filtered_replication_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Filtered with query replication tests",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_succeed_with_query/2} || Pair <- Pairs]
+ }
+ }.
+
should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
RepObject = {[
{<<"source">>, db_url(From, Source)},
@@ -72,6 +94,32 @@ should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
]}.
+should_succeed_with_query({From, To}, {_Ctx, {Source, Target}}) ->
+ RepObject = {[
+ {<<"source">>, db_url(From, Source)},
+ {<<"target">>, db_url(To, Target)},
+ {<<"filter">>, <<"filter_ddoc/queryfilter">>},
+ {<<"query_params">>, {[
+ {<<"starts">>, <<"a">>}
+ ]}}
+ ]},
+ {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ FilterFun = fun(_DocId, {Props}) ->
+ case couch_util:get_value(<<"class">>, Props) of
+ <<"a", _/binary>> -> true;
+ _ -> false
+ end
+ end,
+ {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
+ {"Target DB has proper number of docs",
+ ?_assertEqual(2, proplists:get_value(doc_count, TargetDbInfo))},
+ {"Target DB doesn't have deleted docs",
+ ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
+ {"All the docs filtered as expected",
+ ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
+ ]}.
+
compare_dbs(Source, Target, FilterFun) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
@@ -120,6 +168,7 @@ create_docs(DbName) ->
]}),
Doc2 = couch_doc:from_json_obj({[
{<<"_id">>, <<"doc2">>},
+ {<<"class">>, <<"amphibians">>},
{<<"value">>, 2}
]}),
@@ -129,7 +178,13 @@ create_docs(DbName) ->
{<<"value">>, 3}
]}),
- {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3]),
+ Doc4 = couch_doc:from_json_obj({[
+ {<<"_id">>, <<"doc4">>},
+ {<<"class">>, <<"arthropods">>},
+ {<<"value">>, 2}
+
+ ]}),
+ {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3, Doc4]),
couch_db:ensure_full_commit(Db),
couch_db:close(Db).
[2/2] couch-replicator commit: updated refs/heads/master to 61c59b5
Posted by ei...@apache.org.
Fix filtered replication test
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b18ff40d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b18ff40d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b18ff40d
Branch: refs/heads/master
Commit: b18ff40d3adde03f4f85f9acff4950ddbb3b2e06
Parents: ea40886
Author: Eric Avdey <ei...@eiri.ca>
Authored: Fri Feb 5 11:28:49 2016 -0400
Committer: Eric Avdey <ei...@eiri.ca>
Committed: Thu Feb 11 08:28:07 2016 -0400
----------------------------------------------------------------------
test/couch_replicator_filtered_tests.erl | 152 ++++++++------------------
1 file changed, 47 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b18ff40d/test/couch_replicator_filtered_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_filtered_tests.erl b/test/couch_replicator_filtered_tests.erl
index c1a79e8..71aabfb 100644
--- a/test/couch_replicator_filtered_tests.erl
+++ b/test/couch_replicator_filtered_tests.erl
@@ -25,33 +25,16 @@
]}}
]}).
-setup() ->
- DbName = ?tempdb(),
- {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
- ok = couch_db:close(Db),
- upload_ddoc(DbName),
- create_docs(DbName),
- DbName.
-
-setup(local) ->
- setup();
-setup(remote) ->
- {remote, setup()};
-setup({A, B}) ->
+setup(_) ->
Ctx = test_util:start_couch([couch_replicator]),
- Source = setup(A),
- Target = setup(B),
+ Source = create_db(),
+ create_docs(Source),
+ Target = create_db(),
{Ctx, {Source, Target}}.
-teardown({remote, DbName}) ->
- teardown(DbName);
-teardown(DbName) ->
- ok = couch_server:delete(DbName, [?ADMIN_CTX]),
- ok.
-
teardown(_, {Ctx, {Source, Target}}) ->
- teardown(Source),
- teardown(Target),
+ delete_db(Source),
+ delete_db(Target),
ok = application:stop(couch_replicator),
ok = test_util:stop_couch(Ctx).
@@ -63,95 +46,72 @@ filtered_replication_test_() ->
{
foreachx,
fun setup/1, fun teardown/2,
- [{Pair, fun should_succeed/2}
- || Pair <- Pairs]
+ [{Pair, fun should_succeed/2} || Pair <- Pairs]
}
}.
-
should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
- {ok, RepPid, RepId} = replicate(Source, Target),
+ RepObject = {[
+ {<<"source">>, db_url(From, Source)},
+ {<<"target">>, db_url(To, Target)},
+ {<<"filter">>, <<"filter_ddoc/testfilter">>}
+ ]},
+ {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
%% FilteredFun is an Erlang version of following JS function
%% function(doc, req){if (doc.class == 'mammal') return true;}
FilterFun = fun(_DocId, {Props}) ->
couch_util:get_value(<<"class">>, Props) == <<"mammal">>
end,
- {lists:flatten(io_lib:format("~p -> ~p", [From, To])),
- {inorder, [
- should_ensure_replication_runs(RepPid),
- should_compare_databases(Source, Target, FilterFun),
- should_cancel_replication(RepId, RepPid)
- ]}}.
-
-should_ensure_replication_runs(RepPid) ->
- ?_assert(begin
- ?assertMatch(ok, wait_for_replicator(RepPid)),
- is_process_alive(RepPid)
- end).
-
-should_compare_databases({remote, Source}, Target, FilterFun) ->
- should_compare_databases(Source, Target, FilterFun);
-should_compare_databases(Source, {remote, Target}, FilterFun) ->
- should_compare_databases(Source, Target, FilterFun);
-should_compare_databases(Source, Target, FilterFun) ->
- ?_assertEqual(1, begin
- compare_dbs(Source, Target, FilterFun)
- end).
-
-should_cancel_replication(RepId, RepPid) ->
- ?_assertNot(begin
- ?assertMatch({ok, _}, couch_replicator:cancel_replication(RepId)),
- is_process_alive(RepPid)
- end).
+ {ok, TargetDbInfo, AllReplies} = compare_dbs(Source, Target, FilterFun),
+ {lists:flatten(io_lib:format("~p -> ~p", [From, To])), [
+ {"Target DB has proper number of docs",
+ ?_assertEqual(1, proplists:get_value(doc_count, TargetDbInfo))},
+ {"Target DB doesn't have deleted docs",
+ ?_assertEqual(0, proplists:get_value(doc_del_count, TargetDbInfo))},
+ {"All the docs filtered as expected",
+ ?_assert(lists:all(fun(Valid) -> Valid end, AllReplies))}
+ ]}.
compare_dbs(Source, Target, FilterFun) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
+ {ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
Fun = fun(FullDocInfo, _, Acc) ->
- {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo, ?LINE),
+ {ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
+ TargetReply = read_doc(TargetDb, DocId),
case FilterFun(DocId, SourceDoc) of
true ->
- {ok, DocId, TargetDoc} = read_doc(TargetDb, DocId, ?LINE),
- ?assertEqual(SourceDoc, TargetDoc),
- {ok, Acc + 1};
+ ValidReply = {ok, DocId, SourceDoc} == TargetReply,
+ {ok, [ValidReply|Acc]};
false ->
- {ok, Acc}
+ ValidReply = {not_found, missing} == TargetReply,
+ {ok, [ValidReply|Acc]}
end
end,
- {ok, _, Acc} = couch_db:enum_docs(SourceDb, Fun, 0, []),
+ {ok, _, AllReplies} = couch_db:enum_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb),
- Acc.
+ {ok, TargetDbInfo, AllReplies}.
-read_doc(Db, DocIdOrInfo, Line) ->
+read_doc(Db, DocIdOrInfo) ->
case couch_db:open_doc(Db, DocIdOrInfo) of
{ok, Doc} ->
{Props} = couch_doc:to_json_obj(Doc, [attachments]),
DocId = couch_util:get_value(<<"_id">>, Props),
{ok, DocId, {Props}};
Error ->
- Reason = lists:concat([
- "Error opening document '", ?b2l(doc_id(DocIdOrInfo)),
- "' from target: ", couch_util:to_list(Error)]),
- erlang:error(
- {assertion_failed,
- [{module, ?MODULE}, {line, Line},
- {reason, Reason}]})
+ Error
end.
-doc_id(#full_doc_info{id = Id}) -> Id;
-doc_id(Id) -> Id.
-
-
-wait_for_replicator(Pid) ->
- %% since replicator started asynchronously
- %% we need to wait when it would be in couch_task_status
- %% we query replicator:details to ensure that do_init happen
- ?assertMatch({ok, _}, couch_replicator:details(Pid)),
- ok.
+create_db() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
+ ok = couch_db:close(Db),
+ DbName.
create_docs(DbName) ->
{ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
+ DDoc = couch_doc:from_json_obj(?DDOC),
Doc1 = couch_doc:from_json_obj({[
{<<"_id">>, <<"doc1">>},
{<<"class">>, <<"mammal">>},
@@ -169,34 +129,16 @@ create_docs(DbName) ->
{<<"value">>, 3}
]}),
- {ok, _} = couch_db:update_docs(Db, [Doc1, Doc2, Doc3]),
+ {ok, _} = couch_db:update_docs(Db, [DDoc, Doc1, Doc2, Doc3]),
couch_db:ensure_full_commit(Db),
couch_db:close(Db).
+delete_db(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_CTX]).
-upload_ddoc(DbName) ->
- Url = db_url(DbName) ++ "/_design/filter_ddoc",
- Body = couch_util:json_encode(?DDOC),
- {ok, 201, _Resp, _Body} = test_request:put(Url, Body),
- ok.
-
-replicate({remote, Db}, Target) ->
- replicate(?l2b(db_url(Db)), Target);
-
-replicate(Source, {remote, Db}) ->
- replicate(Source, ?l2b(db_url(Db)));
-
-replicate(Source, Target) ->
- RepObject = {[
- {<<"source">>, Source},
- {<<"target">>, Target},
- {<<"continuous">>, true}
- ]},
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepObject, ?ADMIN_USER),
- {ok, Pid} = couch_replicator:async_replicate(Rep),
- {ok, Pid, Rep#rep.id}.
-
-db_url(DbName) ->
+db_url(local, DbName) ->
+ DbName;
+db_url(remote, DbName) ->
Addr = config:get("httpd", "bind_address", "127.0.0.1"),
- Port = integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
- "http://" ++ Addr ++ ":" ++ Port ++ "/" ++ ?b2l(DbName).
+ Port = mochiweb_socket_server:get(couch_httpd, port),
+ ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).