You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by bb...@apache.org on 2014/10/31 20:53:11 UTC
[2/5] couch-replicator commit: updated refs/heads/master to c9184cf
don't replicate removed document in a view changes
Conflicts:
src/couch_replicator.erl
src/couch_replicator_api_wrap.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7390c7ec
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7390c7ec
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7390c7ec
Branch: refs/heads/master
Commit: 7390c7eccd19552ab842bc6b0a3c31fbbfc724c4
Parents: 3e3e2cb
Author: benoitc <bc...@gmail.com>
Authored: Thu Jul 3 13:08:34 2014 +0200
Committer: Benjamin Bastian <be...@gmail.com>
Committed: Thu Oct 30 13:29:56 2014 -0700
----------------------------------------------------------------------
src/couch_replicator.erl | 53 ++++++++++++++++++++++++++++++++++
src/couch_replicator_api_wrap.erl | 12 ++++++--
src/couch_replicator_worker.erl | 15 ++++++----
3 files changed, 72 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index f7e9d27..6992fff 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -647,6 +647,59 @@ fold_replication_logs([Db | Rest] = Dbs, Vsn, LogId, NewId, Rep, Acc) ->
end.
+spawn_changes_reader(StartSeq, #httpdb{} = Db, ChangesQueue, Options) ->
+ spawn_link(fun() ->
+ put(last_seq, StartSeq),
+ put(retries_left, Db#httpdb.retries),
+ read_changes(StartSeq, Db#httpdb{retries = 0}, ChangesQueue, Options)
+ end);
+spawn_changes_reader(StartSeq, Db, ChangesQueue, Options) ->
+ spawn_link(fun() ->
+ read_changes(StartSeq, Db, ChangesQueue, Options)
+ end).
+
+read_changes(StartSeq, Db, ChangesQueue, Options) ->
+ try
+ couch_replicator_api_wrap:changes_since(Db, all_docs, StartSeq, fun
+ (#doc_info{high_seq = Seq, revs = []}) ->
+ put(last_seq, Seq);
+ (#doc_info{high_seq = Seq, id = Id} = DocInfo) ->
+ case Id of
+ <<>> ->
+ % Previous CouchDB releases had a bug which allowed a doc
+ % with an empty ID to be inserted into databases. Such doc
+ % is impossible to GET.
+ ?LOG_ERROR("Replicator: ignoring document with empty ID in "
+ "source database `~s` (_changes sequence ~p)",
+ [couch_replicator_api_wrap:db_uri(Db), Seq]);
+ _ ->
+ ok = couch_work_queue:queue(ChangesQueue, DocInfo)
+ end,
+ put(last_seq, Seq)
+ end, Options),
+ couch_work_queue:close(ChangesQueue)
+ catch exit:{http_request_failed, _, _, _} = Error ->
+ case get(retries_left) of
+ N when N > 0 ->
+ put(retries_left, N - 1),
+ LastSeq = get(last_seq),
+ Db2 = case LastSeq of
+ StartSeq ->
+ ?LOG_INFO("Retrying _changes request to source database ~s"
+ " with since=~p in ~p seconds",
+ [couch_replicator_api_wrap:db_uri(Db), LastSeq, Db#httpdb.wait / 1000]),
+ ok = timer:sleep(Db#httpdb.wait),
+ Db#httpdb{wait = 2 * Db#httpdb.wait};
+ _ ->
+ ?LOG_INFO("Retrying _changes request to source database ~s"
+ " with since=~p", [couch_replicator_api_wrap:db_uri(Db), LastSeq]),
+ Db
+ end,
+ read_changes(LastSeq, Db2, ChangesQueue, Options);
+ _ ->
+ exit(Error)
+ end
+ end.
spawn_changes_manager(Parent, ChangesQueue, BatchSize) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 97e9b25..c35659d 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -839,12 +839,21 @@ json_to_doc_info({Props}) ->
undefined ->
{last_seq, get_value(<<"last_seq">>, Props)};
Changes ->
- RevsInfo = lists:map(
+ RevsInfo0 = lists:map(
fun({Change}) ->
Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)),
Del = couch_replicator_utils:is_deleted(Change),
#rev_info{rev=Rev, deleted=Del}
end, Changes),
+
+ RevsInfo = case get_value(<<"removed">>, Props) of
+ true ->
+ [_ | RevsInfo1] = RevsInfo0,
+ RevsInfo1;
+ _ ->
+ RevsInfo0
+ end,
+
#doc_info{
id = get_value(<<"id">>, Props),
high_seq = get_value(<<"seq">>, Props),
@@ -852,7 +861,6 @@ json_to_doc_info({Props}) ->
}
end.
-
bulk_results_to_errors(Docs, {ok, Results}, interactive_edit) ->
lists:reverse(lists:foldl(
fun({_, {ok, _}}, Acc) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7390c7ec/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index 7a6cc96..155e11d 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -491,12 +491,15 @@ flush_doc(Target, #doc{id = Id, revs = {Pos, [RevId | _]}} = Doc) ->
find_missing(DocInfos, Target) ->
- {IdRevs, AllRevsCount} = lists:foldr(
- fun(#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) ->
- Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo],
- {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)}
- end,
- {[], 0}, DocInfos),
+ {IdRevs, AllRevsCount} = lists:foldr(fun
+ (#doc_info{revs = []}, {IdRevAcc, CountAcc}) ->
+ {IdRevAcc, CountAcc};
+ (#doc_info{id = Id, revs = RevsInfo}, {IdRevAcc, CountAcc}) ->
+ Revs = [Rev || #rev_info{rev = Rev} <- RevsInfo],
+ {[{Id, Revs} | IdRevAcc], CountAcc + length(Revs)}
+ end, {[], 0}, DocInfos),
+
+
{ok, Missing} = couch_replicator_api_wrap:get_missing_revs(Target, IdRevs),
MissingRevsCount = lists:foldl(
fun({_Id, MissingRevs, _PAs}, Acc) -> Acc + length(MissingRevs) end,