You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by mi...@apache.org on 2015/06/02 19:29:08 UTC
[01/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Repository: couchdb-couch-replicator
Updated Branches:
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork [created] 367562b41
Use randomized, truncated exponential backoff in event of conflict
BugzID: 42053
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/6e8fbb6a3f2622c14ae605c18ec54cbad7d389f3
Conflicts:
src/couch_replicator_manager.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/dcb38869
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/dcb38869
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/dcb38869
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: dcb38869bdb1ed44bb90ed45bc993a6452be1cb6
Parents: bbab85b
Author: Robert Newson <rn...@apache.org>
Authored: Mon Dec 1 11:11:00 2014 +0000
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 17:43:00 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/dcb38869/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index caae55f..0fcb0c4 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -596,11 +596,14 @@ clean_up_replications(DbName) ->
ets:delete(?DB_TO_SEQ,DbName).
-update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
+update_rep_doc(RepDbName, RepDocId, KVs) ->
+ update_rep_doc(RepDbName, RepDocId, KVs, 1).
+
+update_rep_doc(RepDbName, RepDocId, KVs, Wait) when is_binary(RepDocId) ->
try
case open_rep_doc(RepDbName, RepDocId) of
{ok, LastRepDoc} ->
- update_rep_doc(RepDbName, LastRepDoc, KVs);
+ update_rep_doc(RepDbName, LastRepDoc, KVs, Wait * 2);
_ ->
ok
end
@@ -608,10 +611,10 @@ update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
throw:conflict ->
Msg = "Conflict when updating replication document `~s`. Retrying.",
couch_log:error(Msg, [RepDocId]),
- ok = timer:sleep(5),
- update_rep_doc(RepDbName, RepDocId, KVs)
+ ok = timer:sleep(random:uniform(erlang:min(128, Wait)) * 100),
+ update_rep_doc(RepDbName, RepDocId, KVs, Wait * 2)
end;
-update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
+update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs, _Try) ->
NewRepDocBody = lists:foldl(
fun({K, undefined}, Body) ->
lists:keydelete(K, 1, Body);
[09/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Ensure that ibrowse streams are ended properly
I found a situation where we had live lock on a running application due
to an ibrowse request that hadn't been properly terminated. This
manifested as a cesation of updates to the _active_tasks information.
Debugging this lead me to see that the main couch_replicator pid was
stuck on a call to get_pending_changes. This call was stuck because the
ibrowse_http_client process being used was stuck waiting for a changes
request to complete.
This changes request as it turns out had been abandoned by the
couch_replicator_changes_reader. The changes reader was then stuck
trying to do a gen_server:call/2 back to the main couch_replicator
process with the report_seq_done message.
Given all this, it became apparent that the changes feed improperly
ending its ibrowse streams was the underlying culprit. Issuing a call to
ibrowse:stream_next/1 with the abandoned ibrowse stream id resulted in
the replication resuming.
This bug was introduced in this commit:
bfa020b43be20c54ab166c51f5c6e55c34d844c2
BugzId: 47306
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/f9db37a9b293f5f078681e7539fd35a92eb3adec
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/89d316e5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/89d316e5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/89d316e5
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 89d316e51d49c74b6afe5fc5c7de64c77d299df2
Parents: 44e755e
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed May 20 17:04:31 2015 -0500
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:20:06 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_httpc.erl | 29 ++++++++++++++++++++++-------
1 file changed, 22 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/89d316e5/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index 9a6213a..e61fba5 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -27,6 +27,7 @@
-define(replace(L, K, V), lists:keystore(K, 1, L, {K, V})).
-define(MAX_WAIT, 5 * 60 * 1000).
+-define(STREAM_STATUS, ibrowse_stream_status).
setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
@@ -35,6 +36,7 @@ setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
send_req(HttpDb, Params1, Callback) ->
+ put(STREAM_STATUS, init),
couch_stats:increment_counter([couch_replicator, requests]),
Params2 = ?replace(Params1, qs,
[{K, ?b2l(iolist_to_binary(V))} || {K, V} <- get_value(qs, Params1, [])]),
@@ -132,6 +134,7 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) ->
StreamDataFun = fun() ->
stream_data_self(HttpDb, Params, Worker, ReqId, Callback)
end,
+ put(STREAM_STATUS, streaming),
ibrowse:stream_next(ReqId),
try
Ret = Callback(Ok, Headers, StreamDataFun),
@@ -167,13 +170,24 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) ->
% messages for the given ReqId on the floor since we're
% no longer in the HTTP request.
clean_mailbox({ibrowse_req_id, ReqId}) ->
- receive
- {ibrowse_async_response, ReqId, _} ->
- clean_mailbox({ibrowse_req_id, ReqId});
- {ibrowse_async_response_end, ReqId} ->
- clean_mailbox({ibrowse_req_id, ReqId})
- after 0 ->
- ok
+ case get(STREAM_STATUS) of
+ streaming ->
+ ibrowse:stream_next(ReqId),
+ receive
+ {ibrowse_async_response, ReqId, _} ->
+ clean_mailbox({ibrowse_req_id, ReqId});
+ {ibrowse_async_response_end, ReqId} ->
+ put(STREAM_STATUS, ended),
+ ok
+ end;
+ _ ->
+ receive
+ {ibrowse_async_response, ReqId, _} ->
+ clean_mailbox({ibrowse_req_id, ReqId});
+ {ibrowse_async_response_end, ReqId} ->
+ put(STREAM_STATUS, ended),
+ ok
+ end
end;
clean_mailbox(_) ->
ok.
@@ -222,6 +236,7 @@ stream_data_self(#httpdb{timeout = T} = HttpDb, Params, Worker, ReqId, Cb) ->
ibrowse:stream_next(ReqId),
{Data, fun() -> stream_data_self(HttpDb, Params, Worker, ReqId, Cb) end};
{Data, ibrowse_async_response_end} ->
+ put(STREAM_STATUS, ended),
{Data, fun() -> throw({maybe_retry_req, more_data_expected}) end}
end.
[11/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Fix stream cleanup timeouts
The first part of this is adding the `after 0` clause. The issue here is
that ibrowse sends the `ibrowse_async_response_end` message without
waiting for a call to `ibrowse:stream_next/1`. This means that the
continuous changes feed may or may not get this message in
`couch_replicator_httpc:accumulate_messages/3`. If it does then we would
end up on an infinite timeout waiting for it. This was a typo in the
original patch in that I meant to include it but forgot.
The second timeout is so that we don't end up halted waiting for a
changes request to finish. If it takes longer than 30s we just crash the
replication and let the manager restart things.
BugzId: 47306
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/2caf39040e4e50c38a7758d4d09e7a8b22ea92d4
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/50846048
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/50846048
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/50846048
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 508460488656e171ac8a020a9f72d949444cd225
Parents: c595e57
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu May 21 10:02:53 2015 -0500
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:21:23 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_httpc.erl | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/50846048/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index 544c2ef..e591601 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -179,6 +179,8 @@ clean_mailbox({ibrowse_req_id, ReqId}) ->
{ibrowse_async_response_end, ReqId} ->
put(?STREAM_STATUS, ended),
ok
+ after 30000 ->
+ exit({timeout, ibrowse_stream_cleanup})
end;
Status when Status == init; Status == ended ->
receive
@@ -187,6 +189,8 @@ clean_mailbox({ibrowse_req_id, ReqId}) ->
{ibrowse_async_response_end, ReqId} ->
put(?STREAM_STATUS, ended),
ok
+ after 0 ->
+ ok
end
end;
clean_mailbox(_) ->
[03/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Verify that url really points to a database
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/e73714196936c345d54158e674ab36cab20beeec
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/74da56e5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/74da56e5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/74da56e5
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 74da56e5d396a7bc739c1ebb50e6d98a52dac858
Parents: 2e6d5c1
Author: Robert Newson <rn...@apache.org>
Authored: Wed Dec 3 11:41:48 2014 +0000
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 17:43:33 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/74da56e5/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 504a3d1..ef38c73 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -83,9 +83,16 @@ db_open(#httpdb{} = Db1, _Options, Create) ->
ok
end)
end,
- send_req(Db, [{method, head}],
- fun(200, _, _) ->
- {ok, Db};
+ send_req(Db, [{method, get}],
+ fun(200, _, {Props}) ->
+ case get_value(<<"update_seq">>, Props) of
+ undefined ->
+ throw({db_not_found, ?l2b(db_uri(Db))});
+ _ ->
+ {ok, Db}
+ end;
+ (200, _, _Body) ->
+ throw({db_not_found, ?l2b(db_uri(Db))});
(401, _, _) ->
throw({unauthorized, ?l2b(db_uri(Db))});
(_, _, _) ->
[08/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Cleanly stop replication at checkpoint time if no longer owner
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/e5ef7c8a0ee2566b9cd4c02397ee94883d015fa0
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/44e755e5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/44e755e5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/44e755e5
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 44e755e5108262877c2aad48f3846e7139c877b7
Parents: 3675764
Author: Robert Newson <rn...@apache.org>
Authored: Fri May 15 16:31:24 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:19:09 2015 +0100
----------------------------------------------------------------------
src/couch_replicator.erl | 21 ++++++++++++++-------
src/couch_replicator_manager.erl | 13 ++++++++++++-
2 files changed, 26 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/44e755e5/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 6e86a8f..3a744cd 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -475,13 +475,20 @@ handle_cast({db_compacted, DbName},
{noreply, State#rep_state{target = NewTarget}};
handle_cast(checkpoint, State) ->
- case do_checkpoint(State) of
- {ok, NewState} ->
- couch_stats:increment_counter([couch_replicator, checkpoints, success]),
- {noreply, NewState#rep_state{timer = start_timer(State)}};
- Error ->
- couch_stats:increment_counter([couch_replicator, checkpoints, failure]),
- {stop, Error, State}
+ #rep_state{rep_details = #rep{id = RepId} = Rep} = State,
+ case couch_replicator_manager:owner(RepId) of
+ Owner when Owner == node() ->
+ case do_checkpoint(State) of
+ {ok, NewState} ->
+ couch_stats:increment_counter([couch_replicator, checkpoints, success]),
+ {noreply, NewState#rep_state{timer = start_timer(State)}};
+ Error ->
+ couch_stats:increment_counter([couch_replicator, checkpoints, failure]),
+ {stop, Error, State}
+ end;
+ Owner ->
+ couch_replicator_manager:replication_usurped(Rep, Owner),
+ {stop, shutdown, State}
end;
handle_cast({report_seq, Seq},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/44e755e5/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 6410a85..39e06b3 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -17,7 +17,7 @@
% public API
-export([replication_started/1, replication_completed/2, replication_error/2]).
--export([owner/1]).
+-export([owner/1, replication_usurped/2]).
-export([before_doc_update/2, after_doc_read/2]).
@@ -108,6 +108,17 @@ replication_completed(#rep{id = RepId}, Stats) ->
end.
+replication_usurped(#rep{id = RepId}, By) ->
+ case rep_state(RepId) of
+ nil ->
+ ok;
+ #rep_state{rep = #rep{doc_id = DocId}} ->
+ ok = gen_server:call(?MODULE, {rep_complete, RepId}, infinity),
+ couch_log:notice("Replication `~s` usurped by ~s (triggered by document `~s`)",
+ [pp_rep_id(RepId), By, DocId])
+ end.
+
+
replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
case rep_state(RepId) of
nil ->
[06/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Return owner to improve logging output
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/3287da36a24b5d061a64cc93814f2f4580fdd4f9
Conflicts:
src/couch_replicator_manager.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/5dc5c415
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/5dc5c415
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/5dc5c415
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 5dc5c415a570d90918814eaa743747faf449d71d
Parents: dab36db
Author: Robert Newson <rn...@apache.org>
Authored: Fri May 15 16:30:34 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:16:53 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 22 +++++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/5dc5c415/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 9cfda3f..a203b9f 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -17,6 +17,7 @@
% public API
-export([replication_started/1, replication_completed/2, replication_error/2]).
+-export([owner/1]).
-export([before_doc_update/2, after_doc_read/2]).
@@ -120,6 +121,10 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
end.
+owner(RepId) ->
+ gen_server:call(?MODULE, {owner, RepId}).
+
+
handle_config_change("replicator", "db", _, _, S) ->
ok = gen_server:call(S, rep_db_changed),
remove_handler;
@@ -159,6 +164,13 @@ init(_) ->
live = Live
}}.
+handle_call({owner, RepId}, _From, State) ->
+ case rep_state(RepId) of
+ nil ->
+ false;
+ #rep_state{dbname = DbName, rep = #rep{doc_id = DocId}} ->
+ {reply, owner(DbName, DocId, State#state.live), State}
+ end;
handle_call({rep_db_update, DbName, {ChangeProps} = Change}, _From, State) ->
NewState = try
@@ -392,9 +404,11 @@ process_update(State, DbName, {Change}) ->
{_, true} ->
rep_doc_deleted(DbName, DocId),
State;
- {false, false} ->
+ {Owner, false} when Owner /= node() ->
+ couch_log:notice("Not starting '~s' as owner is ~s.", [DocId, Owner]),
State;
- {true, false} ->
+ {_Owner, false} ->
+ couch_log:notice("Maybe starting '~s' as I'm the owner", [DocId]),
case get_json_value(<<"_replication_state">>, RepProps) of
undefined ->
maybe_start_replication(State, DbName, DocId, JsonRepDoc);
@@ -416,9 +430,7 @@ process_update(State, DbName, {Change}) ->
owner(<<"shards/", _/binary>> = DbName, DocId, Live) ->
Nodes = lists:sort([N || #shard{node=N} <- mem3:shards(mem3:dbname(DbName), DocId),
lists:member(N, Live)]),
- node() =:= hd(mem3_util:rotate_list({DbName, DocId}, Nodes));
-owner(_DbName, _DocId, _Live) ->
- true.
+ hd(mem3_util:rotate_list({DbName, DocId}, Nodes)).
rep_db_update_error(Error, DbName, DocId) ->
case Error of
[04/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
delay and splay replication starts
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/d279150d959cfd46cbd77c5dd17f14d6dc3d0291
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ea39d118
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ea39d118
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ea39d118
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: ea39d118a08781f003b8756dee3324d65038f7a9
Parents: 74da56e
Author: Robert Newson <rn...@apache.org>
Authored: Wed Dec 3 12:14:10 2014 +0000
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 17:46:25 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ea39d118/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 17c766e..0cb0ce4 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -454,7 +454,14 @@ maybe_start_replication(State, DbName, DocId, RepDoc) ->
true = ets:insert(?DOC_TO_REP, {{DbName, DocId}, RepId}),
couch_log:notice("Attempting to start replication `~s` (document `~s`).",
[pp_rep_id(RepId), DocId]),
- Pid = spawn_link(?MODULE, start_replication, [Rep, 0]),
+ StartDelaySecs = erlang:max(0, list_to_integer(
+ config:get("replicator", "start_delay", "10"))),
+ StartSplaySecs = erlang:max(1, list_to_integer(
+ config:get("replicator", "start_splay", "50"))),
+ DelaySecs = StartDelaySecs + random:uniform(StartSplaySecs),
+ couch_log:notice("Delaying replication `~s` start by ~p seconds.",
+ [pp_rep_id(RepId), DelaySecs]),
+ Pid = spawn_link(?MODULE, start_replication, [Rep, DelaySecs]),
State#state{rep_start_pids = [Pid | State#state.rep_start_pids]};
#rep_state{rep = #rep{doc_id = DocId}} ->
State;
[02/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Remove anonymous fun when starting replications
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/faa28a6e7f5b460b1d3ca2f77b00ab7d5371021d
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/2e6d5c1a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/2e6d5c1a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/2e6d5c1a
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 2e6d5c1a6a74629509c247b620a877ddfce1daf5
Parents: dcb3886
Author: Robert Newson <rn...@apache.org>
Authored: Wed Dec 3 11:30:51 2014 +0000
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 17:43:07 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2e6d5c1a/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 0fcb0c4..17c766e 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -32,6 +32,9 @@
-export([handle_db_event/3]).
+%% exported but private
+-export([start_replication/2]).
+
-include_lib("couch/include/couch_db.hrl").
-include_lib("mem3/include/mem3.hrl").
-include("couch_replicator.hrl").
@@ -451,7 +454,7 @@ maybe_start_replication(State, DbName, DocId, RepDoc) ->
true = ets:insert(?DOC_TO_REP, {{DbName, DocId}, RepId}),
couch_log:notice("Attempting to start replication `~s` (document `~s`).",
[pp_rep_id(RepId), DocId]),
- Pid = spawn_link(fun() -> start_replication(Rep, 0) end),
+ Pid = spawn_link(?MODULE, start_replication, [Rep, 0]),
State#state{rep_start_pids = [Pid | State#state.rep_start_pids]};
#rep_state{rep = #rep{doc_id = DocId}} ->
State;
@@ -568,7 +571,7 @@ maybe_retry_replication(RepState, Error, State) ->
couch_log:error("Error in replication `~s` (triggered by document `~s`): ~s"
"~nRestarting replication in ~p seconds.",
[pp_rep_id(RepId), DocId, to_binary(error_reason(Error)), Wait]),
- Pid = spawn_link(fun() -> start_replication(Rep, Wait) end),
+ Pid = spawn_link(?MODULE, start_replication, [Rep, Wait]),
State#state{rep_start_pids = [Pid | State#state.rep_start_pids]}.
[12/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
continue jobs that aren't _replicator docs
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/ff1ab1b840019601c3e3e04a1d931db6f2ccd2d1
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9259784d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9259784d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9259784d
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 9259784d589bd27f54ae388355e0e78e5be413ac
Parents: 5084604
Author: Robert Newson <rn...@apache.org>
Authored: Thu May 21 14:08:42 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:22:11 2015 +0100
----------------------------------------------------------------------
src/couch_replicator.erl | 8 ++++----
src/couch_replicator_manager.erl | 12 +++++++-----
2 files changed, 11 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9259784d/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 3a744cd..f8a3e65 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -475,9 +475,9 @@ handle_cast({db_compacted, DbName},
{noreply, State#rep_state{target = NewTarget}};
handle_cast(checkpoint, State) ->
- #rep_state{rep_details = #rep{id = RepId} = Rep} = State,
- case couch_replicator_manager:owner(RepId) of
- Owner when Owner == node() ->
+ #rep_state{rep_details = #rep{} = Rep} = State,
+ case couch_replicator_manager:continue(Rep) of
+ {true, _} ->
case do_checkpoint(State) of
{ok, NewState} ->
couch_stats:increment_counter([couch_replicator, checkpoints, success]),
@@ -486,7 +486,7 @@ handle_cast(checkpoint, State) ->
couch_stats:increment_counter([couch_replicator, checkpoints, failure]),
{stop, Error, State}
end;
- Owner ->
+ {false, Owner} ->
couch_replicator_manager:replication_usurped(Rep, Owner),
{stop, shutdown, State}
end;
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9259784d/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 39e06b3..0961889 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -17,7 +17,7 @@
% public API
-export([replication_started/1, replication_completed/2, replication_error/2]).
--export([owner/1, replication_usurped/2]).
+-export([continue/1, replication_usurped/2]).
-export([before_doc_update/2, after_doc_read/2]).
@@ -131,9 +131,11 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
ok = gen_server:call(?MODULE, {rep_error, RepId, Error}, infinity)
end.
-
-owner(RepId) ->
- gen_server:call(?MODULE, {owner, RepId}).
+continue(#rep{doc_id = null}) ->
+ {true, no_owner};
+continue(#rep{id = RepId}) ->
+ Owner = gen_server:call(?MODULE, {owner, RepId}),
+ {node() == Owner, Owner}.
handle_config_change("replicator", "db", _, _, S) ->
@@ -178,7 +180,7 @@ init(_) ->
handle_call({owner, RepId}, _From, State) ->
case rep_state(RepId) of
nil ->
- false;
+ {reply, nonode, State};
#rep_state{dbname = DbName, rep = #rep{doc_id = DocId}} ->
{reply, owner(DbName, DocId, State#state.live), State}
end;
[10/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Be more explicit on values of ?STREAM_STATUS
Also I should add a note about how the changes ending due to a throw
when processing the last_seq leads to the un-consumed stream messages.
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/20d11c7d342ea77ffd5384d75e9cd570cbcbf5ba
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/c595e572
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/c595e572
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/c595e572
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: c595e5721a7b9483554d578e9891426ad4609f8d
Parents: 89d316e
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed May 20 17:26:59 2015 -0500
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:20:43 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_httpc.erl | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c595e572/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index e61fba5..544c2ef 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -36,7 +36,7 @@ setup(#httpdb{httpc_pool = nil, url = Url, http_connections = MaxConns} = Db) ->
send_req(HttpDb, Params1, Callback) ->
- put(STREAM_STATUS, init),
+ put(?STREAM_STATUS, init),
couch_stats:increment_counter([couch_replicator, requests]),
Params2 = ?replace(Params1, qs,
[{K, ?b2l(iolist_to_binary(V))} || {K, V} <- get_value(qs, Params1, [])]),
@@ -134,7 +134,7 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) ->
StreamDataFun = fun() ->
stream_data_self(HttpDb, Params, Worker, ReqId, Callback)
end,
- put(STREAM_STATUS, streaming),
+ put(?STREAM_STATUS, streaming),
ibrowse:stream_next(ReqId),
try
Ret = Callback(Ok, Headers, StreamDataFun),
@@ -170,22 +170,22 @@ process_stream_response(ReqId, Worker, HttpDb, Params, Callback) ->
% messages for the given ReqId on the floor since we're
% no longer in the HTTP request.
clean_mailbox({ibrowse_req_id, ReqId}) ->
- case get(STREAM_STATUS) of
+ case get(?STREAM_STATUS) of
streaming ->
ibrowse:stream_next(ReqId),
receive
{ibrowse_async_response, ReqId, _} ->
clean_mailbox({ibrowse_req_id, ReqId});
{ibrowse_async_response_end, ReqId} ->
- put(STREAM_STATUS, ended),
+ put(?STREAM_STATUS, ended),
ok
end;
- _ ->
+ Status when Status == init; Status == ended ->
receive
{ibrowse_async_response, ReqId, _} ->
clean_mailbox({ibrowse_req_id, ReqId});
{ibrowse_async_response_end, ReqId} ->
- put(STREAM_STATUS, ended),
+ put(?STREAM_STATUS, ended),
ok
end
end;
@@ -236,7 +236,7 @@ stream_data_self(#httpdb{timeout = T} = HttpDb, Params, Worker, ReqId, Cb) ->
ibrowse:stream_next(ReqId),
{Data, fun() -> stream_data_self(HttpDb, Params, Worker, ReqId, Cb) end};
{Data, ibrowse_async_response_end} ->
- put(STREAM_STATUS, ended),
+ put(?STREAM_STATUS, ended),
{Data, fun() -> throw({maybe_retry_req, more_data_expected}) end}
end.
[07/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Log when node up/down events occur
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/d40bb6f2e603a7c81f777cc1c4c200ad34c3db42
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/36757640
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/36757640
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/36757640
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 36757640ea892b0f083f6584172b3e0dc16857ad
Parents: 5dc5c41
Author: Robert Newson <rn...@apache.org>
Authored: Fri May 15 16:30:55 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:18:07 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/36757640/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index a203b9f..6410a85 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -237,10 +237,12 @@ handle_cast(Msg, State) ->
{stop, {error, {unexpected_cast, Msg}}, State}.
handle_info({nodeup, Node}, State) ->
+ couch_log:notice("Rescanning replicator dbs as ~s came up.", [Node]),
Live = lists:usort([Node | State#state.live]),
{noreply, rescan(State#state{live=Live})};
handle_info({nodedown, Node}, State) ->
+ couch_log:notice("Rescanning replicator dbs ~s went down.", [Node]),
Live = State#state.live -- [Node],
{noreply, rescan(State#state{live=Live})};
[05/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Ensure Live node set is consistent with up/down messages
BugzID: 46617
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/2418c26b0fa7cffb97c2d8348654c42d6a0f1a06
Conflicts:
src/couch_replicator_manager.erl
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/dab36db0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/dab36db0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/dab36db0
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: dab36db0c829ca391b774181cd22aa140852335c
Parents: ea39d11
Author: Robert Newson <rn...@apache.org>
Authored: Wed May 13 19:40:55 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:08:43 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/dab36db0/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 0cb0ce4..9cfda3f 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -68,7 +68,8 @@
event_listener = nil,
scan_pid = nil,
rep_start_pids = [],
- max_retries
+ max_retries,
+ live = []
}).
start_link() ->
@@ -138,6 +139,7 @@ handle_config_terminate(Self, _, _) ->
init(_) ->
process_flag(trap_exit, true),
net_kernel:monitor_nodes(true),
+ Live = [node() | nodes()],
?DOC_TO_REP = ets:new(?DOC_TO_REP, [named_table, set, public]),
?REP_TO_STATE = ets:new(?REP_TO_STATE, [named_table, set, public]),
?DB_TO_SEQ = ets:new(?DB_TO_SEQ, [named_table, set, public]),
@@ -153,7 +155,8 @@ init(_) ->
scan_pid = ScanPid,
max_retries = retries_value(
config:get("replicator", "max_replication_retry_count", "10")),
- rep_start_pids = [Pid]
+ rep_start_pids = [Pid],
+ live = Live
}}.
@@ -221,11 +224,13 @@ handle_cast(Msg, State) ->
couch_log:error("Replication manager received unexpected cast ~p", [Msg]),
{stop, {error, {unexpected_cast, Msg}}, State}.
-handle_info({nodeup, _Node}, State) ->
- {noreply, rescan(State)};
+handle_info({nodeup, Node}, State) ->
+ Live = lists:usort([Node | State#state.live]),
+ {noreply, rescan(State#state{live=Live})};
-handle_info({nodedown, _Node}, State) ->
- {noreply, rescan(State)};
+handle_info({nodedown, Node}, State) ->
+ Live = State#state.live -- [Node],
+ {noreply, rescan(State#state{live=Live})};
handle_info({'EXIT', From, normal}, #state{scan_pid = From} = State) ->
couch_log:debug("Background scan has completed.", []),
@@ -286,6 +291,8 @@ terminate(_Reason, State) ->
couch_event:stop_listener(Listener).
+code_change(1, State, _Extra) ->
+ {ok, erlang:append_element(State, [node() | nodes()])};
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@@ -381,7 +388,7 @@ rescan(#state{scan_pid = ScanPid} = State) ->
process_update(State, DbName, {Change}) ->
{RepProps} = JsonRepDoc = get_json_value(doc, Change),
DocId = get_json_value(<<"_id">>, RepProps),
- case {owner(DbName, DocId), get_json_value(deleted, Change, false)} of
+ case {owner(DbName, DocId, State#state.live), get_json_value(deleted, Change, false)} of
{_, true} ->
rep_doc_deleted(DbName, DocId),
State;
@@ -406,12 +413,11 @@ process_update(State, DbName, {Change}) ->
end
end.
-owner(<<"shards/", _/binary>> = DbName, DocId) ->
- Live = [node()|nodes()],
+owner(<<"shards/", _/binary>> = DbName, DocId, Live) ->
Nodes = lists:sort([N || #shard{node=N} <- mem3:shards(mem3:dbname(DbName), DocId),
lists:member(N, Live)]),
node() =:= hd(mem3_util:rotate_list({DbName, DocId}, Nodes));
-owner(_DbName, _DocId) ->
+owner(_DbName, _DocId, _Live) ->
true.
rep_db_update_error(Error, DbName, DocId) ->
[13/13] couch-replicator commit: updated
refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork to 367562b
Posted by mi...@apache.org.
Infinity timeout, just like all the others :(
This is a cherry-pick of:
https://github.com/cloudant/couch_replicator/commit/e947b392db1eb2de22aac4a4fa12da118fe114b3
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/367562b4
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/367562b4
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/367562b4
Branch: refs/heads/2707-merge-couch_replicator-fixes-from-cloudant-fork
Commit: 367562b4120a4ff0ba1a01dfa96534cf8d02d881
Parents: 9259784
Author: Robert Newson <rn...@apache.org>
Authored: Tue May 26 16:04:24 2015 +0100
Committer: Mike Wallace <mi...@apache.org>
Committed: Tue Jun 2 18:23:01 2015 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/367562b4/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 0961889..d29c826 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -134,7 +134,7 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
continue(#rep{doc_id = null}) ->
{true, no_owner};
continue(#rep{id = RepId}) ->
- Owner = gen_server:call(?MODULE, {owner, RepId}),
+ Owner = gen_server:call(?MODULE, {owner, RepId}, infinity),
{node() == Owner, Owner}.