You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/01/17 23:06:35 UTC
[01/50] git commit: Define LOWEST_SEQ
Updated Branches:
refs/heads/import [created] 7e6f0feb8
Define LOWEST_SEQ
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/72fec79f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/72fec79f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/72fec79f
Branch: refs/heads/import
Commit: 72fec79f05cf4f22fb173dc93c13d04e5d233b02
Parents: 5793283
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jun 13 14:59:51 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:35 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/72fec79f/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 8108e8e..60deb54 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -28,6 +28,8 @@
-include("couch_replicator_api_wrap.hrl").
-include("couch_replicator.hrl").
+-define(LOWEST_SEQ, 0).
+
-import(couch_util, [
get_value/2,
get_value/3,
[37/50] git commit: Use config app instead of couch_config
Posted by da...@apache.org.
Use config app instead of couch_config
BugzID: 13179
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9e2d76f0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9e2d76f0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9e2d76f0
Branch: refs/heads/import
Commit: 9e2d76f03c0f790e5270de4b75ba2b725de8b3cc
Parents: e59c9fb
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Feb 27 09:40:23 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 4 ++--
src/couch_replicator_utils.erl | 30 +++++++++++++++---------------
2 files changed, 17 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9e2d76f0/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index e477b29..d8d1c4a 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -121,7 +121,7 @@ init(_) ->
db_notifier = db_update_notifier(),
scan_pid = ScanPid,
max_retries = retries_value(
- couch_config:get("replicator", "max_replication_retry_count", "10"))
+ config:get("replicator", "max_replication_retry_count", "10"))
}}.
@@ -669,7 +669,7 @@ state_after_error(#rep_state{retries_left = Left, wait = Wait} = State) ->
scan_all_dbs(Server) when is_pid(Server) ->
{ok, Db} = mem3_util:ensure_exists(
- couch_config:get("mem3", "shard_db", "dbs")),
+ config:get("mem3", "shard_db", "dbs")),
ChangesFun = couch_changes:handle_changes(#changes_args{}, nil, Db),
IsReplicatorDbFun = is_replicator_db_fun(),
ChangesFun(fun({change, {Change}, _}, _) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9e2d76f0/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 53d5db7..9b9dd74 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -77,7 +77,7 @@ replication_id(#rep{user_ctx = UserCtx} = Rep, 2) ->
% TODO: we might be under an SSL socket server only, or both under
% SSL and a non-SSL socket.
% ... mochiweb_socket_server:get(https, port)
- list_to_integer(couch_config:get("httpd", "port", "5984"))
+ list_to_integer(config:get("httpd", "port", "5984"))
end,
Src = get_rep_endpoint(UserCtx, Rep#rep.source),
Tgt = get_rep_endpoint(UserCtx, Rep#rep.target),
@@ -224,14 +224,14 @@ maybe_add_trailing_slash(Url) ->
make_options(Props) ->
Options = lists:ukeysort(1, convert_options(Props)),
- DefWorkers = couch_config:get("replicator", "worker_processes", "4"),
- DefBatchSize = couch_config:get("replicator", "worker_batch_size", "500"),
- DefConns = couch_config:get("replicator", "http_connections", "20"),
- DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
- DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
- UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
+ DefWorkers = config:get("replicator", "worker_processes", "4"),
+ DefBatchSize = config:get("replicator", "worker_batch_size", "500"),
+ DefConns = config:get("replicator", "http_connections", "20"),
+ DefTimeout = config:get("replicator", "connection_timeout", "30000"),
+ DefRetries = config:get("replicator", "retries_per_request", "10"),
+ UseCheckpoints = config:get("replicator", "use_checkpoints", "true"),
{ok, DefSocketOptions} = couch_util:parse_term(
- couch_config:get("replicator", "socket_options",
+ config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
lists:ukeymerge(1, Options, lists:keysort(1, [
{connection_timeout, list_to_integer(DefTimeout)},
@@ -312,12 +312,12 @@ ssl_params(Url) ->
case ibrowse_lib:parse_url(Url) of
#url{protocol = https} ->
Depth = list_to_integer(
- couch_config:get("replicator", "ssl_certificate_max_depth", "3")
+ config:get("replicator", "ssl_certificate_max_depth", "3")
),
- VerifyCerts = couch_config:get("replicator", "verify_ssl_certificates"),
- CertFile = couch_config:get("replicator", "cert_file", nil),
- KeyFile = couch_config:get("replicator", "key_file", nil),
- Password = couch_config:get("replicator", "password", nil),
+ VerifyCerts = config:get("replicator", "verify_ssl_certificates"),
+ CertFile = config:get("replicator", "cert_file", nil),
+ KeyFile = config:get("replicator", "key_file", nil),
+ Password = config:get("replicator", "password", nil),
SslOpts = [{depth, Depth} | ssl_verify_options(VerifyCerts =:= "true")],
SslOpts1 = case CertFile /= nil andalso KeyFile /= nil of
true ->
@@ -339,12 +339,12 @@ ssl_verify_options(Value) ->
ssl_verify_options(Value, erlang:system_info(otp_release)).
ssl_verify_options(true, OTPVersion) when OTPVersion >= "R14" ->
- CAFile = couch_config:get("replicator", "ssl_trusted_certificates_file"),
+ CAFile = config:get("replicator", "ssl_trusted_certificates_file"),
[{verify, verify_peer}, {cacertfile, CAFile}];
ssl_verify_options(false, OTPVersion) when OTPVersion >= "R14" ->
[{verify, verify_none}];
ssl_verify_options(true, _OTPVersion) ->
- CAFile = couch_config:get("replicator", "ssl_trusted_certificates_file"),
+ CAFile = config:get("replicator", "ssl_trusted_certificates_file"),
[{verify, 2}, {cacertfile, CAFile}];
ssl_verify_options(false, _OTPVersion) ->
[{verify, 0}].
[18/50] git commit: Fix column widths
Posted by da...@apache.org.
Fix column widths
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/f994f35a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/f994f35a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/f994f35a
Branch: refs/heads/import
Commit: f994f35a0d95da50dbbe16b425169120ec34e0bd
Parents: c27bd42
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Aug 15 14:25:37 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 30 ++++++++++++++++++++++--------
1 file changed, 22 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/f994f35a/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index b1c7954..f8e2fda 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -61,8 +61,15 @@ init({Url, Options}) ->
{ok, State}.
-handle_call(get_worker, From, #state{waiting = Waiting, callers = Callers} = State) ->
- #state{url = Url, limit = Limit, busy = Busy, free = Free} = State,
+handle_call(get_worker, From, State) ->
+ #state{
+ waiting = Waiting,
+ callers = Callers,
+ url = Url,
+ limit = Limit,
+ busy = Busy,
+ free = Free
+ } = State,
case length(Busy) >= Limit of
true ->
{noreply, State#state{waiting = queue:in(From, Waiting)}};
@@ -83,7 +90,8 @@ handle_call(stop, _From, State) ->
{stop, normal, ok, State}.
-handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Callers} = State) ->
+handle_cast({release_worker, Worker}, State) ->
+ #state{waiting = Waiting, callers = Callers} = State,
demonitor_client(Callers, Worker),
case is_process_alive(Worker) andalso
lists:member(Worker, State#state.busy) of
@@ -108,7 +116,14 @@ handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Caller
{noreply, State}
end.
-handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = Waiting, callers = Callers} = State) ->
+handle_info({'EXIT', Pid, _Reason}, State) ->
+ #state{
+ url = Url,
+ busy = Busy,
+ free = Free,
+ waiting = Waiting,
+ callers = Callers
+ } = State,
demonitor_client(Callers, Pid),
case Free -- [Pid] of
Free ->
@@ -120,7 +135,7 @@ handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = W
{empty, _} ->
{noreply, State#state{busy = Busy2}};
{{value, From}, Waiting2} ->
- {ok, Worker} = ibrowse:spawn_link_worker_process(State#state.url),
+ {ok, Worker} = ibrowse:spawn_link_worker_process(Url),
monitor_client(Callers, Worker, From),
gen_server:reply(From, {ok, Worker}),
{noreply, State#state{busy = [Worker | Busy2], waiting = Waiting2}}
@@ -130,9 +145,8 @@ handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = W
{noreply, State#state{free = Free2}}
end;
-handle_info({'DOWN', MonitorRef, process, _Pid, _Info},
- #state{callers = Callers} = State) ->
- case ets:match(Callers,{'$1',MonitorRef}) of
+handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) ->
+ case ets:match(Callers, {'$1', Ref}) of
[] ->
{noreply, State};
[[Worker]] ->
[33/50] git commit: Scope entries to their replicator db
Posted by da...@apache.org.
Scope entries to their replicator db
Add db name to the doc id to ensure they are unique.
BugzID: 177426
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b0e99d4b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b0e99d4b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b0e99d4b
Branch: refs/heads/import
Commit: b0e99d4bc326631043274a747f0119b57a581f37
Parents: 1f99c24
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 20 23:49:29 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 29 +++++++++++++++--------------
1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b0e99d4b/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 1cfaadd..b0a9257 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -317,10 +317,10 @@ process_update(State, DbName, {Change}) ->
DocId = get_value(<<"_id">>, RepProps),
case {mem3_util:owner(DbName, DocId), get_value(deleted, Change, false)} of
{false, _} ->
- replication_complete(DocId),
+ replication_complete(DbName, DocId),
State;
{true, true} ->
- rep_doc_deleted(DocId),
+ rep_doc_deleted(DbName, DocId),
State;
{true, false} ->
case get_value(<<"_replication_state">>, RepProps) of
@@ -329,10 +329,10 @@ process_update(State, DbName, {Change}) ->
<<"triggered">> ->
maybe_start_replication(State, DbName, DocId, JsonRepDoc);
<<"completed">> ->
- replication_complete(DocId),
+ replication_complete(DbName, DocId),
State;
<<"error">> ->
- case ets:lookup(?DOC_TO_REP, DocId) of
+ case ets:lookup(?DOC_TO_REP, {DbName, DocId}) of
[] ->
maybe_start_replication(State, DbName, DocId, JsonRepDoc);
_ ->
@@ -377,7 +377,7 @@ maybe_start_replication(State, DbName, DocId, RepDoc) ->
max_retries = State#state.max_retries
},
true = ets:insert(?REP_TO_STATE, {RepId, RepState}),
- true = ets:insert(?DOC_TO_REP, {DocId, RepId}),
+ true = ets:insert(?DOC_TO_REP, {{DbName, DocId}, RepId}),
twig:log(notice,"Attempting to start replication `~s` (document `~s`).",
[pp_rep_id(RepId), DocId]),
Pid = spawn_link(fun() -> start_replication(Rep, 0) end),
@@ -427,9 +427,9 @@ start_replication(Rep, Wait) ->
replication_error(Rep, Error)
end.
-replication_complete(DocId) ->
- case ets:lookup(?DOC_TO_REP, DocId) of
- [{DocId, {BaseId, Ext} = RepId}] ->
+replication_complete(DbName, DocId) ->
+ case ets:lookup(?DOC_TO_REP, {DbName, DocId}) of
+ [{{DbName, DocId}, {BaseId, Ext} = RepId}] ->
case rep_state(RepId) of
nil ->
% Prior to OTP R14B02, temporary child specs remain in
@@ -448,18 +448,18 @@ replication_complete(DocId) ->
#rep_state{} ->
ok
end,
- true = ets:delete(?DOC_TO_REP, DocId);
+ true = ets:delete(?DOC_TO_REP, {DbName, DocId});
_ ->
ok
end.
-rep_doc_deleted(DocId) ->
- case ets:lookup(?DOC_TO_REP, DocId) of
- [{DocId, RepId}] ->
+rep_doc_deleted(DbName, DocId) ->
+ case ets:lookup(?DOC_TO_REP, {DbName, DocId}) of
+ [{{DbName, DocId}, RepId}] ->
couch_replicator:cancel_replication(RepId),
true = ets:delete(?REP_TO_STATE, RepId),
- true = ets:delete(?DOC_TO_REP, DocId),
+ true = ets:delete(?DOC_TO_REP, {DbName, DocId}),
twig:log(notice, "Stopped replication `~s` because replication document `~s`"
" was deleted", [pp_rep_id(RepId), DocId]);
[] ->
@@ -477,12 +477,13 @@ replication_error(State, RepId, Error) ->
maybe_retry_replication(#rep_state{retries_left = 0} = RepState, Error, State) ->
#rep_state{
+ dbname = DbName,
rep = #rep{id = RepId, doc_id = DocId},
max_retries = MaxRetries
} = RepState,
couch_replicator:cancel_replication(RepId),
true = ets:delete(?REP_TO_STATE, RepId),
- true = ets:delete(?DOC_TO_REP, DocId),
+ true = ets:delete(?DOC_TO_REP, {DbName, DocId}),
twig:log(error, "Error in replication `~s` (triggered by document `~s`): ~s"
"~nReached maximum retry attempts (~p).",
[pp_rep_id(RepId), DocId, to_binary(error_reason(Error)), MaxRetries]),
[27/50] git commit: Restore _replicator fault tolerance
Posted by da...@apache.org.
Restore _replicator fault tolerance
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/695f2a83
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/695f2a83
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/695f2a83
Branch: refs/heads/import
Commit: 695f2a836095fc39b1be791bf6c5b9b08f2dcc59
Parents: 7740888
Author: Robert Newson <ro...@cloudant.com>
Authored: Wed Oct 17 10:55:53 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/695f2a83/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index a6cd22d..2e5dff9 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -106,6 +106,7 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
init(_) ->
process_flag(trap_exit, true),
+ net_kernel:monitor_nodes(true),
?DOC_TO_REP = ets:new(?DOC_TO_REP, [named_table, set, protected]),
?REP_TO_STATE = ets:new(?REP_TO_STATE, [named_table, set, protected]),
?DB_TO_SEQ = ets:new(?DB_TO_SEQ, [named_table, set, protected]),
[28/50] git commit: Better handling of _replicator db deletes
Posted by da...@apache.org.
Better handling of _replicator db deletes
When a _replicator db is deleted all running replications based on that db
need to be canceled.
BugzID: 17426
BugzID: 17571
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/e59c9fb1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/e59c9fb1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/e59c9fb1
Branch: refs/heads/import
Commit: e59c9fb179d94579c5afb4e28210de40e776b0b6
Parents: b0e99d4
Author: Bob Dionne <bo...@cloudant.com>
Authored: Mon Feb 25 15:47:39 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/e59c9fb1/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index b0a9257..e477b29 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -107,9 +107,9 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
init(_) ->
process_flag(trap_exit, true),
net_kernel:monitor_nodes(true),
- ?DOC_TO_REP = ets:new(?DOC_TO_REP, [named_table, set, protected]),
- ?REP_TO_STATE = ets:new(?REP_TO_STATE, [named_table, set, protected]),
- ?DB_TO_SEQ = ets:new(?DB_TO_SEQ, [named_table, set, protected]),
+ ?DOC_TO_REP = ets:new(?DOC_TO_REP, [named_table, set, public]),
+ ?REP_TO_STATE = ets:new(?REP_TO_STATE, [named_table, set, public]),
+ ?DB_TO_SEQ = ets:new(?DB_TO_SEQ, [named_table, set, public]),
Server = self(),
ok = couch_config:register(
fun("replicator", "max_replication_retry_count", V) ->
@@ -253,7 +253,7 @@ changes_feed_loop(DbName, Since) ->
end,
{ok, Acc};
({stop, EndSeq}, Acc) ->
- ok = gen_server:call(Server, {rep_db_checkpoint, DbName, EndSeq}),
+ ok = gen_server:call(Server, {rep_db_checkpoint, DbName, EndSeq}, infinity),
{ok, Acc};
(_, Acc) ->
{ok, Acc}
@@ -285,14 +285,14 @@ db_update_notifier() ->
fun({updated, DbName}) ->
case IsReplicatorDbFun(DbName) of
true ->
- ok = gen_server:call(Server, {resume_scan, mem3:dbname(DbName)});
+ ok = gen_server:call(Server, {resume_scan, mem3:dbname(DbName)}, infinity);
_ ->
ok
end;
({deleted, DbName}) ->
case IsReplicatorDbFun(DbName) of
true ->
- ets:delete(?DB_TO_SEQ,DbName);
+ clean_up_replications(mem3:dbname(DbName));
_ ->
ok
end;
@@ -514,6 +514,18 @@ stop_all_replications() ->
true = ets:delete_all_objects(?DOC_TO_REP),
true = ets:delete_all_objects(?DB_TO_SEQ).
+clean_up_replications(DbName) ->
+ ets:foldl(
+ fun({{Name, DocId}, RepId}, _) when Name =:= DbName ->
+ couch_replicator:cancel_replication(RepId),
+ ets:delete(?DOC_TO_REP,{Name, DocId}),
+ ets:delete(?REP_TO_STATE, RepId);
+ ({_,_}, _) ->
+ ok
+ end,
+ ok, ?DOC_TO_REP),
+ ets:delete(?DB_TO_SEQ,DbName).
+
update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
{Pid, Ref} =
[22/50] git commit: fix div by zero error
Posted by da...@apache.org.
fix div by zero error
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/724bc866
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/724bc866
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/724bc866
Branch: refs/heads/import
Commit: 724bc866944a1e6cf7f79b0815bdd6a653e3233a
Parents: 1d7a371
Author: Robert Newson <ro...@cloudant.com>
Authored: Sun Sep 2 16:15:39 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/724bc866/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 5e4f6d5..4712490 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -942,7 +942,7 @@ update_task(State) ->
rep_stats(State) ++ [
{source_seq, SourceCurSeq},
case {unpack_seq(CurSeq), unpack_seq(SourceCurSeq)} of
- {0, _} ->
+ {_, 0} ->
{progress, 0};
{CurSeq1, SourceCurSeq1} ->
{progress, (CurSeq1 * 100) div SourceCurSeq1}
[17/50] git commit: Add README.md documenting usage and API
Posted by da...@apache.org.
Add README.md documenting usage and API
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/daf662fc
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/daf662fc
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/daf662fc
Branch: refs/heads/import
Commit: daf662fc86ef2b2a310e401a086754199f6c0e7f
Parents: 3937013
Author: Bob Dionne <bo...@cloudant.com>
Authored: Mon Aug 20 10:19:11 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
README.md | 200 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 200 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/daf662fc/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..376b3fc
--- /dev/null
+++ b/README.md
@@ -0,0 +1,200 @@
+## Replicator
+
+### Overview
+
+Replication is a process that synchronizes two databases. All changes
+in one database (the source), inserts, updates, and deletes, are made
+to the second database (the target). The replication can be run on
+either the node containing the source (a *push* replication), or on the
+node containing the target database (a *pull* replication) or on a
+third node different from either the source or the target.
+
+#### Background
+
+This current version is based on the latest from CouchDB as partially
+documented on the CouchDB [WIKI][1]. The first solid version that worked
+was implemented by [Adam Kocoloski][4] and subsequently picked up by [Filipe
+Manana][3], another CouchDB committer, who did two major reworkings. The
+first added the [replicator][2] db and the second some performance
+improvements and configurability to adjust factors that affect
+performance.
+
+This was then ported and integrated with dbcore. The changes required
+were modest but there are notable differences in the use of the
+replicator db as noted below. These notes are intended to be the
+definitive resource for Cloudant users and will be kept in sync with
+code changes
+
+### Usage
+
+The following examples all assume the use of the `_replicate2` end
+point. Eventualy this will be replaced by `_replicate` but this allows
+us to support both the new and old replicator during the testing
+phase, .eg.:
+
+ curl -X POST -H 'content-type:application/json'
+ http://bitdiddle.cloudant.com/_replicate2 -d
+ '{"source":http://cust1.cloudant.com/foo","target":"http://cust2.cloudant.com/bar"}'
+
+In the remaining examples we'll just note the JSON
+bodies. `_replicate2` is really all there is to calling the
+replicator, all the parameters are in the JSON body. For example a
+simple pull replication (run from the target machine):
+
+ {"source":"http://bitdiddle.cloudant.com/foo",
+ "target":"http://nordier.cloudant.com/foo"}
+To create the target db also:
+
+ {"source":"http://mazincas.cloudant.com/foo",
+ "target":"http://nordier.cloudant.com/foo",
+ "create_target": true}
+
+A continuous replication will stay running and as changes occur in the
+source, replicate them to the target. Under the covers it makes use of
+a continuous `_changes` feed:
+
+ {"source":"http://bitdiddle.cloudant.com/foo",
+ "target":"http://nordier.cloudant.com/foo",
+ "continuous":true}
+
+A replication can be stopped by posting the same body but with a
+cancel property added:
+
+ {"source":"http://bitdiddle.cloudant.com/foo",
+ "target":"http://nordier.cloudant.com/foo",
+ "continuous":true,
+ "cancel":true}
+
+When a continuous replication is run a replication id is returned,
+that provides an aditional method for cancelling the replication:
+
+ {"source":"http://bitdiddle.cloudant.com/foo",
+ "target":"http://nordier.cloudant.com/foo",
+ "continuous":true}
+
+ {"ok":true,"_local_id":"0a81b645497e6270611ec3419767a584+continuous+create_target"}
+
+To cancel:
+
+ {"replication_id":
+ "0a81b645497e6270611ec3419767a584+continuous+create_target",
+ "cancel": true}
+
+These are the main use cases. Additionally there are filter functions
+supported, JS functions that control whether a doc is replicated. They
+work similar to views, .eg.:
+
+ {
+ "_id":"_design/foo",
+ "filters": {
+ "foo": function(doc, req) {
+ if (doc.name == req.query.key) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+
+It's use is specified in the replication as follows:
+
+ {"source":" "http://nordier.cloudant.com/foo",
+ "target":"http://beezlechaus.cloudant.com/foo",
+ "filter":"foo/foo",
+ "query_params": {"key":"value"}
+ }
+
+Some applications might need to just replicate a few docs in which
+case `doc_ids` can be used:
+
+ {"source":"http://nordier.cloudant.com/foo",
+ "target":"http://mazincas.cloudant.com/foo",
+ "doc_ids":["foo","bar","baz"]
+ }
+
+### Replicator DB
+
+A replicator db is a new way to manage replications better by storing
+them in a database. One just puts a JSON doc to the replicator db with
+a body the same as if is were posted to _replicate2
+
+ curl -X PUT
+ http://bitdiddle.cloudant.com/_replicator/repl1 -d
+ '{"source":http://cust1.cloudant.com/foo",
+ "target":"http://cust2.cloudant.com/bar"}'
+
+The doc might look something like:
+
+<pre>
+{
+ "_id": "repl1",
+ "source": "http://cust1.cloudant.com/foo",
+ "target": "http://cust1.cloudant.com/foo",
+ "create_target": true,
+ "_replication_id": "c0ebe9256695ff083347cbf95f93e280",
+ "_replication_state": "triggered",
+ "_replication_state_time": "2011-06-07T16:54:35+01:00"
+}
+</pre>
+
+This replication now persists across server restarts and the
+replication can be cancelled by simply deleting the doc. The
+`_replication_state` will change to `completed` when a replication
+finishes, which may not happen in the case of a continuous
+replication. It can also change to `error` if it fails.
+
+### Advanced Configurations
+
+There are a few additional arguments that can be passed to a
+replication that govern it's performance. These are to be used with
+care as they can have signifificant impact on the system in a
+multi-tenant environment.
+
+ * worker_processes - The default is 4. This controls how many
+ separate processes will read from the changes manager and write to
+ the target. A higher number can improve throughput.
+
+ * worker_batch_size - The default is 500. This controls how many
+ documents are processed. After each batch a checkpoint is written
+ so this controls how frequently checkpointing occurs.
+
+ * http_connections - The maximum number of http connections used per
+ replication. The default is 20.
+
+ * connection_timeout - How long a connection can remain idle, the
+ default is 30000 (30s)
+
+ * retries_per_request - When requests fail the number of times to
+ retry, the default is 10. There is a wait period between each
+ attempt, that begins with 0.25s and doubles on each iteration, with
+ a cap at 5 minutes.
+
+ * socket_options - a list of options that can be used with the socket
+ connections. See the [erlang documentation][6] for details.
+
+
+### Differences from CouchDB
+
+CouchDB allows the replicator db to be shared by multiple users,
+controlling access via update handlers. It also allows the replicator
+db to be changed on the fly by setting a value in the config file. We
+support neither of these in dbcore. Each account has it's own
+replicator db, named _replicator, and that's it.
+
+Another difference to note is in specifying urls. Using the name of a
+local db works fine in a single CouchDB instance but makes less sense
+in a clustered environment. Though we try to determine a full url to
+use this doesn't work well in practice so it's a good rule of thumb to
+always use the full urls in both the source and target.
+
+### Overview of Code and Design
+
+----
+
+[1]: http://wiki.apache.org/couchdb/Replication
+[2]: https://gist.github.com/832610
+[3]: https://github.com/fdmanana
+[4]: https://github.com/kocolosk
+[5]: http://guide.couchdb.org/draft/replication.html
+[6]: http://www.erlang.org/doc/man/inet.html#setopts-2
[07/50] git commit: Tolerate JSON_ENCODE returning an iolist
Posted by da...@apache.org.
Tolerate JSON_ENCODE returning an iolist
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a2d0c429
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a2d0c429
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a2d0c429
Branch: refs/heads/import
Commit: a2d0c4290dde2015e5fb6184696fec3f89c81a4b
Parents: 815ac0a
Author: Robert Newson <ro...@cloudant.com>
Authored: Tue Jun 19 16:00:59 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a2d0c429/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 5e014b2..8a40d7c 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -466,7 +466,7 @@ options_to_query_args([revs | Rest], Acc) ->
options_to_query_args([{open_revs, all} | Rest], Acc) ->
options_to_query_args(Rest, [{"open_revs", "all"} | Acc]);
options_to_query_args([{open_revs, Revs} | Rest], Acc) ->
- JsonRevs = ?b2l(?JSON_ENCODE(couch_doc:revs_to_strs(Revs))),
+ JsonRevs = ?b2l(iolist_to_binary(?JSON_ENCODE(couch_doc:revs_to_strs(Revs)))),
options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]).
[16/50] git commit: Update db_to_seq table when _replicator db deleted
Posted by da...@apache.org.
Update db_to_seq table when _replicator db deleted
If a _replicator db is deleted then recreated the db_to_seq ets
table needs to be updated, otherwise the sequence will be out of sync
and replication docs can fail to trigger.
BugzID: 12759
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/38c2f8b6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/38c2f8b6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/38c2f8b6
Branch: refs/heads/import
Commit: 38c2f8b666079b4c7029b7d20a63a1eca6c0ea88
Parents: daf662f
Author: Bob Dionne <bo...@cloudant.com>
Authored: Wed Aug 22 18:02:21 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/38c2f8b6/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index d31f0b9..b7e7d24 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -288,6 +288,13 @@ db_update_notifier() ->
_ ->
ok
end;
+ ({deleted, DbName}) ->
+ case IsReplicatorDbFun(DbName) of
+ true ->
+ ets:delete(?DB_TO_SEQ,DbName);
+ _ ->
+ ok
+ end;
(_) ->
ok
end
[43/50] git commit: Enable clustered multipart-mime uploads
Posted by da...@apache.org.
Enable clustered multipart-mime uploads
We have to insert a middleman process into the parsing so that each of
our copies can retrieve the data as its uploaded.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/18d2da59
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/18d2da59
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/18d2da59
Branch: refs/heads/import
Commit: 18d2da5948ace3191eb8af056962de8e2ebbaff1
Parents: b6f3c98
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Mar 8 12:43:25 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:44 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/18d2da59/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 01b9b68..d072187 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -530,17 +530,18 @@ receive_docs(Streamer, UserFun, Ref, UserAcc) ->
{headers, Ref, Headers} ->
case get_value("content-type", Headers) of
{"multipart/related", _} = ContentType ->
- case doc_from_multi_part_stream(
+ case couch_doc:doc_from_multi_part_stream(
ContentType,
fun() -> receive_doc_data(Streamer, Ref) end,
Ref) of
- {ok, Doc, Parser} ->
+ {ok, Doc, WaitFun, Parser} ->
case UserFun({ok, Doc}, UserAcc) of
{ok, UserAcc2} ->
ok;
{skip, UserAcc2} ->
couch_doc:abort_multi_part_stream(Parser)
end,
+ WaitFun(),
receive_docs(Streamer, UserFun, Ref, UserAcc2)
end;
{"application/json", []} ->
[34/50] git commit: Replace heartbeat in continuous feed with timeout
Posted by da...@apache.org.
Replace heartbeat in continuous feed with timeout
BugzID: 17662
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/317131b9
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/317131b9
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/317131b9
Branch: refs/heads/import
Commit: 317131b9f7380313f1a17e90ff85b4c9c0b4793a
Parents: 7915756
Author: Bob Dionne <bo...@cloudant.com>
Authored: Wed Feb 27 09:16:55 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/317131b9/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 1b15d60..cad384a 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -321,9 +321,8 @@ update_docs(Db, DocList, Options, UpdateType) ->
{ok, bulk_results_to_errors(DocList, Result, UpdateType)}.
-changes_since(#httpdb{headers = Headers1} = HttpDb, Style, StartSeq,
- UserFun, Options) ->
- HeartBeat = erlang:max(1000, HttpDb#httpdb.timeout div 3),
+changes_since(#httpdb{headers = Headers1, timeout = Timeout} = HttpDb,
+ Style, StartSeq, UserFun, Options) ->
BaseQArgs = case get_value(continuous, Options, false) of
false ->
[{"feed", "normal"}];
@@ -331,8 +330,8 @@ changes_since(#httpdb{headers = Headers1} = HttpDb, Style, StartSeq,
[{"feed", "continuous"}]
end ++ [
{"style", atom_to_list(Style)}, {"since", ?JSON_ENCODE(StartSeq)},
- {"heartbeat", integer_to_list(HeartBeat)}
- ],
+ {"timeout", integer_to_list(Timeout)}
+ ],
DocIds = get_value(doc_ids, Options),
{QArgs, Method, Body, Headers} = case DocIds of
undefined ->
[25/50] git commit: simplify regex
Posted by da...@apache.org.
simplify regex
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7740888a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7740888a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7740888a
Branch: refs/heads/import
Commit: 7740888a8784807dd0b485cad5a205e9e7feee8c
Parents: 724bc86
Author: Robert Newson <ro...@cloudant.com>
Authored: Wed Sep 5 17:46:28 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7740888a/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 4712490..8ebbc6e 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -969,7 +969,7 @@ unpack_seq(Seq) when is_number(Seq) ->
unpack_seq([SeqNum, _]) ->
SeqNum;
unpack_seq(Seq) when is_binary(Seq) ->
- Pattern = "^[\\[?]?(?<seqnum>[0-9]+)",
+ Pattern = "^\\[?(?<seqnum>[0-9]+)",
Options = [{capture, [seqnum], list}],
{match, [SeqNum]} = re:run(Seq, Pattern, Options),
list_to_integer(SeqNum).
[41/50] git commit: Remove couch_ref_counter and use process monitors
Posted by da...@apache.org.
Remove couch_ref_counter and use process monitors
This patch removes the ref counting code that used process links and a
gen_server to manage when databases are idle. This is important ground
work for the refactoring of couch_server which relies on having a sane
interface to managing database.
This code has two major parts. The first part removes the gen_server
from couch_db.erl and instead relies on checking who's monitoring the
database's couch_file gen_server. The database is idle when this list
only includes the updater and possibly the couch_stats_collector which
monitors open databases.
The second half is how couch_file's are managed by listening for who is
monitoring them. If they're monitored by only zero or one process then
they close themselves. This allows for the semantics of staying open
long enough to handle any current connections while being replaced with
a new couch_file (ie, due to compaction).
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9ef3d133
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9ef3d133
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9ef3d133
Branch: refs/heads/import
Commit: 9ef3d133c5173a0545712a9bde71970401b03a81
Parents: 0e9e5b6
Author: Paul Joseph Davis <da...@apache.org>
Authored: Tue Mar 20 16:48:14 2012 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:42 2013 -0500
----------------------------------------------------------------------
test/03-replication-compact.t | 41 +++++++++++++++++++++++++-------------
1 file changed, 27 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9ef3d133/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index e3f59fc..003b04c 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -26,11 +26,10 @@
-record(db, {
main_pid = nil,
- update_pid = nil,
compactor_pid = nil,
instance_start_time, % number of microsecs since jan 1 1970 as a binary string
fd,
- fd_ref_counter,
+ fd_monitor,
header = nil,
committed_update_seq,
fulldocinfo_by_id_btree,
@@ -138,13 +137,13 @@ populate_and_compact_test(RepPid, SourceDb0, TargetDb0) ->
etap:is(is_process_alive(RepPid), true,
"Replication process is alive after source database compaction"),
check_db_alive("source", SourceDb),
- check_ref_counter("source", SourceDb),
+ check_fd("source", SourceDb),
compact_db("target", TargetDb),
etap:is(is_process_alive(RepPid), true,
"Replication process is alive after target database compaction"),
check_db_alive("target", TargetDb),
- check_ref_counter("target", TargetDb),
+ check_fd("target", TargetDb),
{ok, SourceDb2} = reopen_db(SourceDb),
{ok, TargetDb2} = reopen_db(TargetDb),
@@ -157,7 +156,7 @@ populate_and_compact_test(RepPid, SourceDb0, TargetDb0) ->
"Replication process is alive after source database compaction"),
check_db_alive("source", SourceDb2),
pause_writer(Writer),
- check_ref_counter("source", SourceDb2),
+ check_fd("source", SourceDb2),
resume_writer(Writer),
compact_db("target", TargetDb2),
@@ -165,7 +164,7 @@ populate_and_compact_test(RepPid, SourceDb0, TargetDb0) ->
"Replication process is alive after target database compaction"),
check_db_alive("target", TargetDb2),
pause_writer(Writer),
- check_ref_counter("target", TargetDb2),
+ check_fd("target", TargetDb2),
resume_writer(Writer),
{ok, SourceDb3} = reopen_db(SourceDb2),
@@ -200,18 +199,32 @@ compact_db(Type, #db{name = Name}) ->
ok = couch_db:close(Db).
-check_ref_counter(Type, #db{name = Name, fd_ref_counter = OldRefCounter}) ->
- MonRef = erlang:monitor(process, OldRefCounter),
- receive
- {'DOWN', MonRef, process, OldRefCounter, _} ->
- etap:diag("Old " ++ Type ++ " database ref counter terminated")
+check_fd(Type, #db{name = Name, fd = Fd, fd_monitor=OldMonRef}) ->
+ {_, MonRef} = spawn_monitor(fun() ->
+ MC = fun(F) ->
+ % Speed up the close after the switch
+ Fd ! maybe_close,
+ {ok, Db} = couch_db:open_int(Name, []),
+ couch_db:close(Db),
+ case Db#db.fd of
+ Fd ->
+ receive _ -> ok after 1000 -> ok end,
+ F(F);
+ _ ->
+ ok
+ end
+ end,
+ MC(MC)
+ end),
+ receive {'DOWN', MonRef, process, _, _} ->
+ etap:diag("Old " ++ Type ++ " database fd terminated")
after 30000 ->
- etap:bail("Old " ++ Type ++ " database ref counter didn't terminate")
+ etap:bail("Old " ++ Type ++ " database fd didn't terminate")
end,
- {ok, #db{fd_ref_counter = NewRefCounter} = Db} = couch_db:open_int(Name, []),
+ {ok, #db{fd_monitor = NewMonRef} = Db} = couch_db:open_int(Name, []),
ok = couch_db:close(Db),
etap:isnt(
- NewRefCounter, OldRefCounter, Type ++ " database has new ref counter").
+ NewMonRef, OldMonRef, Type ++ " database has new fd monitor").
reopen_db(#db{name = Name}) ->
[09/50] git commit: Notify replication manager when replication is
shutdown
Posted by da...@apache.org.
Notify replication manager when replication is shutdown
BugzID: 14273
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/18cf011f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/18cf011f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/18cf011f
Branch: refs/heads/import
Commit: 18cf011fa5e7162c6e93ef2c4b3508ef0086953b
Parents: b57bbbe
Author: Bob Dionne <bo...@cloudant.com>
Authored: Tue Aug 14 05:50:25 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/18cf011f/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index a1148cc..b04506c 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -495,7 +495,8 @@ terminate(shutdown, {error, Class, Error, Stack, InitArgs}) ->
_ ->
NotifyError = Error
end,
- couch_replicator_notifier:notify({error, RepId, NotifyError}).
+ couch_replicator_notifier:notify({error, RepId, NotifyError}),
+ couch_replicator_manager:replication_error(InitArgs, NotifyError).
terminate_cleanup(State) ->
update_task(State),
[23/50] git commit: Include user info
Posted by da...@apache.org.
Include user info
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a2756e19
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a2756e19
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a2756e19
Branch: refs/heads/import
Commit: a2756e195cd1ba9a503147aae855d2b3e398d5ec
Parents: 9a30604
Author: Robert Newson <ro...@cloudant.com>
Authored: Thu Aug 30 17:44:30 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a2756e19/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index b04506c..95e5241 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -228,7 +228,7 @@ cancel_replication(RepId, #user_ctx{name = Name, roles = Roles}) ->
init(InitArgs) ->
{ok, InitArgs, 0}.
-do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
+do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx=UserCtx} = Rep) ->
process_flag(trap_exit, true),
#rep_state{
@@ -267,6 +267,7 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
couch_task_status:add_task([
{type, replication},
+ {user, UserCtx#user_ctx.name},
{replication_id, ?l2b(BaseId ++ Ext)},
{doc_id, Rep#rep.doc_id},
{source, ?l2b(SourceName)},
[44/50] git commit: Major change to use the new config app
Posted by da...@apache.org.
Major change to use the new config app
This replaces couch_config with the config app. The config app is mostly
a direct port of couch_config to a new namespace with the addition of a
rewritten config change notification mechanism. The new change mechanism
removes the ability to register an anonymous function that breaks code
upgrades and generally cleans up the various listening patterns using a
new behavior definition.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9f968dc3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9f968dc3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9f968dc3
Branch: refs/heads/import
Commit: 9f968dc3e7c30c3e4d7837aaec85780a3765f1e6
Parents: 18d2da5
Author: Robert Newson <rn...@apache.org>
Authored: Sat Mar 9 18:22:19 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:53 2013 -0500
----------------------------------------------------------------------
test/02-httpc-pool.t | 4 ++--
test/03-replication-compact.t | 2 +-
test/04-replication-large-atts.t | 4 ++--
test/05-replication-many-leaves.t | 4 ++--
test/06-doc-missing-stubs.t | 2 +-
5 files changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9f968dc3/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index a7bde6c..23d7ab7 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -239,8 +239,8 @@ loop(Parent, Ref, Worker, Pool) ->
spawn_pool() ->
- Host = couch_config:get("httpd", "bind_address", "127.0.0.1"),
- Port = couch_config:get("httpd", "port", "5984"),
+ Host = config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = config:get("httpd", "port", "5984"),
{ok, Pool} = couch_replicator_httpc_pool:start_link(
"http://" ++ Host ++ ":5984", [{max_connections, 3}]),
Pool.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9f968dc3/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 8d42d46..619d2b4 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -444,7 +444,7 @@ maybe_pause(Parent, Counter) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9f968dc3/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index 5386179..caf2f67 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -66,7 +66,7 @@ test() ->
couch_server_sup:start_link(test_util:config_files()),
ibrowse:start(),
crypto:start(),
- couch_config:set("attachments", "compressible_types", "text/*", false),
+ config:set("attachments", "compressible_types", "text/*", false),
Pairs = [
{source_db_name(), target_db_name()},
@@ -227,7 +227,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9f968dc3/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index 212ee99..be63d41 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -78,7 +78,7 @@ test() ->
couch_server_sup:start_link(test_util:config_files()),
ibrowse:start(),
crypto:start(),
- couch_config:set("replicator", "connection_timeout", "90000", false),
+ config:set("replicator", "connection_timeout", "90000", false),
Pairs = [
{source_db_name(), target_db_name()},
@@ -254,7 +254,7 @@ add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9f968dc3/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index e17efc9..0fffec2 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -264,7 +264,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
- "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ "http://", config:get("httpd", "bind_address", "127.0.0.1"),
":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
"/", DbName
]).
[40/50] git commit: Shorter internal btree names in records
Posted by da...@apache.org.
Shorter internal btree names in records
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b6f3c988
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b6f3c988
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b6f3c988
Branch: refs/heads/import
Commit: b6f3c9888a18b97899e75c00a672abac607527bf
Parents: 9ef3d13
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu Mar 7 13:37:03 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:42 2013 -0500
----------------------------------------------------------------------
test/03-replication-compact.t | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b6f3c988/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 003b04c..8d42d46 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -32,9 +32,9 @@
fd_monitor,
header = nil,
committed_update_seq,
- fulldocinfo_by_id_btree,
- docinfo_by_seq_btree,
- local_docs_btree,
+ id_tree,
+ seq_tree,
+ local_tree,
update_seq,
name,
filepath,
[08/50] git commit: Rework monitor funs so they're symmetrical
Posted by da...@apache.org.
Rework monitor funs so they're symmetrical
Also, accept a Client tuple directly to simplify the API a bit.
BugzID: 14132
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/c27bd42f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/c27bd42f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/c27bd42f
Branch: refs/heads/import
Commit: c27bd42f8da0cdfb57b29f9774cdeae145377eba
Parents: 18cf011
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Aug 15 14:09:20 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 27 +++++++++++++--------------
1 file changed, 13 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c27bd42f/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index 75d8d5f..b1c7954 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -61,7 +61,7 @@ init({Url, Options}) ->
{ok, State}.
-handle_call(get_worker, {FromPid, _} = From, #state{waiting = Waiting, callers = Callers} = State) ->
+handle_call(get_worker, From, #state{waiting = Waiting, callers = Callers} = State) ->
#state{url = Url, limit = Limit, busy = Busy, free = Free} = State,
case length(Busy) >= Limit of
true ->
@@ -74,7 +74,7 @@ handle_call(get_worker, {FromPid, _} = From, #state{waiting = Waiting, callers =
[Worker | Free2] ->
ok
end,
- monitor_use(FromPid, Worker, Callers),
+ monitor_client(Callers, Worker, From),
NewState = State#state{free = Free2, busy = [Worker | Busy]},
{reply, {ok, Worker}, NewState}
end;
@@ -84,7 +84,7 @@ handle_call(stop, _From, State) ->
handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Callers} = State) ->
- demonitor_worker(Callers,Worker),
+ demonitor_client(Callers, Worker),
case is_process_alive(Worker) andalso
lists:member(Worker, State#state.busy) of
true ->
@@ -92,8 +92,8 @@ handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Caller
{empty, Waiting2} ->
Busy2 = State#state.busy -- [Worker],
Free2 = [Worker | State#state.free];
- {{value, {Pid, _} = From}, Waiting2} ->
- monitor_use(Pid, Worker, Callers),
+ {{value, From}, Waiting2} ->
+ monitor_client(Callers, Worker, From),
gen_server:reply(From, {ok, Worker}),
Busy2 = State#state.busy,
Free2 = State#state.free
@@ -109,7 +109,7 @@ handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Caller
end.
handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = Waiting, callers = Callers} = State) ->
- demonitor_worker(Callers,Pid),
+ demonitor_client(Callers, Pid),
case Free -- [Pid] of
Free ->
case Busy -- [Pid] of
@@ -119,9 +119,9 @@ handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = W
case queue:out(Waiting) of
{empty, _} ->
{noreply, State#state{busy = Busy2}};
- {{value, {FromPid, _Ref} = From}, Waiting2} ->
+ {{value, From}, Waiting2} ->
{ok, Worker} = ibrowse:spawn_link_worker_process(State#state.url),
- monitor_use(FromPid, Worker, Callers),
+ monitor_client(Callers, Worker, From),
gen_server:reply(From, {ok, Worker}),
{noreply, State#state{busy = [Worker | Busy2], waiting = Waiting2}}
end
@@ -147,14 +147,13 @@ terminate(_Reason, State) ->
lists:foreach(fun ibrowse_http_client:stop/1, State#state.free),
lists:foreach(fun ibrowse_http_client:stop/1, State#state.busy).
-monitor_use(Pid, Worker, Callers) ->
- MonitorRef = erlang:monitor(process, Pid),
- ets:insert(Callers, {Worker, MonitorRef}).
+monitor_client(Tab, Worker, {ClientPid, _}) ->
+ ets:insert(Tab, {Worker, erlang:monitor(process, ClientPid)}).
-demonitor_worker(Callers, Worker) ->
- case ets:lookup(Callers,Worker) of
+demonitor_client(Tab, Worker) ->
+ case ets:lookup(Tab, Worker) of
[] -> ok;
[{Worker, MonRef}] ->
- ets:delete(Callers,Worker),
+ ets:delete(Tab, Worker),
erlang:demonitor(MonRef)
end.
[20/50] git commit: Fix code_change function
Posted by da...@apache.org.
Fix code_change function
BugzID: 14132
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/50f1edb1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/50f1edb1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/50f1edb1
Branch: refs/heads/import
Commit: 50f1edb16006ff66bc32fada9b771f4465cb6635
Parents: f994f35
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Aug 15 14:29:03 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/50f1edb1/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index f8e2fda..ddfb94b 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -153,8 +153,10 @@ handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) ->
handle_cast({release_worker, Worker}, State)
end.
+code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 6 ->
+ {ok, erlang:append_element(OldState, ets:new(callers))};
code_change(_OldVsn, State, _Extra) ->
- {ok, State#state{callers = ets:new(callers,[set])}}.
+ {ok, State}.
terminate(_Reason, State) ->
[36/50] git commit: Remove entries from the callers list properly
Posted by da...@apache.org.
Remove entries from the callers list properly
A standard SSA bug using the wrong version of a variable was preventing
the removal of {Worker, Ref} entries in the callers list. This corrects
the issue by using the right variable version.
BugzId: 17731
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/d17aced7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/d17aced7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/d17aced7
Branch: refs/heads/import
Commit: d17aced77fc0bcde3c45bd4c89132b143308bbee
Parents: a18f8e9
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Mar 1 15:00:29 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/d17aced7/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index cb72d42..0a42284 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -105,7 +105,7 @@ handle_cast({release_worker, Worker}, State) ->
Busy2 = State#state.busy -- [Worker],
Free2 = [Worker | State#state.free];
{{value, From}, Waiting2} ->
- NewCallers1 = monitor_client(Callers, Worker, From),
+ NewCallers1 = monitor_client(NewCallers0, Worker, From),
gen_server:reply(From, {ok, Worker}),
Busy2 = State#state.busy,
Free2 = State#state.free
@@ -141,7 +141,7 @@ handle_info({'EXIT', Pid, _Reason}, State) ->
{noreply, State#state{busy = Busy2, callers = NewCallers0}};
{{value, From}, Waiting2} ->
{ok, Worker} = ibrowse:spawn_link_worker_process(Url),
- NewCallers1 = monitor_client(Callers, Worker, From),
+ NewCallers1 = monitor_client(NewCallers0, Worker, From),
gen_server:reply(From, {ok, Worker}),
NewState = State#state{
busy = [Worker | Busy2],
[06/50] git commit: Parse replication doc inside replicator
Posted by da...@apache.org.
Parse replication doc inside replicator
This is to ensure that gethostname is run on the node running
the replication, as it becomes part of the replication id
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/c3431265
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/c3431265
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/c3431265
Branch: refs/heads/import
Commit: c34312659db13d8f4320464d01dc2fafa35a3ebc
Parents: a2d0c42
Author: Bob Dionne <bo...@cloudant.com>
Authored: Wed Jul 4 07:18:25 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 6 ++++--
src/couch_replicator_httpd.erl | 3 +--
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c3431265/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 60deb54..d02f5e6 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -14,7 +14,7 @@
-behaviour(gen_server).
% public API
--export([replicate/1]).
+-export([replicate/2]).
% meant to be used only by the replicator database listener
-export([async_replicate/1]).
@@ -74,7 +74,9 @@
}).
-replicate(#rep{id = RepId, options = Options, user_ctx = UserCtx} = Rep) ->
+replicate(PostBody, Ctx) ->
+ {ok, #rep{id = RepId, options = Options, user_ctx = UserCtx} = Rep} =
+ couch_replicator_utils:parse_rep_doc(PostBody, Ctx),
case get_value(cancel, Options, false) of
true ->
case get_value(id, Options, nil) of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c3431265/src/couch_replicator_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpd.erl b/src/couch_replicator_httpd.erl
index f225e82..f84152c 100644
--- a/src/couch_replicator_httpd.erl
+++ b/src/couch_replicator_httpd.erl
@@ -31,8 +31,7 @@ handle_req(#httpd{method = 'POST', user_ctx = UserCtx} = Req) ->
couch_httpd:validate_ctype(Req, "application/json"),
RepDoc = {Props} = couch_httpd:json_body_obj(Req),
validate_rep_props(Props),
- {ok, Rep} = couch_replicator_utils:parse_rep_doc(RepDoc, UserCtx),
- case couch_replicator:replicate(Rep) of
+ case couch_replicator:replicate(RepDoc, UserCtx) of
{error, {Error, Reason}} ->
send_json(
Req, 404,
[30/50] git commit: Remove dynamic ets tables from the httpc pool
Posted by da...@apache.org.
Remove dynamic ets tables from the httpc pool
This is a pretty straightforward swap to using the lists:key* BIFs
instead of an ets table. The number of HTTP clients is not expected to
ever exceed roughly 100 so this keeps the table size inside the
acceptable limits for a list.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/17671fad
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/17671fad
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/17671fad
Branch: refs/heads/import
Commit: 17671fadc4498f8b73569d89f19247e493979446
Parents: 79df0f2
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 20 23:38:37 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 77 ++++++++++++++++++++------------
1 file changed, 48 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/17671fad/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index b9d4a43..cb72d42 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -32,8 +32,8 @@
limit, % max # of workers allowed
free = [], % free workers (connections)
busy = [], % busy workers (connections)
- waiting = queue:new(), % blocked clients waiting for a worker
- callers = ets:new(callers,[set]) % clients who've been given a worker
+ waiting = queue:new(), % blocked clients waiting for a worker
+ callers = [] % clients who've been given a worker
}).
@@ -81,8 +81,11 @@ handle_call(get_worker, From, State) ->
[Worker | Free2] ->
ok
end,
- monitor_client(Callers, Worker, From),
- NewState = State#state{free = Free2, busy = [Worker | Busy]},
+ NewState = State#state{
+ free = Free2,
+ busy = [Worker | Busy],
+ callers = monitor_client(Callers, Worker, From)
+ },
{reply, {ok, Worker}, NewState}
end;
@@ -92,16 +95,17 @@ handle_call(stop, _From, State) ->
handle_cast({release_worker, Worker}, State) ->
#state{waiting = Waiting, callers = Callers} = State,
- demonitor_client(Callers, Worker),
+ NewCallers0 = demonitor_client(Callers, Worker),
case is_process_alive(Worker) andalso
lists:member(Worker, State#state.busy) of
true ->
case queue:out(Waiting) of
{empty, Waiting2} ->
+ NewCallers1 = NewCallers0,
Busy2 = State#state.busy -- [Worker],
Free2 = [Worker | State#state.free];
{{value, From}, Waiting2} ->
- monitor_client(Callers, Worker, From),
+ NewCallers1 = monitor_client(Callers, Worker, From),
gen_server:reply(From, {ok, Worker}),
Busy2 = State#state.busy,
Free2 = State#state.free
@@ -109,11 +113,12 @@ handle_cast({release_worker, Worker}, State) ->
NewState = State#state{
busy = Busy2,
free = Free2,
- waiting = Waiting2
+ waiting = Waiting2,
+ callers = NewCallers1
},
{noreply, NewState};
false ->
- {noreply, State}
+ {noreply, State#state{callers = NewCallers0}}
end.
handle_info({'EXIT', Pid, _Reason}, State) ->
@@ -124,37 +129,50 @@ handle_info({'EXIT', Pid, _Reason}, State) ->
waiting = Waiting,
callers = Callers
} = State,
- demonitor_client(Callers, Pid),
+ NewCallers0 = demonitor_client(Callers, Pid),
case Free -- [Pid] of
Free ->
case Busy -- [Pid] of
Busy ->
- {noreply, State};
+ {noreply, State#state{callers = NewCallers0}};
Busy2 ->
case queue:out(Waiting) of
{empty, _} ->
- {noreply, State#state{busy = Busy2}};
+ {noreply, State#state{busy = Busy2, callers = NewCallers0}};
{{value, From}, Waiting2} ->
{ok, Worker} = ibrowse:spawn_link_worker_process(Url),
- monitor_client(Callers, Worker, From),
+ NewCallers1 = monitor_client(Callers, Worker, From),
gen_server:reply(From, {ok, Worker}),
- {noreply, State#state{busy = [Worker | Busy2], waiting = Waiting2}}
+ NewState = State#state{
+ busy = [Worker | Busy2],
+ waiting = Waiting2,
+ callers = NewCallers1
+ },
+ {noreply, NewState}
end
end;
Free2 ->
- {noreply, State#state{free = Free2}}
+ {noreply, State#state{free = Free2, callers = NewCallers0}}
end;
handle_info({'DOWN', Ref, process, _, _}, #state{callers = Callers} = State) ->
- case ets:match(Callers, {'$1', Ref}) of
- [] ->
- {noreply, State};
- [[Worker]] ->
- handle_cast({release_worker, Worker}, State)
+ case lists:keysearch(Ref, 2, Callers) of
+ {value, {Worker, Ref}} ->
+ handle_cast({release_worker, Worker}, State);
+ false ->
+ {noreply, State}
end.
-code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 6 ->
- {ok, erlang:append_element(OldState, ets:new(callers))};
+code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 7 ->
+ case element(7, OldState) of
+ EtsTable when is_integer(EtsTable) ->
+ NewState = setelement(7, OldState, ets:tab2list(EtsTable)),
+ ets:delete(EtsTable),
+ {ok, NewState};
+ Callers when is_list(Callers) ->
+ % Already upgraded
+ {ok, OldState}
+ end;
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@@ -163,13 +181,14 @@ terminate(_Reason, State) ->
lists:foreach(fun ibrowse_http_client:stop/1, State#state.free),
lists:foreach(fun ibrowse_http_client:stop/1, State#state.busy).
-monitor_client(Tab, Worker, {ClientPid, _}) ->
- ets:insert(Tab, {Worker, erlang:monitor(process, ClientPid)}).
+monitor_client(Callers, Worker, {ClientPid, _}) ->
+ [{Worker, erlang:monitor(process, ClientPid)} | Callers].
-demonitor_client(Tab, Worker) ->
- case ets:lookup(Tab, Worker) of
- [] -> ok;
- [{Worker, MonRef}] ->
- ets:delete(Tab, Worker),
- erlang:demonitor(MonRef, [flush])
+demonitor_client(Callers, Worker) ->
+ case lists:keysearch(Worker, 1, Callers) of
+ {value, {Worker, MonRef}} ->
+ erlang:demonitor(MonRef, [flush]),
+ lists:keydelete(Worker, 1, Callers);
+ false ->
+ Callers
end.
[15/50] git commit: Use longpoll rather than normal for changes feed
on _replicator db
Posted by da...@apache.org.
Use longpoll rather than normal for changes feed on _replicator db
This helps prevent the case where changes to the _replicator db have not
fully propagated, resuting in an empty changes feed and a stuck replication
BugzID: 12759
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9a306045
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9a306045
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9a306045
Branch: refs/heads/import
Commit: 9a306045857da6a7170b3930abbc61f9eb8fd4e5
Parents: 38c2f8b
Author: Bob Dionne <bo...@cloudant.com>
Authored: Wed Aug 22 18:02:59 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9a306045/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index b7e7d24..a6cd22d 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -260,7 +260,7 @@ changes_feed_loop(DbName, Since) ->
nil,
#changes_args{
include_docs = true,
- feed = "normal",
+ feed = "longpoll",
since = Since,
filter = main_only,
timeout = infinity
[02/50] git commit: Refactor couch_replicator_httpc_pool to monitor
processes that have requested workers
Posted by da...@apache.org.
Refactor couch_replicator_httpc_pool to monitor processes that have requested workers
The caller pids are now monitored and mapped to the workers they receive using
an ets table in the gen_server state. When caller processes go down the workers
are reclaimed.
BugzID: 14132
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b35a5b9c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b35a5b9c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b35a5b9c
Branch: refs/heads/import
Commit: b35a5b9c54fbc4af4be1ec21ab682f9fed6ccab5
Parents: df6057f
Author: Bob Dionne <bo...@cloudant.com>
Authored: Sun Jul 29 08:50:57 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 45 ++++++++++++++++++++++++--------
1 file changed, 34 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b35a5b9c/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index cf596a1..75d8d5f 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -32,14 +32,14 @@
limit, % max # of workers allowed
free = [], % free workers (connections)
busy = [], % busy workers (connections)
- waiting = queue:new() % blocked clients waiting for a worker
+ waiting = queue:new(), % blocked clients waiting for a worker
+ callers = ets:new(callers,[set]) % clients who've been given a worker
}).
start_link(Url, Options) ->
gen_server:start_link(?MODULE, {Url, Options}, []).
-
stop(Pool) ->
ok = gen_server:call(Pool, stop, infinity).
@@ -61,7 +61,7 @@ init({Url, Options}) ->
{ok, State}.
-handle_call(get_worker, From, #state{waiting = Waiting} = State) ->
+handle_call(get_worker, {FromPid, _} = From, #state{waiting = Waiting, callers = Callers} = State) ->
#state{url = Url, limit = Limit, busy = Busy, free = Free} = State,
case length(Busy) >= Limit of
true ->
@@ -74,6 +74,7 @@ handle_call(get_worker, From, #state{waiting = Waiting} = State) ->
[Worker | Free2] ->
ok
end,
+ monitor_use(FromPid, Worker, Callers),
NewState = State#state{free = Free2, busy = [Worker | Busy]},
{reply, {ok, Worker}, NewState}
end;
@@ -82,7 +83,8 @@ handle_call(stop, _From, State) ->
{stop, normal, ok, State}.
-handle_cast({release_worker, Worker}, #state{waiting = Waiting} = State) ->
+handle_cast({release_worker, Worker}, #state{waiting = Waiting, callers = Callers} = State) ->
+ demonitor_worker(Callers,Worker),
case is_process_alive(Worker) andalso
lists:member(Worker, State#state.busy) of
true ->
@@ -90,7 +92,8 @@ handle_cast({release_worker, Worker}, #state{waiting = Waiting} = State) ->
{empty, Waiting2} ->
Busy2 = State#state.busy -- [Worker],
Free2 = [Worker | State#state.free];
- {{value, From}, Waiting2} ->
+ {{value, {Pid, _} = From}, Waiting2} ->
+ monitor_use(Pid, Worker, Callers),
gen_server:reply(From, {ok, Worker}),
Busy2 = State#state.busy,
Free2 = State#state.free
@@ -105,33 +108,53 @@ handle_cast({release_worker, Worker}, #state{waiting = Waiting} = State) ->
{noreply, State}
end.
-
-handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free} = State) ->
+handle_info({'EXIT', Pid, _Reason}, #state{busy = Busy, free = Free, waiting = Waiting, callers = Callers} = State) ->
+ demonitor_worker(Callers,Pid),
case Free -- [Pid] of
Free ->
case Busy -- [Pid] of
Busy ->
{noreply, State};
Busy2 ->
- case queue:out(State#state.waiting) of
+ case queue:out(Waiting) of
{empty, _} ->
{noreply, State#state{busy = Busy2}};
- {{value, From}, Waiting2} ->
+ {{value, {FromPid, _Ref} = From}, Waiting2} ->
{ok, Worker} = ibrowse:spawn_link_worker_process(State#state.url),
+ monitor_use(FromPid, Worker, Callers),
gen_server:reply(From, {ok, Worker}),
{noreply, State#state{busy = [Worker | Busy2], waiting = Waiting2}}
end
end;
Free2 ->
{noreply, State#state{free = Free2}}
- end.
+ end;
+handle_info({'DOWN', MonitorRef, process, _Pid, _Info},
+ #state{callers = Callers} = State) ->
+ case ets:match(Callers,{'$1',MonitorRef}) of
+ [] ->
+ {noreply, State};
+ [[Worker]] ->
+ handle_cast({release_worker, Worker}, State)
+ end.
code_change(_OldVsn, State, _Extra) ->
- {ok, State}.
+ {ok, State#state{callers = ets:new(callers,[set])}}.
terminate(_Reason, State) ->
lists:foreach(fun ibrowse_http_client:stop/1, State#state.free),
lists:foreach(fun ibrowse_http_client:stop/1, State#state.busy).
+monitor_use(Pid, Worker, Callers) ->
+ MonitorRef = erlang:monitor(process, Pid),
+ ets:insert(Callers, {Worker, MonitorRef}).
+
+demonitor_worker(Callers, Worker) ->
+ case ets:lookup(Callers,Worker) of
+ [] -> ok;
+ [{Worker, MonRef}] ->
+ ets:delete(Callers,Worker),
+ erlang:demonitor(MonRef)
+ end.
[05/50] git commit: Support either atom or binary type for deleted
field in changes
Posted by da...@apache.org.
Support either atom or binary type for deleted field in changes
This was changed in couchdb recently and either format needs to be handled
for the time being.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/df6057fb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/df6057fb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/df6057fb
Branch: refs/heads/import
Commit: df6057fb3130907fc829810bd0ff677be474f65c
Parents: c343126
Author: Bob Dionne <bo...@cloudant.com>
Authored: Thu Jul 5 06:02:56 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 +-
src/couch_replicator_manager.erl | 2 +-
src/couch_replicator_utils.erl | 11 ++++++++++-
3 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/df6057fb/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 8a40d7c..a2aa8a0 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -693,7 +693,7 @@ json_to_doc_info({Props}) ->
RevsInfo = lists:map(
fun({Change}) ->
Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)),
- Del = (true =:= get_value(<<"deleted">>, Change)),
+ Del = couch_replicator_utils:is_deleted(Change),
#rev_info{rev=Rev, deleted=Del}
end, get_value(<<"changes">>, Props)),
#doc_info{
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/df6057fb/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index f02a56b..a160f51 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -645,7 +645,7 @@ scan_all_dbs(Server) when is_pid(Server) ->
ChangesFun(fun({change, {Change}, _}, _) ->
DbName = couch_util:get_value(<<"id">>, Change),
case DbName of <<"_design/", _/binary>> -> ok; _Else ->
- case couch_util:get_value(<<"deleted">>, Change, false) of
+ case couch_replicator_utils:is_deleted(Change) of
true ->
ok;
false ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/df6057fb/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index dfc0244..75f79b9 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -16,7 +16,7 @@
-export([open_db/1, close_db/1]).
-export([start_db_compaction_notifier/2, stop_db_compaction_notifier/1]).
-export([replication_id/2]).
--export([sum_stats/2]).
+-export([sum_stats/2, is_deleted/1]).
-export([mp_parse_doc/2]).
-include_lib("couch/include/couch_db.hrl").
@@ -417,3 +417,12 @@ mp_parse_atts({body, Bytes}) ->
fun mp_parse_atts/1;
mp_parse_atts(body_end) ->
fun mp_parse_atts/1.
+
+is_deleted(Change) ->
+ case couch_util:get_value(<<"deleted">>, Change) of
+ undefined ->
+ % keep backwards compatibility for a while
+ couch_util:get_value(deleted, Change, false);
+ Else ->
+ Else
+ end.
[21/50] git commit: Allow the use of checkpoint to be optional
Posted by da...@apache.org.
Allow the use of checkpoint to be optional
A new parameter use_checkpoints which defaults to be true can be either used in the replicator section of
ini files and/or used in the replication command or doc added to the replicator db. When this option is set
to false no checkpoints are taken. It should be used with caution and only for small dbs.
BugzID:14327
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/64ee0303
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/64ee0303
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/64ee0303
Branch: refs/heads/import
Commit: 64ee03033f45b203211638cb33c7a60a2a63d716
Parents: 695f2a8
Author: Bob Dionne <bo...@cloudant.com>
Authored: Sun Aug 19 11:38:28 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 11 +++++++++--
src/couch_replicator_utils.erl | 6 +++++-
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/64ee0303/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 8ebbc6e..43afe33 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -70,7 +70,8 @@
target_db_compaction_notifier = nil,
source_monitor = nil,
target_monitor = nil,
- source_seq = nil
+ source_seq = nil,
+ use_checkpoints = true
}).
@@ -457,6 +458,8 @@ handle_cast({report_seq, Seq},
{noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}.
+code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 30 ->
+ {ok, erlang:append_element(OldState, true)};
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@@ -579,7 +582,8 @@ init_state(Rep) ->
start_db_compaction_notifier(Target, self()),
source_monitor = db_monitor(Source),
target_monitor = db_monitor(Target),
- source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ)
+ source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ),
+ use_checkpoints = get_value(use_checkpoints, Options, true)
},
State#rep_state{timer = start_timer(State)}.
@@ -688,6 +692,9 @@ changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) ->
checkpoint_interval(_State) ->
5000.
+do_checkpoint(#rep_state{use_checkpoints=false} = State) ->
+ NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} },
+ {ok, NewState};
do_checkpoint(#rep_state{current_through_seq=Seq, committed_seq=Seq} = State) ->
SourceCurSeq = source_cur_seq(State),
NewState = State#rep_state{source_seq = SourceCurSeq},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/64ee0303/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 75f79b9..53d5db7 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -229,6 +229,7 @@ make_options(Props) ->
DefConns = couch_config:get("replicator", "http_connections", "20"),
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
+ UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
@@ -238,7 +239,8 @@ make_options(Props) ->
{http_connections, list_to_integer(DefConns)},
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
- {worker_processes, list_to_integer(DefWorkers)}
+ {worker_processes, list_to_integer(DefWorkers)},
+ {use_checkpoints, list_to_existing_atom(UseCheckpoints)}
])).
@@ -280,6 +282,8 @@ convert_options([{<<"socket_options">>, V} | R]) ->
[{socket_options, SocketOptions} | convert_options(R)];
convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
+convert_options([{<<"use_checkpoints">>, V} | R]) ->
+ [{use_checkpoints, V} | convert_options(R)];
convert_options([_ | R]) -> % skip unknown option
convert_options(R).
[47/50] git commit: Fix core etap tests
Posted by da...@apache.org.
Fix core etap tests
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/6f0123b9
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/6f0123b9
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/6f0123b9
Branch: refs/heads/import
Commit: 6f0123b92d40393af393835d3e02e1307222f43c
Parents: b4d569f
Author: Paul J. Davis <pa...@gmail.com>
Authored: Tue Mar 12 15:23:07 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:57 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 351 +++++++++++++++++++++++++---------
1 file changed, 263 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6f0123b9/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index c213199..047f573 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -17,6 +17,7 @@
% public API
-export([replication_started/1, replication_completed/2, replication_error/2]).
+-export([before_doc_update/2, after_doc_read/2]).
% gen_server callbacks
-export([start_link/0, init/1, handle_call/3, handle_info/2, handle_cast/2]).
@@ -50,8 +51,6 @@
}).
-import(couch_util, [
- get_value/2,
- get_value/3,
to_binary/1
]).
@@ -108,6 +107,9 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
end.
+handle_config_change("replicator", "db", _, _, S) ->
+ ok = gen_server:call(S, rep_db_changed),
+ remove_handler;
handle_config_change("replicator", "max_replication_retry_count", V, _, S) ->
ok = gen_server:cast(S, {set_max_retries, retries_value(V)}),
{ok, S};
@@ -124,11 +126,15 @@ init(_) ->
Server = self(),
ok = config:listen_for_changes(?MODULE, Server),
ScanPid = spawn_link(fun() -> scan_all_dbs(Server) end),
+ % Automatically start node local changes feed loop
+ LocalRepDb = ?l2b(config:get("replicator", "db", "_replicator")),
+ Pid = changes_feed_loop(LocalRepDb, 0),
{ok, #state{
db_notifier = db_update_notifier(),
scan_pid = ScanPid,
max_retries = retries_value(
- config:get("replicator", "max_replication_retry_count", "10"))
+ config:get("replicator", "max_replication_retry_count", "10")),
+ rep_start_pids = [Pid]
}}.
@@ -137,8 +143,8 @@ handle_call({rep_db_update, DbName, {ChangeProps} = Change}, _From, State) ->
process_update(State, DbName, Change)
catch
_Tag:Error ->
- {RepProps} = get_value(doc, ChangeProps),
- DocId = get_value(<<"_id">>, RepProps),
+ {RepProps} = get_json_value(doc, ChangeProps),
+ DocId = get_json_value(<<"_id">>, RepProps),
rep_db_update_error(Error, DbName, DocId),
State
end,
@@ -180,6 +186,9 @@ handle_call({rep_db_checkpoint, DbName, EndSeq}, _From, State) ->
true = ets:insert(?DB_TO_SEQ, {DbName, EndSeq}),
{reply, ok, State};
+handle_call(rep_db_changed, _From, State) ->
+ {stop, shutdown, ok, State};
+
handle_call(Msg, From, State) ->
twig:log(error, "Replication manager received unexpected call ~p from ~p",
[Msg, From]),
@@ -256,7 +265,7 @@ terminate(_Reason, State) ->
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
-changes_feed_loop(DbName, Since) ->
+changes_feed_loop(<<"shards/", _/binary>>=DbName, Since) ->
Server = self(),
Pid = spawn_link(
fun() ->
@@ -286,11 +295,42 @@ changes_feed_loop(DbName, Since) ->
}
)
end),
- Pid.
+ Pid;
+changes_feed_loop(DbName, Since) ->
+ ensure_rep_db_exists(DbName),
+ Server = self(),
+ spawn_link(fun() ->
+ UserCtx = #user_ctx{roles = [<<"_admin">>, <<"_replicator">>]},
+ DbOpenOptions = [{user_ctx, UserCtx}, sys_db],
+ {ok, Db} = couch_db:open_int(DbName, DbOpenOptions),
+ ChangesFeedFun = couch_changes:handle_changes(
+ #changes_args{
+ include_docs = true,
+ since = Since,
+ feed = "continuous",
+ timeout = infinity
+ },
+ {json_req, null},
+ Db
+ ),
+ EnumFun = fun
+ ({change, Change, _}, _) ->
+ case has_valid_rep_id(Change) of
+ true ->
+ Msg = {rep_db_update, DbName, Change},
+ ok = gen_server:call(Server, Msg, infinity);
+ false ->
+ ok
+ end;
+ (_, _) ->
+ ok
+ end,
+ ChangesFeedFun(EnumFun)
+ end).
has_valid_rep_id({Change}) ->
- has_valid_rep_id(get_value(<<"id">>, Change));
+ has_valid_rep_id(get_json_value(<<"id">>, Change));
has_valid_rep_id(<<?DESIGN_DOC_PREFIX, _Rest/binary>>) ->
false;
has_valid_rep_id(_Else) ->
@@ -299,22 +339,24 @@ has_valid_rep_id(_Else) ->
db_update_notifier() ->
Server = self(),
IsReplicatorDbFun = is_replicator_db_fun(),
- {ok, Notifier} = couch_db_update_notifier:start_link(
- fun({updated, DbName}) ->
- case IsReplicatorDbFun(DbName) of
- true ->
- ok = gen_server:call(Server, {resume_scan, mem3:dbname(DbName)}, infinity);
- _ ->
- ok
- end;
- ({deleted, DbName}) ->
- case IsReplicatorDbFun(DbName) of
- true ->
- clean_up_replications(mem3:dbname(DbName));
- _ ->
- ok
+ {ok, Notifier} = couch_db_update_notifier:start_link(fun
+ ({Event, ShardDbName})
+ when Event == created; Event == updated; Event == deleted ->
+ DbName = mem3:dbname(ShardDbName),
+ IsRepDb = IsReplicatorDbFun(DbName),
+ case Event of
+ created when IsRepDb ->
+ ensure_rep_ddoc_exists(DbName);
+ updated when IsRepDb ->
+ ensure_rep_ddoc_exists(DbName),
+ Msg = {resume_scan, DbName},
+ ok = gen_server:call(Server, Msg, infinity);
+ deleted when IsRepDb ->
+ clean_up_replications(DbName);
+ _ ->
+ ok
end;
- (_) ->
+ (_Event) ->
ok
end
),
@@ -331,9 +373,9 @@ rescan(#state{scan_pid = ScanPid} = State) ->
rescan(State#state{scan_pid = nil}).
process_update(State, DbName, {Change}) ->
- {RepProps} = JsonRepDoc = get_value(doc, Change),
- DocId = get_value(<<"_id">>, RepProps),
- case {mem3_util:owner(DbName, DocId), get_value(deleted, Change, false)} of
+ {RepProps} = JsonRepDoc = get_json_value(doc, Change),
+ DocId = get_json_value(<<"_id">>, RepProps),
+ case {is_owner(DbName, DocId), get_json_value(deleted, Change, false)} of
{false, _} ->
replication_complete(DbName, DocId),
State;
@@ -341,7 +383,7 @@ process_update(State, DbName, {Change}) ->
rep_doc_deleted(DbName, DocId),
State;
{true, false} ->
- case get_value(<<"_replication_state">>, RepProps) of
+ case get_json_value(<<"_replication_state">>, RepProps) of
undefined ->
maybe_start_replication(State, DbName, DocId, JsonRepDoc);
<<"triggered">> ->
@@ -359,6 +401,13 @@ process_update(State, DbName, {Change}) ->
end
end.
+
+is_owner(<<"shards/", _/binary>>=DbName, DocId) ->
+ mem3_util:owner(DbName, DocId);
+is_owner(_, _) ->
+ true.
+
+
rep_db_update_error(Error, DbName, DocId) ->
case Error of
{bad_rep_doc, Reason} ->
@@ -372,13 +421,13 @@ rep_db_update_error(Error, DbName, DocId) ->
rep_user_ctx({RepDoc}) ->
- case get_value(<<"user_ctx">>, RepDoc) of
+ case get_json_value(<<"user_ctx">>, RepDoc) of
undefined ->
#user_ctx{};
{UserCtx} ->
#user_ctx{
- name = get_value(<<"name">>, UserCtx, null),
- roles = get_value(<<"roles">>, UserCtx, [])
+ name = get_json_value(<<"name">>, UserCtx, null),
+ roles = get_json_value(<<"roles">>, UserCtx, [])
}
end.
@@ -428,7 +477,7 @@ parse_rep_doc(RepDoc) ->
maybe_tag_rep_doc(DbName, DocId, {RepProps}, RepId) ->
- case get_value(<<"_replication_id">>, RepProps) of
+ case get_json_value(<<"_replication_id">>, RepProps) of
RepId ->
ok;
_ ->
@@ -546,42 +595,26 @@ clean_up_replications(DbName) ->
update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
- {Pid, Ref} =
- spawn_monitor(fun() ->
- try
- case fabric:open_doc(mem3:dbname(RepDbName), RepDocId, []) of
- {ok, LatestRepDoc} ->
- update_rep_doc(RepDbName, LatestRepDoc, KVs);
- _ ->
- ok
- end
- catch
+ try
+ case open_rep_doc(RepDbName, RepDocId) of
+ {ok, LastRepDoc} ->
+ update_rep_doc(RepDbName, LastRepDoc, KVs);
+ _ ->
+ ok
+ end
+ catch
throw:conflict ->
- % a race condition may cause an update conflict,
- % in which cae update_rep_doc is called again to refetch
- twig:log(error, "Conflict error when updating replication document `~s`."
- " Retrying.", [RepDocId]),
+ Msg = "Conflict when updating replication document `~s`. Retrying.",
+ twig:log(error, Msg, [RepDocId]),
ok = timer:sleep(5),
- update_rep_doc(RepDbName, RepDocId, KVs);
- Type:Error ->
- exit({Type, Error})
- end
- end),
- receive
- {'DOWN', Ref, process, Pid, normal} ->
- ok;
- {'DOWN', Ref, process, Pid, {throw, Error}} ->
- throw(Error);
- {'DOWN', Ref, process, Pid, {error, Error}} ->
- erlang:error(Error)
+ update_rep_doc(RepDbName, RepDocId, KVs)
end;
-
update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
NewRepDocBody = lists:foldl(
fun({K, undefined}, Body) ->
lists:keydelete(K, 1, Body);
({<<"_replication_state">> = K, State} = KV, Body) ->
- case get_value(K, Body) of
+ case get_json_value(K, Body) of
State ->
Body;
_ ->
@@ -600,9 +633,55 @@ update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
_ ->
% Might not succeed - when the replication doc is deleted right
% before this update (not an error, ignore).
- fabric:update_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}}, [?CTX])
+ save_rep_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}})
+ end.
+
+
+open_rep_doc(<<"shards/", _/binary>>=ShardDbName, DocId) ->
+ defer_call(fun() ->
+ fabric:open_doc(mem3:dbname(ShardDbName), DocId, [])
+ end);
+open_rep_doc(DbName, DocId) ->
+ {ok, Db} = couch_db:open_int(DbName, [?CTX, sys_db]),
+ try
+ couch_db:open_doc(Db, DocId, [ejson_body])
+ after
+ couch_db:close(Db)
+ end.
+
+save_rep_doc(<<"shards/", _/binary>>=DbName, Doc) ->
+ defer_call(fun() ->
+ fabric:update_doc(DbName, Doc, [?CTX])
+ end);
+save_rep_doc(DbName, Doc) ->
+ {ok, Db} = couch_db:open_int(DbName, [?CTX, sys_db]),
+ try
+ couch_db:update_doc(Db, Doc, [])
+ after
+ couch_db:close(Db)
end.
+defer_call(Fun) ->
+ {Pid, Ref} = erlang:spawn_monitor(fun() ->
+ try
+ exit({exit_ok, Fun()})
+ catch
+ Type:Reason ->
+ exit({exit_err, Type, Reason})
+ end
+ end),
+ receive
+ {'DOWN', Ref, process, Pid, {exit_ok, Resp}} ->
+ Resp;
+ {'DOWN', Ref, process, Pid, {exit_err, throw, Error}} ->
+ throw(Error);
+ {'DOWN', Ref, process, Pid, {exit_err, error, Error}} ->
+ erlang:error(Error);
+ {'DOWN', Ref, process, Pid, {exit_err, exit, Error}} ->
+ exit(Error)
+ end.
+
+
% RFC3339 timestamps.
% Note: doesn't include the time seconds fraction (RFC3339 says it's optional).
timestamp() ->
@@ -623,31 +702,38 @@ zone(Hr, Min) ->
io_lib:format("-~2..0w:~2..0w", [abs(Hr), abs(Min)]).
-ensure_rep_db_exists() ->
- DbName = ?l2b(couch_config:get("replicator", "db", "_replicator")),
- UserCtx = #user_ctx{roles = [<<"_admin">>, <<"_replicator">>]},
- case couch_db:open_int(DbName, [sys_db, {user_ctx, UserCtx}, nologifmissing]) of
- {ok, Db} ->
- Db;
- _Error ->
- {ok, Db} = couch_db:create(DbName, [sys_db, {user_ctx, UserCtx}])
+ensure_rep_db_exists(DbName) ->
+ Db = case couch_db:open_int(DbName, [?CTX, sys_db, nologifmissing]) of
+ {ok, Db0} ->
+ Db0;
+ _Error ->
+ {ok, Db0} = couch_db:create(DbName, [?CTX, sys_db]),
+ Db0
end,
- ensure_rep_ddoc_exists(Db, <<"_design/_replicator">>),
+ ensure_rep_ddoc_exists(DbName),
{ok, Db}.
-ensure_rep_ddoc_exists(RepDb, DDocID) ->
- case couch_db:open_doc(RepDb, DDocID, []) of
- {ok, _Doc} ->
- ok;
- _ ->
- DDoc = couch_doc:from_json_obj({[
- {<<"_id">>, DDocID},
- {<<"language">>, <<"javascript">>},
- {<<"validate_doc_update">>, ?REP_DB_DOC_VALIDATE_FUN}
- ]}),
- {ok, _Rev} = couch_db:update_doc(RepDb, DDoc, [])
- end.
+ensure_rep_ddoc_exists(RepDb) ->
+ DDocId = <<"_design/_replicator">>,
+ case open_rep_doc(RepDb, DDocId) of
+ {ok, _Doc} ->
+ ok;
+ _ ->
+ DDoc = couch_doc:from_json_obj({[
+ {<<"_id">>, DDocId},
+ {<<"language">>, <<"javascript">>},
+ {<<"validate_doc_update">>, ?REP_DB_DOC_VALIDATE_FUN}
+ ]}),
+ try
+ {ok, _} = save_rep_doc(RepDb, DDoc)
+ catch
+ throw:conflict ->
+ % NFC what to do about this other than
+ % not kill the process.
+ ok
+ end
+ end.
% pretty-print replication id
@@ -687,20 +773,85 @@ state_after_error(#rep_state{retries_left = Left, wait = Wait} = State) ->
State#rep_state{retries_left = Left - 1, wait = Wait2}
end.
+
+before_doc_update(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db) ->
+ Doc;
+before_doc_update(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
+ #user_ctx{roles = Roles, name = Name} = UserCtx,
+ case lists:member(<<"_replicator">>, Roles) of
+ true ->
+ Doc;
+ false ->
+ case couch_util:get_value(?OWNER, Body) of
+ undefined ->
+ Doc#doc{body = {?replace(Body, ?OWNER, Name)}};
+ Name ->
+ Doc;
+ Other ->
+ case (catch couch_db:check_is_admin(Db)) of
+ ok when Other =:= null ->
+ Doc#doc{body = {?replace(Body, ?OWNER, Name)}};
+ ok ->
+ Doc;
+ _ ->
+ throw({forbidden, <<"Can't update replication documents",
+ " from other users.">>})
+ end
+ end
+ end.
+
+
+after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db) ->
+ Doc;
+after_doc_read(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
+ #user_ctx{name = Name} = UserCtx,
+ case (catch couch_db:check_is_admin(Db)) of
+ ok ->
+ Doc;
+ _ ->
+ case couch_util:get_value(?OWNER, Body) of
+ Name ->
+ Doc;
+ _Other ->
+ Source = strip_credentials(couch_util:get_value(<<"source">>,
+Body)),
+ Target = strip_credentials(couch_util:get_value(<<"target">>,
+Body)),
+ NewBody0 = ?replace(Body, <<"source">>, Source),
+ NewBody = ?replace(NewBody0, <<"target">>, Target),
+ #doc{revs = {Pos, [_ | Revs]}} = Doc,
+ NewDoc = Doc#doc{body = {NewBody}, revs = {Pos - 1, Revs}},
+ NewRevId = couch_db:new_revid(NewDoc),
+ NewDoc#doc{revs = {Pos, [NewRevId | Revs]}}
+ end
+ end.
+
+strip_credentials(Url) when is_binary(Url) ->
+ re:replace(Url,
+ "http(s)?://(?:[^:]+):[^@]+@(.*)$",
+ "http\\1://\\2",
+ [{return, binary}]);
+strip_credentials({Props}) ->
+ {lists:keydelete(<<"oauth">>, 1, Props)}.
+
scan_all_dbs(Server) when is_pid(Server) ->
- {ok, Db} = mem3_util:ensure_exists(
- config:get("mem3", "shard_db", "dbs")),
+ {ok, Db} = mem3_util:ensure_exists(config:get("mem3", "shard_db", "dbs")),
ChangesFun = couch_changes:handle_changes(#changes_args{}, nil, Db),
IsReplicatorDbFun = is_replicator_db_fun(),
ChangesFun(fun({change, {Change}, _}, _) ->
- DbName = couch_util:get_value(<<"id">>, Change),
+ DbName = get_json_value(<<"id">>, Change),
case DbName of <<"_design/", _/binary>> -> ok; _Else ->
case couch_replicator_utils:is_deleted(Change) of
true ->
ok;
false ->
- IsReplicatorDbFun(DbName) andalso
- gen_server:call(Server, {resume_scan, DbName})
+ case IsReplicatorDbFun(DbName) of
+ true ->
+ ensure_rep_ddoc_exists(DbName),
+ gen_server:call(Server, {resume_scan, DbName});
+ false ->
+ ok
+ end
end
end;
(_, _) -> ok
@@ -709,6 +860,30 @@ scan_all_dbs(Server) when is_pid(Server) ->
is_replicator_db_fun() ->
{ok, RegExp} = re:compile("^([a-z][a-z0-9\\_\\$()\\+\\-\\/]*/)?_replicator$"),
- fun(DbName) ->
- match =:= re:run(mem3:dbname(DbName), RegExp, [{capture,none}])
+ fun
+ (<<"shards/", _/binary>>=DbName) ->
+ match =:= re:run(mem3:dbname(DbName), RegExp, [{capture,none}]);
+ (DbName) ->
+ LocalRepDb = ?l2b(config:get("replicator", "db", "_replicator")),
+ DbName == LocalRepDb
+ end.
+
+get_json_value(Key, Props) ->
+ get_json_value(Key, Props, undefined).
+
+get_json_value(Key, Props, Default) when is_atom(Key) ->
+ Ref = make_ref(),
+ case couch_util:get_value(Key, Props, Ref) of
+ Ref ->
+ couch_util:get_value(?l2b(atom_to_list(Key)), Props, Default);
+ Else ->
+ Else
+ end;
+get_json_value(Key, Props, Default) when is_binary(Key) ->
+ Ref = make_ref(),
+ case couch_util:get_value(Key, Props, Ref) of
+ Ref ->
+ couch_util:get_value(list_to_atom(?b2l(Key)), Props, Default);
+ Else ->
+ Else
end.
[13/50] git commit: remove io:format
Posted by da...@apache.org.
remove io:format
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/e1b7cc19
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/e1b7cc19
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/e1b7cc19
Branch: refs/heads/import
Commit: e1b7cc190651da7f64558b411743a8e2a72b126f
Parents: b35a5b9
Author: Robert Newson <ro...@cloudant.com>
Authored: Thu Aug 2 16:34:36 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/e1b7cc19/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index a160f51..d31f0b9 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -358,7 +358,6 @@ rep_user_ctx({RepDoc}) ->
maybe_start_replication(State, DbName, DocId, RepDoc) ->
- io:format("should I start a repl here for ~p ~n",[DocId]),
#rep{id = {BaseId, _} = RepId} = Rep = parse_rep_doc(RepDoc),
case rep_state(RepId) of
nil ->
[38/50] git commit: Use config:listen_for_changes in replicator
manager
Posted by da...@apache.org.
Use config:listen_for_changes in replicator manager
BugzID: 13179
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7915756a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7915756a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7915756a
Branch: refs/heads/import
Commit: 7915756a3a19034f9a90d79773fccf126af42a3b
Parents: 9e2d76f
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Feb 27 09:40:34 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 25 ++++++++++++++++++++-----
1 file changed, 20 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7915756a/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index d8d1c4a..1db7929 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -12,6 +12,7 @@
-module(couch_replicator_manager).
-behaviour(gen_server).
+-behaviour(config_listener).
% public API
-export([replication_started/1, replication_completed/2, replication_error/2]).
@@ -21,6 +22,9 @@
-export([start_link/0, init/1, handle_call/3, handle_info/2, handle_cast/2]).
-export([code_change/3, terminate/2]).
+% config_listener callback
+-export([handle_config_change/5]).
+
-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator.hrl").
-include("couch_replicator_js_functions.hrl").
@@ -104,6 +108,13 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
end.
+handle_config_change("replicator", "max_replication_retry_count", V, _, S) ->
+ ok = gen_server:cast(S, {set_max_retries, retries_value(V)}),
+ {ok, S};
+handle_config_change(_, _, _, _, S) ->
+ {ok, S}.
+
+
init(_) ->
process_flag(trap_exit, true),
net_kernel:monitor_nodes(true),
@@ -111,11 +122,7 @@ init(_) ->
?REP_TO_STATE = ets:new(?REP_TO_STATE, [named_table, set, public]),
?DB_TO_SEQ = ets:new(?DB_TO_SEQ, [named_table, set, public]),
Server = self(),
- ok = couch_config:register(
- fun("replicator", "max_replication_retry_count", V) ->
- ok = gen_server:cast(Server, {set_max_retries, retries_value(V)})
- end
- ),
+ ok = config:listen_for_changes(?MODULE, Server),
ScanPid = spawn_link(fun() -> scan_all_dbs(Server) end),
{ok, #state{
db_notifier = db_update_notifier(),
@@ -211,6 +218,14 @@ handle_info({'DOWN', _Ref, _, _, _}, State) ->
% From a db monitor created by a replication process. Ignore.
{noreply, State};
+handle_info({gen_event_EXIT, {config_listener, ?MODULE}, _Reason}, State) ->
+ erlang:send_after(5000, self(), restart_config_listener),
+ {noreply, State};
+
+handle_info(restart_config_listener, State) ->
+ ok = config:listen_for_changes(?MODULE, self()),
+ {noreply, State};
+
handle_info(Msg, State) ->
twig:log(error,"Replication manager received unexpected message ~p", [Msg]),
{stop, {unexpected_msg, Msg}, State}.
[10/50] git commit: Use latest=true when retrieving revisions
Posted by da...@apache.org.
Use latest=true when retrieving revisions
This allows the replicator to keep up with revisions that may have
occured after missing_revs was called.
BugzID: 14241
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/5485ec0b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/5485ec0b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/5485ec0b
Branch: refs/heads/import
Commit: 5485ec0be71baefeb45e30a332e57621f53b3dfa
Parents: 97bfe8e
Author: Bob Dionne <bo...@cloudant.com>
Authored: Tue Aug 7 12:42:36 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 ++
src/couch_replicator_worker.erl | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/5485ec0b/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index a2aa8a0..ecf3064 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -465,6 +465,8 @@ options_to_query_args([revs | Rest], Acc) ->
options_to_query_args(Rest, [{"revs", "true"} | Acc]);
options_to_query_args([{open_revs, all} | Rest], Acc) ->
options_to_query_args(Rest, [{"open_revs", "all"} | Acc]);
+options_to_query_args([latest | Rest], Acc) ->
+ options_to_query_args(Rest, [{"latest", "true"} | Acc]);
options_to_query_args([{open_revs, Revs} | Rest], Acc) ->
JsonRevs = ?b2l(iolist_to_binary(?JSON_ENCODE(couch_doc:revs_to_strs(Revs)))),
options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/5485ec0b/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index ebdeada..78d66ea 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -295,13 +295,13 @@ spawn_doc_reader(Source, Target, FetchParams) ->
fetch_doc(Source, {Id, Revs, PAs}, DocHandler, Acc) ->
try
couch_replicator_api_wrap:open_doc_revs(
- Source, Id, Revs, [{atts_since, PAs}], DocHandler, Acc)
+ Source, Id, Revs, [{atts_since, PAs}, latest], DocHandler, Acc)
catch
throw:{missing_stub, _} ->
twig:log(error,"Retrying fetch and update of document `~s` due to out of "
"sync attachment stubs. Missing revisions are: ~s",
[Id, couch_doc:revs_to_strs(Revs)]),
- couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [], DocHandler, Acc)
+ couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc)
end.
[04/50] git commit: Add mp_parse to _utils instead of importing from
couch
Posted by da...@apache.org.
Add mp_parse to _utils instead of importing from couch
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b4b3dd1e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b4b3dd1e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b4b3dd1e
Branch: refs/heads/import
Commit: b4b3dd1e121c3756810a1a3cb0a6e16aef6a4688
Parents: 72fec79
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jun 13 15:40:41 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 +-
src/couch_replicator_utils.erl | 30 ++++++++++++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b4b3dd1e/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index ecf3b8d..5e014b2 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -619,7 +619,7 @@ doc_from_multi_part_stream(ContentType, DataFun, Ref) ->
Parser = spawn_link(fun() ->
{<<"--">>, _, _} = couch_httpd:parse_multipart_request(
ContentType, DataFun,
- fun(Next) -> couch_doc:mp_parse_doc1(Next, []) end),
+ fun(Next) -> couch_replicator_utils:mp_parse_doc(Next, []) end),
unlink(Self)
end),
Parser ! {get_doc_bytes, Ref, self()},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b4b3dd1e/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index f0f6d41..dfc0244 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -17,6 +17,7 @@
-export([start_db_compaction_notifier/2, stop_db_compaction_notifier/1]).
-export([replication_id/2]).
-export([sum_stats/2]).
+-export([mp_parse_doc/2]).
-include_lib("couch/include/couch_db.hrl").
-include_lib("ibrowse/include/ibrowse.hrl").
@@ -387,3 +388,32 @@ sum_stats(#rep_stats{} = S1, #rep_stats{} = S2) ->
doc_write_failures =
S1#rep_stats.doc_write_failures + S2#rep_stats.doc_write_failures
}.
+
+mp_parse_doc({headers, H}, []) ->
+ case couch_util:get_value("content-type", H) of
+ {"application/json", _} ->
+ fun (Next) ->
+ mp_parse_doc(Next, [])
+ end
+ end;
+mp_parse_doc({body, Bytes}, AccBytes) ->
+ fun (Next) ->
+ mp_parse_doc(Next, [Bytes | AccBytes])
+ end;
+mp_parse_doc(body_end, AccBytes) ->
+ receive {get_doc_bytes, Ref, From} ->
+ From ! {doc_bytes, Ref, lists:reverse(AccBytes)}
+ end,
+ fun mp_parse_atts/1.
+
+mp_parse_atts(eof) ->
+ ok;
+mp_parse_atts({headers, _H}) ->
+ fun mp_parse_atts/1;
+mp_parse_atts({body, Bytes}) ->
+ receive {get_bytes, Ref, From} ->
+ From ! {bytes, Ref, Bytes}
+ end,
+ fun mp_parse_atts/1;
+mp_parse_atts(body_end) ->
+ fun mp_parse_atts/1.
[39/50] git commit: Don't add undefined stats to body
Posted by da...@apache.org.
Don't add undefined stats to body
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/3160f54f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/3160f54f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/3160f54f
Branch: refs/heads/import
Commit: 3160f54f56f454c23c78e6a45c5404728b888d60
Parents: d17aced
Author: Robert Newson <ro...@cloudant.com>
Authored: Sun Mar 3 10:19:32 2013 +0000
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/3160f54f/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 1db7929..93f0e89 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -575,7 +575,9 @@ update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
NewRepDocBody = lists:foldl(
- fun({<<"_replication_state">> = K, State} = KV, Body) ->
+ fun({K, undefined}, Body) ->
+ lists:keydelete(K, 1, Body);
+ ({<<"_replication_state">> = K, State} = KV, Body) ->
case get_value(K, Body) of
State ->
Body;
[26/50] git commit: Monitor spawned calls to fabric
Posted by da...@apache.org.
Monitor spawned calls to fabric
When continuous replications crash, a race condition prevents the replication doc from
being updated to reflect the error status. Monitoring the spawned calls to fabric update
prevents this.
BugzID: 17134
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/0207480b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/0207480b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/0207480b
Branch: refs/heads/import
Commit: 0207480b152bfd90486cea3b4768689b39faef83
Parents: 64ee030
Author: Bob Dionne <bo...@cloudant.com>
Authored: Thu Feb 7 15:16:25 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/0207480b/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 2e5dff9..1cfaadd 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -515,7 +515,8 @@ stop_all_replications() ->
update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
- spawn_link(fun() ->
+ {Pid, Ref} =
+ spawn_monitor(fun() ->
try
case fabric:open_doc(mem3:dbname(RepDbName), RepDocId, []) of
{ok, LatestRepDoc} ->
@@ -523,15 +524,26 @@ update_rep_doc(RepDbName, RepDocId, KVs) when is_binary(RepDocId) ->
_ ->
ok
end
- catch throw:conflict ->
- % Shouldn't happen, as by default only the role _replicator can
- % update replication documents.
+ catch
+ throw:conflict ->
+ % a race condition may cause an update conflict,
+ % in which cae update_rep_doc is called again to refetch
twig:log(error, "Conflict error when updating replication document `~s`."
- " Retrying.", [RepDocId]),
+ " Retrying.", [RepDocId]),
ok = timer:sleep(5),
- update_rep_doc(RepDbName, RepDocId, KVs)
+ update_rep_doc(RepDbName, RepDocId, KVs);
+ Type:Error ->
+ exit({Type, Error})
end
- end);
+ end),
+ receive
+ {'DOWN', Ref, process, Pid, normal} ->
+ ok;
+ {'DOWN', Ref, process, Pid, {throw, Error}} ->
+ throw(Error);
+ {'DOWN', Ref, process, Pid, {error, Error}} ->
+ erlang:error(Error)
+ end;
update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
NewRepDocBody = lists:foldl(
@@ -555,9 +567,7 @@ update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs) ->
_ ->
% Might not succeed - when the replication doc is deleted right
% before this update (not an error, ignore).
- spawn_link(fun() ->
- fabric:update_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}}, [?CTX])
- end)
+ fabric:update_doc(RepDbName, RepDoc#doc{body = {NewRepDocBody}}, [?CTX])
end.
% RFC3339 timestamps.
[14/50] git commit: Better propagation of errors when replication
fails
Posted by da...@apache.org.
Better propagation of errors when replication fails
Fix twig call in handle_terminate. Detect and handle db_not_found error when
source is missing and propagate to calling node
BugxID: 14273
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b57bbbeb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b57bbbeb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b57bbbeb
Branch: refs/heads/import
Commit: b57bbbeb025d20c14988b69ce7ce5288dd211494
Parents: 5485ec0
Author: Bob Dionne <bo...@cloudant.com>
Authored: Mon Aug 13 17:14:50 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b57bbbeb/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 6639edb..a1148cc 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -484,8 +484,18 @@ terminate(Reason, #rep_state{} = State) ->
couch_replicator_manager:replication_error(Rep, Reason);
terminate(shutdown, {error, Class, Error, Stack, InitArgs}) ->
- twig:log("~p:~p: Replication failed to start for args ~p: ~p",
- [Class, Error, InitArgs, Stack]).
+ #rep{id=RepId} = InitArgs,
+ twig:log(error,"~p:~p: Replication failed to start for args ~p: ~p",
+ [Class, Error, InitArgs, Stack]),
+ case Error of
+ {unauthorized, DbUri} ->
+ NotifyError = {unauthorized, <<"unauthorized to access or create database ", DbUri/binary>>};
+ {db_not_found, DbUri} ->
+ NotifyError = {db_not_found, <<"could not open ", DbUri/binary>>};
+ _ ->
+ NotifyError = Error
+ end,
+ couch_replicator_notifier:notify({error, RepId, NotifyError}).
terminate_cleanup(State) ->
update_task(State),
[31/50] git commit: Avoid deadlocking the httpc pool
Posted by da...@apache.org.
Avoid deadlocking the httpc pool
The multipart mime parsing code was getting into a state where it would
orphan parsers that held onto pool connections. This just adds a monitor
to make sure that the parser process dies when the parent dies normally
without finishing reading the rest of the parser.
BugzID: 16751
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/1f99c245
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/1f99c245
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/1f99c245
Branch: refs/heads/import
Commit: 1f99c2459525977f95825f3bdf0b4d560a779b2d
Parents: c1f3d72
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Feb 22 13:44:20 2013 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/1f99c245/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index abdd548..1b15d60 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -177,6 +177,20 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
end),
unlink(Self)
end),
+ % If this process dies normally we can leave
+ % the Streamer process hanging around keeping an
+ % HTTP connection open. This is a bit of a
+ % hammer approach to making sure it releases
+ % that connection back to the pool.
+ spawn(fun() ->
+ Ref = erlang:monitor(process, Self),
+ receive
+ {'DOWN', Ref, process, Self, normal} ->
+ exit(Streamer, {streamer_parent_died, Self});
+ {'DOWN', Ref, process, Self, _} ->
+ ok
+ end
+ end),
receive
{started_open_doc_revs, Ref} ->
receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc)
[35/50] git commit: Ensure _changes terminates before
inactivity_timeout
Posted by da...@apache.org.
Ensure _changes terminates before inactivity_timeout
This will allow the replicator to reconnect cleanly instead of
considering the request a failure.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a18f8e9e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a18f8e9e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a18f8e9e
Branch: refs/heads/import
Commit: a18f8e9eb49a488800a647b9f032b0d405318c79
Parents: 317131b
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Thu Feb 28 16:51:09 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:41 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a18f8e9e/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index cad384a..01b9b68 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -321,8 +321,9 @@ update_docs(Db, DocList, Options, UpdateType) ->
{ok, bulk_results_to_errors(DocList, Result, UpdateType)}.
-changes_since(#httpdb{headers = Headers1, timeout = Timeout} = HttpDb,
+changes_since(#httpdb{headers = Headers1, timeout = InactiveTimeout} = HttpDb,
Style, StartSeq, UserFun, Options) ->
+ Timeout = erlang:max(1000, InactiveTimeout - 5000),
BaseQArgs = case get_value(continuous, Options, false) of
false ->
[{"feed", "normal"}];
[19/50] git commit: Save ourselves some work by flushing on demonitor
Posted by da...@apache.org.
Save ourselves some work by flushing on demonitor
BugzID: 14132
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/39370132
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/39370132
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/39370132
Branch: refs/heads/import
Commit: 39370132b0224d825ddbc0717bb1beb6fafd006a
Parents: 50f1edb
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Aug 15 14:33:12 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:38 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_httpc_pool.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/39370132/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index ddfb94b..b9d4a43 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -171,5 +171,5 @@ demonitor_client(Tab, Worker) ->
[] -> ok;
[{Worker, MonRef}] ->
ets:delete(Tab, Worker),
- erlang:demonitor(MonRef)
+ erlang:demonitor(MonRef, [flush])
end.
[32/50] git commit: Teach replicator to gracefully reconnect
Posted by da...@apache.org.
Teach replicator to gracefully reconnect
This patch allows the replicator to recognize the "last_seq" field at
the end of a continuous _changes feed and reconnect to the feed using
that value.
BugzID: 17502
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/c1f3d72f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/c1f3d72f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/c1f3d72f
Branch: refs/heads/import
Commit: c1f3d72f523b3cb3cc34206ce9c9c157b9b88f34
Parents: 17671fa
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Fri Feb 22 10:45:23 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 16 +++++++++++++++-
src/couch_replicator_api_wrap.erl | 27 ++++++++++++++++-----------
2 files changed, 31 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c1f3d72f/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index c72a398..e1ff15c 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -642,7 +642,21 @@ read_changes(StartSeq, Db, ChangesQueue, Options) ->
_ ->
ok = couch_work_queue:queue(ChangesQueue, DocInfo)
end,
- put(last_seq, Seq)
+ put(last_seq, Seq);
+ ({last_seq, LS}) ->
+ case get_value(continuous, Options) of
+ true ->
+ % LS should never be undefined, but it doesn't hurt to be
+ % defensive inside the replicator.
+ Seq = case LS of undefined -> get(last_seq); _ -> LS end,
+ read_changes(Seq, Db, ChangesQueue, Options);
+ _ ->
+ % This clause is unreachable today, but let's plan ahead
+ % for the future where we checkpoint against last_seq
+ % instead of the sequence of the last change. The two can
+ % differ substantially in the case of a restrictive filter.
+ ok
+ end
end, Options),
couch_work_queue:close(ChangesQueue)
catch exit:{http_request_failed, _, _, _} = Error ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c1f3d72f/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index ecf3064..abdd548 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -692,17 +692,22 @@ parse_changes_line(object_start, UserFun) ->
end.
json_to_doc_info({Props}) ->
- RevsInfo = lists:map(
- fun({Change}) ->
- Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)),
- Del = couch_replicator_utils:is_deleted(Change),
- #rev_info{rev=Rev, deleted=Del}
- end, get_value(<<"changes">>, Props)),
- #doc_info{
- id = get_value(<<"id">>, Props),
- high_seq = get_value(<<"seq">>, Props),
- revs = RevsInfo
- }.
+ case get_value(<<"changes">>, Props) of
+ undefined ->
+ {last_seq, get_value(<<"last_seq">>, Props)};
+ Changes ->
+ RevsInfo = lists:map(
+ fun({Change}) ->
+ Rev = couch_doc:parse_rev(get_value(<<"rev">>, Change)),
+ Del = couch_replicator_utils:is_deleted(Change),
+ #rev_info{rev=Rev, deleted=Del}
+ end, Changes),
+ #doc_info{
+ id = get_value(<<"id">>, Props),
+ high_seq = get_value(<<"seq">>, Props),
+ revs = RevsInfo
+ }
+ end.
bulk_results_to_errors(Docs, {ok, Results}, interactive_edit) ->
[29/50] git commit: Added a new field to task status, changes_pending
Posted by da...@apache.org.
Added a new field to task status, changes_pending
This will provide a rough idea as to how work is remaining in a replication.
It's computed by subtracting the current committed seq from the current update seq.
BugzID: 16058
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/79df0f22
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/79df0f22
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/79df0f22
Branch: refs/heads/import
Commit: 79df0f22255096422ba9dc9ad0141bcd9889ca34
Parents: 0207480
Author: Bob Dionne <bo...@cloudant.com>
Authored: Wed Feb 13 13:03:43 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:40 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/79df0f22/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 43afe33..c72a398 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -278,6 +278,7 @@ do_init(#rep{options = Options, id = {BaseId, Ext}, user_ctx=UserCtx} = Rep) ->
{missing_revisions_found, 0},
{docs_read, 0},
{docs_written, 0},
+ {changes_pending, pending(SourceCurSeq, CommittedSeq)},
{doc_write_failures, 0},
{source_seq, SourceCurSeq},
{checkpointed_source_seq, CommittedSeq},
@@ -960,17 +961,22 @@ update_task(State) ->
rep_stats(State) ->
#rep_state{
committed_seq = {_, CommittedSeq},
- stats = Stats
+ stats = Stats,
+ source_seq = SourceCurSeq
} = State,
[
{revisions_checked, Stats#rep_stats.missing_checked},
{missing_revisions_found, Stats#rep_stats.missing_found},
{docs_read, Stats#rep_stats.docs_read},
{docs_written, Stats#rep_stats.docs_written},
+ {changes_pending, pending(SourceCurSeq, CommittedSeq)},
{doc_write_failures, Stats#rep_stats.doc_write_failures},
{checkpointed_source_seq, CommittedSeq}
].
+pending(SourceCurSeq, CommittedSeq) ->
+ unpack_seq(SourceCurSeq) - unpack_seq(CommittedSeq).
+
unpack_seq(Seq) when is_number(Seq) ->
Seq;
unpack_seq([SeqNum, _]) ->
[48/50] git commit: Fix up copyright headers
Posted by da...@apache.org.
Fix up copyright headers
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9921223a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9921223a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9921223a
Branch: refs/heads/import
Commit: 9921223ad9f5787db2f6349543a824a40a1c83e5
Parents: 53e6dbe
Author: Robert Newson <rn...@apache.org>
Authored: Thu Jun 13 13:42:11 2013 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Thu Jun 13 13:46:12 2013 +0100
----------------------------------------------------------------------
rebar.config | 2 --
src/couch_replicator_sup.erl | 2 --
2 files changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9921223a/rebar.config
----------------------------------------------------------------------
diff --git a/rebar.config b/rebar.config
index 4af6b85..48deb03 100644
--- a/rebar.config
+++ b/rebar.config
@@ -1,5 +1,3 @@
-% Copyright 2011 Cloudant
-%
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9921223a/src/couch_replicator_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_sup.erl b/src/couch_replicator_sup.erl
index 848cb13..bad9747 100644
--- a/src/couch_replicator_sup.erl
+++ b/src/couch_replicator_sup.erl
@@ -1,5 +1,3 @@
-% Copyright 2010 Cloudant
-%
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
[03/50] git commit: We're not using dynamic modules
Posted by da...@apache.org.
We're not using dynamic modules
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/815ac0a7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/815ac0a7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/815ac0a7
Branch: refs/heads/import
Commit: 815ac0a77842cf1653b736577014bdb37926de1f
Parents: b4b3dd1
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Fri Jun 15 11:22:22 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:36 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_sup.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/815ac0a7/src/couch_replicator_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_sup.erl b/src/couch_replicator_sup.erl
index 9dcd48a..848cb13 100644
--- a/src/couch_replicator_sup.erl
+++ b/src/couch_replicator_sup.erl
@@ -32,7 +32,7 @@ init(_Args) ->
permanent,
brutal_kill,
worker,
- dynamic},
+ [couch_replicator_manager]},
{couch_replicator_job_sup,
{couch_replicator_job_sup, start_link, []},
permanent,
[12/50] git commit: Do a better job of breaking the race
Posted by da...@apache.org.
Do a better job of breaking the race
BugzID: 13405
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/11d8c162
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/11d8c162
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/11d8c162
Branch: refs/heads/import
Commit: 11d8c16202859870dff8def6cfdbcc97ac067a38
Parents: e1b7cc1
Author: Bob Dionne <bo...@cloudant.com>
Authored: Thu Aug 2 13:24:29 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/11d8c162/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index d02f5e6..ccafb97 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -141,11 +141,8 @@ async_replicate(#rep{id = {BaseId, Ext}, source = Src, target = Tgt} = Rep) ->
%% this error occurs if multiple replicators are racing
%% each other to start and somebody else won. Just grab
%% the Pid by calling start_child again.
- {error, {already_started, Pid}} =
- supervisor:start_child(couch_replicator_job_sup, ChildSpec),
- twig:log(notice,"replication `~s` already running at ~p (`~s` -> `~s`)",
- [RepChildId, Pid, Source, Target]),
- {ok, Pid};
+ timer:sleep(50 + random:uniform(100)),
+ async_replicate(Rep);
{error, {'EXIT', {badarg,
[{erlang, apply, [gen_server, start_link, undefined]} | _]}}} ->
% Clause to deal with a change in the supervisor module introduced
[24/50] git commit: Calculate progress for sharded replication too
Posted by da...@apache.org.
Calculate progress for sharded replication too
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/1d7a3711
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/1d7a3711
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/1d7a3711
Branch: refs/heads/import
Commit: 1d7a37110fa8c9616268f873c656f0ffc8777579
Parents: a2756e1
Author: Robert Newson <ro...@cloudant.com>
Authored: Sun Sep 2 16:13:18 2012 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:39 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 22 +++++++++++++---------
1 file changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/1d7a3711/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 95e5241..5e4f6d5 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -941,16 +941,11 @@ update_task(State) ->
couch_task_status:update(
rep_stats(State) ++ [
{source_seq, SourceCurSeq},
- case is_number(CurSeq) andalso is_number(SourceCurSeq) of
- true ->
- case SourceCurSeq of
- 0 ->
+ case {unpack_seq(CurSeq), unpack_seq(SourceCurSeq)} of
+ {0, _} ->
{progress, 0};
- _ ->
- {progress, (CurSeq * 100) div SourceCurSeq}
- end;
- false ->
- {progress, null}
+ {CurSeq1, SourceCurSeq1} ->
+ {progress, (CurSeq1 * 100) div SourceCurSeq1}
end
]).
@@ -969,3 +964,12 @@ rep_stats(State) ->
{checkpointed_source_seq, CommittedSeq}
].
+unpack_seq(Seq) when is_number(Seq) ->
+ Seq;
+unpack_seq([SeqNum, _]) ->
+ SeqNum;
+unpack_seq(Seq) when is_binary(Seq) ->
+ Pattern = "^[\\[?]?(?<seqnum>[0-9]+)",
+ Options = [{capture, [seqnum], list}],
+ {match, [SeqNum]} = re:run(Seq, Pattern, Options),
+ list_to_integer(SeqNum).
[49/50] git commit: Remove binary rebar and obsolete config
Posted by da...@apache.org.
Remove binary rebar and obsolete config
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/93c70698
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/93c70698
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/93c70698
Branch: refs/heads/import
Commit: 93c7069884958444c81db6ad89326d35e41eea14
Parents: 9921223
Author: Robert Newson <rn...@apache.org>
Authored: Mon Dec 16 17:48:43 2013 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Mon Dec 16 17:48:43 2013 +0100
----------------------------------------------------------------------
rebar | Bin 100732 -> 0 bytes
rebar.config | 15 ---------------
2 files changed, 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/93c70698/rebar
----------------------------------------------------------------------
diff --git a/rebar b/rebar
deleted file mode 100755
index 30c43ba..0000000
Binary files a/rebar and /dev/null differ
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/93c70698/rebar.config
----------------------------------------------------------------------
diff --git a/rebar.config b/rebar.config
deleted file mode 100644
index 48deb03..0000000
--- a/rebar.config
+++ /dev/null
@@ -1,15 +0,0 @@
-% Licensed under the Apache License, Version 2.0 (the "License"); you may not
-% use this file except in compliance with the License. You may obtain a copy of
-% the License at
-%
-% http://www.apache.org/licenses/LICENSE-2.0
-%
-% Unless required by applicable law or agreed to in writing, software
-% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-% License for the specific language governing permissions and limitations under
-% the License.
-
-{deps, [
- {twig, ".*", {git, "https://github.com/cloudant/twig.git", {tag, "0.2.1"}}}
-]}.
[11/50] git commit: Do not block supervisor for replicator init
Posted by da...@apache.org.
Do not block supervisor for replicator init
This prevents the replication supervisor rom backing up under load and
and causing all sorts of other problems
BugzID: 13405
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/97bfe8e7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/97bfe8e7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/97bfe8e7
Branch: refs/heads/import
Commit: 97bfe8e75b388164dc3dfb2143d82a789ed6692a
Parents: 11d8c16
Author: Bob Dionne <bo...@cloudant.com>
Authored: Thu Aug 2 14:24:20 2012 -0400
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:37 2013 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 28 ++++++++++++++--------------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/97bfe8e7/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index ccafb97..6639edb 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -226,17 +226,7 @@ cancel_replication(RepId, #user_ctx{name = Name, roles = Roles}) ->
end.
init(InitArgs) ->
- try
- do_init(InitArgs)
- catch
- throw:{unauthorized, DbUri} ->
- {stop, {unauthorized,
- <<"unauthorized to access or create database ", DbUri/binary>>}};
- throw:{db_not_found, DbUri} ->
- {stop, {db_not_found, <<"could not open ", DbUri/binary>>}};
- throw:Error ->
- {stop, Error}
- end.
+ {ok, InitArgs, 0}.
do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
process_flag(trap_exit, true),
@@ -389,8 +379,15 @@ handle_info({'EXIT', Pid, Reason}, #rep_state{workers = Workers} = State) ->
true ->
twig:log(error,"Worker ~p died with reason: ~p", [Pid, Reason]),
{stop, {worker_died, Pid, Reason}, State2}
- end.
+ end;
+handle_info(timeout, InitArgs) ->
+ try do_init(InitArgs) of {ok, State} ->
+ {noreply, State}
+ catch Class:Error ->
+ Stack = erlang:get_stacktrace(),
+ {stop, shutdown, {error, Class, Error, Stack, InitArgs}}
+ end.
handle_call(get_details, _From, #rep_state{rep_details = Rep} = State) ->
{reply, {ok, Rep}, State};
@@ -474,7 +471,7 @@ terminate(shutdown, #rep_state{rep_details = #rep{id = RepId}} = State) ->
couch_replicator_notifier:notify({error, RepId, <<"cancelled">>}),
terminate_cleanup(State);
-terminate(Reason, State) ->
+terminate(Reason, #rep_state{} = State) ->
#rep_state{
source_name = Source,
target_name = Target,
@@ -484,8 +481,11 @@ terminate(Reason, State) ->
[BaseId ++ Ext, Source, Target, to_binary(Reason)]),
terminate_cleanup(State),
couch_replicator_notifier:notify({error, RepId, Reason}),
- couch_replicator_manager:replication_error(Rep, Reason).
+ couch_replicator_manager:replication_error(Rep, Reason);
+terminate(shutdown, {error, Class, Error, Stack, InitArgs}) ->
+ twig:log("~p:~p: Replication failed to start for args ~p: ~p",
+ [Class, Error, InitArgs, Stack]).
terminate_cleanup(State) ->
update_task(State),
[50/50] git commit: Build with rebar
Posted by da...@apache.org.
Build with rebar
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7e6f0feb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7e6f0feb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7e6f0feb
Branch: refs/heads/import
Commit: 7e6f0feb84223ac2c6b7c0b3ec588717cff1d1da
Parents: 93c7069
Author: Robert Newson <rn...@apache.org>
Authored: Wed Dec 18 14:04:59 2013 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Dec 18 23:04:18 2013 +0000
----------------------------------------------------------------------
.gitignore | 4 ---
Makefile.am | 76 ---------------------------------------
src/couch_replicator.app.src | 2 +-
3 files changed, 1 insertion(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7e6f0feb/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
deleted file mode 100644
index b3099f5..0000000
--- a/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-*.beam
-.eunit
-ebin/replicator.app
-.DS_Store
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7e6f0feb/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
deleted file mode 100644
index ea5d117..0000000
--- a/Makefile.am
+++ /dev/null
@@ -1,76 +0,0 @@
-## Licensed under the Apache License, Version 2.0 (the "License"); you may not
-## use this file except in compliance with the License. You may obtain a copy of
-## the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-## License for the specific language governing permissions and limitations under
-## the License.
-
-couch_replicatorlibdir = $(localerlanglibdir)/couch_replicator-0.1
-couch_replicatorincludedir = $(couch_replicatorlibdir)/include
-couch_replicatorebindir = $(couch_replicatorlibdir)/ebin
-
-couch_replicatorinclude_DATA = $(include_files)
-couch_replicatorebin_DATA = $(compiled_files)
-
-include_files = \
- src/couch_replicator_api_wrap.hrl \
- src/couch_replicator.hrl \
- src/couch_replicator_js_functions.hrl
-
-source_files = \
- src/couch_replicator_api_wrap.erl \
- src/couch_replicator_httpc_pool.erl \
- src/couch_replicator_httpc.erl \
- src/couch_replicator_httpd.erl \
- src/couch_replicator_job_sup.erl \
- src/couch_replicator_notifier.erl \
- src/couch_replicator_manager.erl \
- src/couch_replicator_utils.erl \
- src/couch_replicator_worker.erl \
- src/couch_replicator.app.src \
- src/couch_replicator.erl
-
-test_files = \
- test/01-load.t \
- test/02-httpc-pool.t \
- test/03-replication-compact.t \
- test/04-replication-large-atts.t \
- test/05-replication-many-leaves.t \
- test/06-doc-missing-stubs.t
-
-compiled_files = \
- ebin/couch_replicator_api_wrap.beam \
- ebin/couch_replicator_httpc_pool.beam \
- ebin/couch_replicator_httpc.beam \
- ebin/couch_replicator_httpd.beam \
- ebin/couch_replicator_job_sup.beam \
- ebin/couch_replicator_notifier.beam \
- ebin/couch_replicator_manager.beam \
- ebin/couch_replicator_utils.beam \
- ebin/couch_replicator_worker.beam \
- ebin/couch_replicator.app \
- ebin/couch_replicator.beam
-
-EXTRA_DIST = $(include_files) $(source_files) $(test_files)
-CLEANFILES = $(compiled_files)
-
-check:
-if TESTS
- $(abs_top_builddir)/test/etap/run $(abs_top_srcdir)/src/couch_replicator/test
-endif
-
-ebin/%.app: src/%.app.src
- @mkdir -p ebin/
- sed -e "s|%version%|@version@|g" \
- < $< > $@
-
-ebin/%.beam: src/%.erl $(include_files)
- @mkdir -p ebin/
- $(ERLC) -Wall -I$(top_srcdir)/src -o ebin/ $(ERLC_FLAGS) ${TEST} $<;
-
-
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7e6f0feb/src/couch_replicator.app.src
----------------------------------------------------------------------
diff --git a/src/couch_replicator.app.src b/src/couch_replicator.app.src
index bd6b3e3..2da0d4a 100644
--- a/src/couch_replicator.app.src
+++ b/src/couch_replicator.app.src
@@ -12,7 +12,7 @@
{application, couch_replicator, [
{description, "CouchDB replicator"},
- {vsn, "%version%"},
+ {vsn, git},
{modules, [
couch_replicator,
couch_replicator_api_wrap,
[42/50] git commit: Remove second fd from couch_db_updater
Posted by da...@apache.org.
Remove second fd from couch_db_updater
This reverts the change to use a second file descriptor for
couch_db_udpaters to split read and writes. This is ground work for the
next few patches that change how databases are managed.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/0e9e5b69
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/0e9e5b69
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/0e9e5b69
Branch: refs/heads/import
Commit: 0e9e5b6975e9b70c431c9d1834f068d9fa3dfb6a
Parents: 3160f54
Author: Paul Joseph Davis <da...@apache.org>
Authored: Tue Mar 20 00:28:26 2012 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:42 2013 -0500
----------------------------------------------------------------------
test/03-replication-compact.t | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/0e9e5b69/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 7c4d38c..e3f59fc 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -30,7 +30,6 @@
compactor_pid = nil,
instance_start_time, % number of microsecs since jan 1 1970 as a binary string
fd,
- updater_fd,
fd_ref_counter,
header = nil,
committed_update_seq,
[46/50] git commit: [squash] to etap updates
Posted by da...@apache.org.
[squash] to etap updates
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/53e6dbec
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/53e6dbec
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/53e6dbec
Branch: refs/heads/import
Commit: 53e6dbec4203ee77e5c5ba1d22844489bf13fe74
Parents: 6f0123b
Author: Paul J. Davis <pa...@gmail.com>
Authored: Thu Mar 14 02:06:36 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:57 2013 -0500
----------------------------------------------------------------------
test/02-httpc-pool.t | 14 ++------------
test/03-replication-compact.t | 23 ++++++++---------------
test/04-replication-large-atts.t | 17 +++--------------
test/05-replication-many-leaves.t | 17 +++--------------
test/06-doc-missing-stubs.t | 17 +++--------------
5 files changed, 19 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/53e6dbec/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index 23d7ab7..fc86cce 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -13,21 +13,11 @@
% the License.
main(_) ->
- test_util:init_code_path(),
-
- etap:plan(55),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
+ test_util:run(55, fun() -> test() end).
test() ->
- couch_server_sup:start_link(test_util:config_files()),
+ test_util:start_couch(),
ibrowse:start(),
test_pool_full(),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/53e6dbec/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 619d2b4..888d123 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -16,6 +16,8 @@
% target of a replication doesn't affect the replication and that the
% replication doesn't hold their reference counters forever.
+-mode(compile).
+
-define(b2l(B), binary_to_list(B)).
-record(user_ctx, {
@@ -66,21 +68,11 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
main(_) ->
- test_util:init_code_path(),
-
- etap:plan(376),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
+ test_util:run(376, fun() -> test() end).
test() ->
- couch_server_sup:start_link(test_util:config_files()),
+ test_util:start_couch(),
ibrowse:start(),
Pairs = [
@@ -112,13 +104,13 @@ test() ->
delete_db(SourceDb),
delete_db(TargetDb),
- couch_server_sup:stop(),
+ ok = test_util:stop_couch(),
ok = timer:sleep(1000),
- couch_server_sup:start_link(test_util:config_files())
+ ok = test_util:start_couch()
end,
Pairs),
- couch_server_sup:stop(),
+ ok = test_util:stop_couch(),
ok.
@@ -284,6 +276,7 @@ compare_dbs(#db{name = SourceName}, #db{name = TargetName}) ->
check_active_tasks(RepPid, {BaseId, Ext} = _RepId, Src, Tgt) ->
+ timer:sleep(1000),
Source = case Src of
{remote, NameSrc} ->
<<(db_url(NameSrc))/binary, $/>>;
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/53e6dbec/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index caf2f67..9adaead 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -49,21 +49,11 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
main(_) ->
- test_util:init_code_path(),
-
- etap:plan(1192),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
+ test_util:run(1192, fun() -> test() end).
test() ->
- couch_server_sup:start_link(test_util:config_files()),
+ test_util:start_couch(),
ibrowse:start(),
crypto:start(),
config:set("attachments", "compressible_types", "text/*", false),
@@ -98,8 +88,7 @@ test() ->
Pairs),
delete_db(SourceDb),
- couch_server_sup:stop(),
- ok.
+ test_util:stop_couch().
populate_db(Db, DocCount) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/53e6dbec/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index be63d41..bd795ec 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -61,21 +61,11 @@ doc_num_conflicts(<<"doc3">>) -> 210.
main(_) ->
- test_util:init_code_path(),
-
- etap:plan(56),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
+ test_util:run(56, fun() -> test() end).
test() ->
- couch_server_sup:start_link(test_util:config_files()),
+ test_util:start_couch(),
ibrowse:start(),
crypto:start(),
config:set("replicator", "connection_timeout", "90000", false),
@@ -125,8 +115,7 @@ test() ->
end,
Pairs),
- couch_server_sup:stop(),
- ok.
+ test_util:stop_couch().
populate_db(Db) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/53e6dbec/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index 0fffec2..116550c 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -52,22 +52,12 @@ target_revs_limit() -> 3.
main(_) ->
- test_util:init_code_path(),
-
- etap:plan(128),
- case (catch test()) of
- ok ->
- etap:end_tests();
- Other ->
- etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
- etap:bail(Other)
- end,
- ok.
+ test_util:run(128, fun() -> test() end).
% Test motivated by COUCHDB-1365.
test() ->
- couch_server_sup:start_link(test_util:config_files()),
+ test_util:start_couch(),
ibrowse:start(),
Pairs = [
@@ -109,8 +99,7 @@ test() ->
end,
Pairs),
- couch_server_sup:stop(),
- ok.
+ test_util:stop_couch().
populate_db(Db) ->
[45/50] git commit: Rewrite the boot procedure for the couch app
Posted by da...@apache.org.
Rewrite the boot procedure for the couch app
We had a lot of historical cruft lying around from before people knew
hot OTP worked. This does some cleanup to make things a bit more tidy
through this section but we still have a long ways to go.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b4d569f0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b4d569f0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b4d569f0
Branch: refs/heads/import
Commit: b4d569f04002ce68e968816cad30c57cad62c571
Parents: 9f968dc
Author: Paul J. Davis <pa...@gmail.com>
Authored: Mon Mar 11 16:13:14 2013 -0500
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Wed Mar 20 06:02:54 2013 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b4d569f0/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 93f0e89..c213199 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -226,6 +226,9 @@ handle_info(restart_config_listener, State) ->
ok = config:listen_for_changes(?MODULE, self()),
{noreply, State};
+handle_info(shutdown, State) ->
+ {stop, shutdown, State};
+
handle_info(Msg, State) ->
twig:log(error,"Replication manager received unexpected message ~p", [Msg]),
{stop, {unexpected_msg, Msg}, State}.