You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by va...@apache.org on 2017/03/14 20:49:59 UTC
[1/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Repository: couchdb-couch-replicator
Updated Branches:
refs/heads/63012-scheduler 27a5eae90 -> eda73a2c9
Remove unused mp_parse_doc function from replicator
It was left accidentally when merging Cloudant's dbcore work.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/30915e33
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/30915e33
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/30915e33
Branch: refs/heads/63012-scheduler
Commit: 30915e3309fb30c2164e668d33dbd393e77925c0
Parents: 50a88ba
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 14:38:29 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 14:38:29 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_utils.erl | 30 ------------------------------
1 file changed, 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/30915e33/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 223bac8..e96d52a 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -17,7 +17,6 @@
-export([start_db_compaction_notifier/2, stop_db_compaction_notifier/1]).
-export([replication_id/2]).
-export([sum_stats/2, is_deleted/1]).
--export([mp_parse_doc/2]).
-export([handle_db_event/3]).
@@ -449,35 +448,6 @@ handle_db_event(_DbName, _Event, Server) ->
sum_stats(S1, S2) ->
couch_replicator_stats:sum_stats(S1, S2).
-mp_parse_doc({headers, H}, []) ->
- case couch_util:get_value("content-type", H) of
- {"application/json", _} ->
- fun (Next) ->
- mp_parse_doc(Next, [])
- end
- end;
-mp_parse_doc({body, Bytes}, AccBytes) ->
- fun (Next) ->
- mp_parse_doc(Next, [Bytes | AccBytes])
- end;
-mp_parse_doc(body_end, AccBytes) ->
- receive {get_doc_bytes, Ref, From} ->
- From ! {doc_bytes, Ref, lists:reverse(AccBytes)}
- end,
- fun mp_parse_atts/1.
-
-mp_parse_atts(eof) ->
- ok;
-mp_parse_atts({headers, _H}) ->
- fun mp_parse_atts/1;
-mp_parse_atts({body, Bytes}) ->
- receive {get_bytes, Ref, From} ->
- From ! {bytes, Ref, Bytes}
- end,
- fun mp_parse_atts/1;
-mp_parse_atts(body_end) ->
- fun mp_parse_atts/1.
-
is_deleted(Change) ->
case couch_util:get_value(<<"deleted">>, Change) of
undefined ->
[4/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Merge branch '64229-add-new-request-parameter'
Closes #61
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ffe9697e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ffe9697e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ffe9697e
Branch: refs/heads/63012-scheduler
Commit: ffe9697e728c4725f5ff8915cf592d2d4d0074cc
Parents: 1166759 6495809
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 19:17:26 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 19:17:26 2017 -0500
----------------------------------------------------------------------
test/couch_replicator_small_max_request_size_target.erl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[8/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Merge branch 'master' into 63012-scheduler
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/eda73a2c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/eda73a2c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/eda73a2c
Branch: refs/heads/63012-scheduler
Commit: eda73a2c93fc075efd72b868c32eb00540d291b0
Parents: 27a5eae d00b981
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 14 16:49:46 2017 -0400
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 14 16:49:46 2017 -0400
----------------------------------------------------------------------
test/couch_replicator_small_max_request_size_target.erl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[7/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Prevent replicator manager change feeds from getting stuck
Switch them them from `longpoll` to `normal`
This would prevent them being stuck. That could happen if more than one
`resume_scan` message arrives for the same shard. The first time a longpoll
changef feed would finish and end sequence is checkpointed. But if another
resume_scan arrives and database hasn't changed then the longpoll change
feed would hang until db is updated.
The reason there would be multiple `resume_scan` messages is because there
is a race condition between db update handler and scanner component. They are
both started asynchronously roughly at the same. Scanner finds new shard while
db handler notices changes for those shards. If shards are modified quickly
after they are discovered by the scanner both of those components would issue
a resume_scan.
The effect of this would be more pronounced if there are a large number of
_replicator shards and constant db creation/deletion/updates.
COUCHDB-2964
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/d00b9814
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/d00b9814
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/d00b9814
Branch: refs/heads/63012-scheduler
Commit: d00b981445c03622497088eb872059ab4f48b298
Parents: f63efa7
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Mar 10 01:15:47 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Fri Mar 10 01:15:47 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/d00b9814/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 2bcad69..85dd428 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -375,7 +375,7 @@ changes_reader({Server, Epoch}, DbName, Since) ->
#changes_args{
include_docs = true,
since = Since,
- feed = "longpoll",
+ feed = "normal",
timeout = infinity
},
{json_req, null},
[2/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Merge remote-tracking branch 'cloudant/couchdb-2992-remove-dead-code'
Closes #60
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/11667591
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/11667591
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/11667591
Branch: refs/heads/63012-scheduler
Commit: 11667591c1211ef2817945393ec22e0361d900ad
Parents: 50a88ba 30915e3
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 14:45:59 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 14:45:59 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_utils.erl | 30 ------------------------------
1 file changed, 30 deletions(-)
----------------------------------------------------------------------
[5/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Don't scan empty replicator databases
Every account gets a _replicator database created by default, the
burden of scanning them all is considerable.
Don't start a changes reader if the database is empty (excluding the
injected _design/replicator design document)
BugzID: 84311
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/46aa27fa
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/46aa27fa
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/46aa27fa
Branch: refs/heads/63012-scheduler
Commit: 46aa27fa674a4c1e590aeecd76123e4f91d78fd5
Parents: ffe9697
Author: Robert Newson <rn...@apache.org>
Authored: Thu Mar 9 18:03:14 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Thu Mar 9 18:35:22 2017 +0000
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 46 ++++++++++++++++++++++++++++++++---
1 file changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/46aa27fa/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 2bcad69..9f17087 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -266,9 +266,17 @@ handle_cast({resume_scan, DbName}, State) ->
end,
true = ets:insert(?DB_TO_SEQ, {DbName, Since, false}),
ensure_rep_ddoc_exists(DbName),
- Pid = start_changes_reader(DbName, Since, State#state.epoch),
- couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
- [{DbName, Pid} | Pids]
+ case has_replication_docs(DbName) of
+ false ->
+ %% the database is empty save for the rep_ddoc,
+ %% don't bother scanning it.
+ couch_log:debug("ignoring empty ~s", [DbName]),
+ Pids;
+ true ->
+ Pid = start_changes_reader(DbName, Since, State#state.epoch),
+ couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
+ [{DbName, Pid} | Pids]
+ end
end,
{noreply, State#state{rep_start_pids = NewPids}};
@@ -1005,6 +1013,38 @@ get_json_value(Key, Props, Default) when is_binary(Key) ->
end.
+has_replication_docs(DbName) ->
+ {ok, Db} = couch_db:open(DbName, []),
+ try
+ case couch_db:get_doc_count(Db) of
+ {ok, 0} ->
+ false;
+ {ok, 1} ->
+ case first_doc_id(Db) of
+ <<"_design/_replicator">> ->
+ false;
+ _Else ->
+ true
+ end;
+ _Else ->
+ true
+ end
+ after
+ couch_db:close(Db)
+ end.
+
+
+first_doc_id(#db{} = Db) ->
+ Fun = fun
+ (#full_doc_info{deleted = true}, _Reds, Acc) ->
+ {ok, Acc};
+ (FDI, _Reds, _Acc) ->
+ {stop, FDI#full_doc_info.id}
+ end,
+ {ok, _, Id} = couch_btree:fold(Db#db.id_tree, Fun, nil, []),
+ Id.
+
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
[6/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Revert "Don't scan empty replicator databases"
This reverts commit 46aa27fa674a4c1e590aeecd76123e4f91d78fd5.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/f63efa78
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/f63efa78
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/f63efa78
Branch: refs/heads/63012-scheduler
Commit: f63efa786fec5cc40e6c6193470399da99385abf
Parents: 46aa27f
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Mar 10 01:13:28 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Fri Mar 10 01:13:28 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 46 +++--------------------------------
1 file changed, 3 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/f63efa78/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 9f17087..2bcad69 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -266,17 +266,9 @@ handle_cast({resume_scan, DbName}, State) ->
end,
true = ets:insert(?DB_TO_SEQ, {DbName, Since, false}),
ensure_rep_ddoc_exists(DbName),
- case has_replication_docs(DbName) of
- false ->
- %% the database is empty save for the rep_ddoc,
- %% don't bother scanning it.
- couch_log:debug("ignoring empty ~s", [DbName]),
- Pids;
- true ->
- Pid = start_changes_reader(DbName, Since, State#state.epoch),
- couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
- [{DbName, Pid} | Pids]
- end
+ Pid = start_changes_reader(DbName, Since, State#state.epoch),
+ couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
+ [{DbName, Pid} | Pids]
end,
{noreply, State#state{rep_start_pids = NewPids}};
@@ -1013,38 +1005,6 @@ get_json_value(Key, Props, Default) when is_binary(Key) ->
end.
-has_replication_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, []),
- try
- case couch_db:get_doc_count(Db) of
- {ok, 0} ->
- false;
- {ok, 1} ->
- case first_doc_id(Db) of
- <<"_design/_replicator">> ->
- false;
- _Else ->
- true
- end;
- _Else ->
- true
- end
- after
- couch_db:close(Db)
- end.
-
-
-first_doc_id(#db{} = Db) ->
- Fun = fun
- (#full_doc_info{deleted = true}, _Reds, Acc) ->
- {ok, Acc};
- (FDI, _Reds, _Acc) ->
- {stop, FDI#full_doc_info.id}
- end,
- {ok, _, Id} = couch_btree:fold(Db#db.id_tree, Fun, nil, []),
- Id.
-
-
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
[3/8] couch-replicator commit: updated refs/heads/63012-scheduler to
eda73a2
Posted by va...@apache.org.
Fix unit test after renaming max_document_size config parameter
`couchdb.max_document_size` was renamed to `httpd.max_http_request_size`
The unit tests was testing how replicator behaves when faced with reduced
request size configuration on the target.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/64958096
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/64958096
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/64958096
Branch: refs/heads/63012-scheduler
Commit: 64958096d4f9a940c01cbc472da5265f349c9545
Parents: 1166759
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 16:42:43 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 16:42:43 2017 -0500
----------------------------------------------------------------------
test/couch_replicator_small_max_request_size_target.erl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/64958096/test/couch_replicator_small_max_request_size_target.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_small_max_request_size_target.erl b/test/couch_replicator_small_max_request_size_target.erl
index c46619d..b7d9fbe 100644
--- a/test/couch_replicator_small_max_request_size_target.erl
+++ b/test/couch_replicator_small_max_request_size_target.erl
@@ -27,7 +27,7 @@ setup(remote) ->
setup({A, B}) ->
Ctx = test_util:start_couch([couch_replicator]),
- config:set("couchdb", "max_document_size", "10000", false),
+ config:set("httpd", "max_http_request_size", "10000", false),
Source = setup(A),
Target = setup(B),
{Ctx, {Source, Target}}.
@@ -49,7 +49,7 @@ teardown(_, {Ctx, {Source, Target}}) ->
reduce_max_request_size_test_() ->
Pairs = [{local, remote}, {remote, remote}],
{
- "Replicate docs when target has a small max_document_size",
+ "Replicate docs when target has a small max_http_request_size",
{
foreachx,
fun setup/1, fun teardown/2,
@@ -66,8 +66,8 @@ reduce_max_request_size_test_() ->
}.
-% Test documents which are below max_document_size but when batched, batch size
-% will be greater than max_document_size. Replicator could automatically split
+% Test documents which are below max_http_request_size but when batched, batch size
+% will be greater than max_http_request_size. Replicator could automatically split
% the batch into smaller batches and POST those separately.
should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) ->
{lists:flatten(io_lib:format("~p -> ~p", [From, To])),