You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2017/04/04 21:18:04 UTC
[01/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab [Forced Update!]
Repository: couchdb-couch-replicator
Updated Branches:
refs/heads/COUCHDB-3287-pluggable-storage-engines bfcda98e4 -> 4e45eab60 (forced update)
Allow configuring maximum document ID length during replication
Currently due to a bug in http parser and lack of document ID length
enforcement, large document IDs will break replication jobs. Large IDs
will pass through the _change feed, revs diffs, but then fail
during open_revs get request. open_revs request will keep retrying until
it gives up after long enough time, then replication task crashes and
restart again with the same pattern. The current effective limit is
around 8k or so. (The buffer size default 8192 and if the first line
of the request is larger than that, request will fail).
(See http://erlang.org/pipermail/erlang-questions/2011-June/059567.html
for more information about the possible failure mechanism).
Bypassing the parser bug by increasing recbuf size, will alow replication
to finish, however that means simply spreading the abnormal document through
the rest of the system, and might not be desirable always.
Also once long document IDs have been inserted in the source DB. Simply deleting
them doesn't work as they'd still appear in the change feed. They'd have to
be purged or somehow skipped during the replication step. This commit helps
do the later.
Operators can configure maximum length via this setting:
```
replicator.max_document_id_length=0
```
The default value is 0 which means there is no maximum enforced, which is
backwards compatible behavior.
During replication if maximum is hit by a document, that document is skipped,
an error is written to the log:
```
Replicator: document id `aaaaaaaaaaaaaaaaaaaaa...` from source db `http://.../cdyno-0000001/` is too long, ignoring.
```
and `"doc_write_failures"` statistic is bumped.
COUCHDB-3291
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/d23025eb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/d23025eb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/d23025eb
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: d23025ebd7176f6c307ddf49902cf20b33bd55c4
Parents: be0060f
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Feb 3 20:49:32 2017 -0500
Committer: ILYA Khlopotov <ii...@apache.org>
Committed: Mon Feb 6 12:18:02 2017 -0800
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 20 ++++-
test/couch_replicator_id_too_long_tests.erl | 94 ++++++++++++++++++++++++
2 files changed, 111 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/d23025eb/src/couch_replicator_changes_reader.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_changes_reader.erl b/src/couch_replicator_changes_reader.erl
index bed318a..f9d9097 100644
--- a/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator_changes_reader.erl
@@ -89,9 +89,19 @@ process_change(#doc_info{id = <<>>} = DocInfo, {_, Db, _, _, _}) ->
"source database `~s` (_changes sequence ~p)",
[couch_replicator_api_wrap:db_uri(Db), DocInfo#doc_info.high_seq]);
-process_change(#doc_info{} = DocInfo, {_, _, ChangesQueue, _, _}) ->
- ok = couch_work_queue:queue(ChangesQueue, DocInfo),
- put(last_seq, DocInfo#doc_info.high_seq);
+process_change(#doc_info{id = Id} = DocInfo, {Parent, Db, ChangesQueue, _, _}) ->
+ case is_doc_id_too_long(byte_size(Id)) of
+ true ->
+ ShortId = lists:sublist(binary_to_list(Id), 64),
+ SourceDb = couch_replicator_api_wrap:db_uri(Db),
+ couch_log:error("Replicator: document id `~s...` from source db "
+ " `~s` is too long, ignoring.", [ShortId, SourceDb]),
+ Stats = couch_replicator_stats:new([{doc_write_failures, 1}]),
+ ok = gen_server:call(Parent, {add_stats, Stats}, infinity);
+ false ->
+ ok = couch_work_queue:queue(ChangesQueue, DocInfo),
+ put(last_seq, DocInfo#doc_info.high_seq)
+ end;
process_change({last_seq, LS}, {Parent, _, _, true = _Continuous, Ts}) ->
% LS should never be undefined, but it doesn't hurt to be defensive inside
@@ -111,3 +121,7 @@ process_change({last_seq, _}, _) ->
% change. The two can differ substantially in the case of a restrictive
% filter.
ok.
+
+is_doc_id_too_long(IdLength) ->
+ ConfigMax = config:get_integer("replicator", "max_document_id_length", 0),
+ ConfigMax > 0 andalso IdLength > ConfigMax.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/d23025eb/test/couch_replicator_id_too_long_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_id_too_long_tests.erl b/test/couch_replicator_id_too_long_tests.erl
new file mode 100644
index 0000000..f5d7165
--- /dev/null
+++ b/test/couch_replicator_id_too_long_tests.erl
@@ -0,0 +1,94 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_id_too_long_tests).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+-include_lib("couch_replicator/src/couch_replicator.hrl").
+
+
+setup(_) ->
+ Ctx = test_util:start_couch([couch_replicator]),
+ Source = create_db(),
+ create_doc(Source),
+ Target = create_db(),
+ {Ctx, {Source, Target}}.
+
+
+teardown(_, {Ctx, {Source, Target}}) ->
+ delete_db(Source),
+ delete_db(Target),
+ config:set("replicator", "max_document_id_length", "0"),
+ ok = test_util:stop_couch(Ctx).
+
+
+id_too_long_replication_test_() ->
+ Pairs = [{local, local}, {local, remote},
+ {remote, local}, {remote, remote}],
+ {
+ "Doc id too long tests",
+ {
+ foreachx,
+ fun setup/1, fun teardown/2,
+ [{Pair, fun should_succeed/2} || Pair <- Pairs] ++
+ [{Pair, fun should_fail/2} || Pair <- Pairs]
+ }
+ }.
+
+
+should_succeed({From, To}, {_Ctx, {Source, Target}}) ->
+ RepObject = {[
+ {<<"source">>, db_url(From, Source)},
+ {<<"target">>, db_url(To, Target)}
+ ]},
+ config:set("replicator", "max_document_id_length", "5"),
+ {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ ?_assertEqual(ok, couch_replicator_test_helper:compare_dbs(Source, Target)).
+
+
+should_fail({From, To}, {_Ctx, {Source, Target}}) ->
+ RepObject = {[
+ {<<"source">>, db_url(From, Source)},
+ {<<"target">>, db_url(To, Target)}
+ ]},
+ config:set("replicator", "max_document_id_length", "4"),
+ {ok, _} = couch_replicator:replicate(RepObject, ?ADMIN_USER),
+ ?_assertError({badmatch, {not_found, missing}},
+ couch_replicator_test_helper:compare_dbs(Source, Target)).
+
+
+create_db() ->
+ DbName = ?tempdb(),
+ {ok, Db} = couch_db:create(DbName, [?ADMIN_CTX]),
+ ok = couch_db:close(Db),
+ DbName.
+
+
+create_doc(DbName) ->
+ {ok, Db} = couch_db:open(DbName, [?ADMIN_CTX]),
+ Doc = couch_doc:from_json_obj({[{<<"_id">>, <<"12345">>}]}),
+ {ok, _} = couch_db:update_doc(Db, Doc, []),
+ couch_db:ensure_full_commit(Db),
+ couch_db:close(Db).
+
+
+delete_db(DbName) ->
+ ok = couch_server:delete(DbName, [?ADMIN_CTX]).
+
+
+db_url(local, DbName) ->
+ DbName;
+db_url(remote, DbName) ->
+ Addr = config:get("httpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(couch_httpd, port),
+ ?l2b(io_lib:format("http://~s:~b/~s", [Addr, Port, DbName])).
[03/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Switch replicator max_document_id_length config to use infinity
Default value switched to be `infinity` instead of 0
COUCHDB-3291
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/46f70c73
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/46f70c73
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/46f70c73
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 46f70c73427e618774872a388287ba682c1376f1
Parents: b18b31c
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Feb 8 11:46:13 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Feb 8 11:46:13 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 9 +++++++--
test/couch_replicator_id_too_long_tests.erl | 2 +-
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/46f70c73/src/couch_replicator_changes_reader.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_changes_reader.erl b/src/couch_replicator_changes_reader.erl
index f9d9097..439f5c9 100644
--- a/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator_changes_reader.erl
@@ -123,5 +123,10 @@ process_change({last_seq, _}, _) ->
ok.
is_doc_id_too_long(IdLength) ->
- ConfigMax = config:get_integer("replicator", "max_document_id_length", 0),
- ConfigMax > 0 andalso IdLength > ConfigMax.
+ case config:get("replicator", "max_document_id_length", "infinity") of
+ "infinity" ->
+ false;
+ ConfigMaxStr ->
+ ConfigMax = list_to_integer(ConfigMaxStr),
+ ConfigMax > 0 andalso IdLength > ConfigMax
+ end.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/46f70c73/test/couch_replicator_id_too_long_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_id_too_long_tests.erl b/test/couch_replicator_id_too_long_tests.erl
index f5d7165..70eda05 100644
--- a/test/couch_replicator_id_too_long_tests.erl
+++ b/test/couch_replicator_id_too_long_tests.erl
@@ -28,7 +28,7 @@ setup(_) ->
teardown(_, {Ctx, {Source, Target}}) ->
delete_db(Source),
delete_db(Target),
- config:set("replicator", "max_document_id_length", "0"),
+ config:set("replicator", "max_document_id_length", "infinity"),
ok = test_util:stop_couch(Ctx).
[16/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Don't scan empty replicator databases
Every account gets a _replicator database created by default, the
burden of scanning them all is considerable.
Don't start a changes reader if the database is empty (excluding the
injected _design/replicator design document)
BugzID: 84311
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/46aa27fa
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/46aa27fa
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/46aa27fa
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 46aa27fa674a4c1e590aeecd76123e4f91d78fd5
Parents: ffe9697
Author: Robert Newson <rn...@apache.org>
Authored: Thu Mar 9 18:03:14 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Thu Mar 9 18:35:22 2017 +0000
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 46 ++++++++++++++++++++++++++++++++---
1 file changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/46aa27fa/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 2bcad69..9f17087 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -266,9 +266,17 @@ handle_cast({resume_scan, DbName}, State) ->
end,
true = ets:insert(?DB_TO_SEQ, {DbName, Since, false}),
ensure_rep_ddoc_exists(DbName),
- Pid = start_changes_reader(DbName, Since, State#state.epoch),
- couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
- [{DbName, Pid} | Pids]
+ case has_replication_docs(DbName) of
+ false ->
+ %% the database is empty save for the rep_ddoc,
+ %% don't bother scanning it.
+ couch_log:debug("ignoring empty ~s", [DbName]),
+ Pids;
+ true ->
+ Pid = start_changes_reader(DbName, Since, State#state.epoch),
+ couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
+ [{DbName, Pid} | Pids]
+ end
end,
{noreply, State#state{rep_start_pids = NewPids}};
@@ -1005,6 +1013,38 @@ get_json_value(Key, Props, Default) when is_binary(Key) ->
end.
+has_replication_docs(DbName) ->
+ {ok, Db} = couch_db:open(DbName, []),
+ try
+ case couch_db:get_doc_count(Db) of
+ {ok, 0} ->
+ false;
+ {ok, 1} ->
+ case first_doc_id(Db) of
+ <<"_design/_replicator">> ->
+ false;
+ _Else ->
+ true
+ end;
+ _Else ->
+ true
+ end
+ after
+ couch_db:close(Db)
+ end.
+
+
+first_doc_id(#db{} = Db) ->
+ Fun = fun
+ (#full_doc_info{deleted = true}, _Reds, Acc) ->
+ {ok, Acc};
+ (FDI, _Reds, _Acc) ->
+ {stop, FDI#full_doc_info.id}
+ end,
+ {ok, _, Id} = couch_btree:fold(Db#db.id_tree, Fun, nil, []),
+ Id.
+
+
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
[02/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
closes #54
Merge branch 'couchdb-3291-limit-doc-id-size-in-replicator'
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b18b31c6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b18b31c6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b18b31c6
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: b18b31c6350a6684fe1dc6400979473a9a14f418
Parents: be0060f d23025e
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Mon Feb 6 19:46:26 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Mon Feb 6 19:46:26 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 20 ++++-
test/couch_replicator_id_too_long_tests.erl | 94 ++++++++++++++++++++++++
2 files changed, 111 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[08/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Revert "Restore adding some jitter-ed sleep to shard scanning code."
This reverts commit 45d739af3fcf8b4f8e3ccca152cb3c2d781dc2fc.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/1e413b8a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/1e413b8a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/1e413b8a
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 1e413b8afa6eb156eb5293d1dc0d5726cd5063fc
Parents: 45d739a
Author: Robert Newson <rn...@apache.org>
Authored: Wed Mar 1 11:15:55 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Mar 1 11:15:55 2017 +0000
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 74 +++++++----------------------------
1 file changed, 14 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/1e413b8a/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 4e5073e..bdc3b8f 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -934,30 +934,22 @@ scan_all_dbs(Server) when is_pid(Server) ->
{ok, Db} = mem3_util:ensure_exists(
config:get("mem3", "shards_db", "_dbs")),
ChangesFun = couch_changes:handle_changes(#changes_args{}, nil, Db, nil),
- ChangesFun({fun scan_changes_cb/3, {Server, 1}}),
+ ChangesFun(fun({change, {Change}, _}, _) ->
+ DbName = couch_util:get_value(<<"id">>, Change),
+ case DbName of <<"_design/", _/binary>> -> ok; _Else ->
+ case couch_replicator_utils:is_deleted(Change) of
+ true ->
+ ok;
+ false ->
+ [gen_server:cast(Server, {resume_scan, ShardName})
+ || ShardName <- replicator_shards(DbName)],
+ ok
+ end
+ end;
+ (_, _) -> ok
+ end),
couch_db:close(Db).
-scan_changes_cb({change, {Change}, _}, _, {Server, AccCount}) ->
- DbName = couch_util:get_value(<<"id">>, Change),
- case DbName of <<"_design/", _/binary>> -> {Server, AccCount}; _Else ->
- case couch_replicator_utils:is_deleted(Change) of
- true ->
- {Server, AccCount};
- false ->
- UpdatedCount = lists:foldl(fun(ShardName, Count) ->
- spawn_link(fun() ->
- timer:sleep(jitter(Count)),
- gen_server:cast(Server, {resume_scan, ShardName})
- end),
- Count + 1
- end, AccCount, replicator_shards(DbName)),
- {Server, UpdatedCount}
- end
- end;
-
-scan_changes_cb(_, _, {Server, AccCount}) ->
- {Server, AccCount}.
-
replicator_shards(DbName) ->
case is_replicator_db(DbName) of
@@ -1035,42 +1027,4 @@ t_fail_non_replicator_shard() ->
end).
-scan_dbs_test_() ->
-{
- foreach,
- fun() -> test_util:start_couch([mem3, fabric]) end,
- fun(Ctx) -> test_util:stop_couch(Ctx) end,
- [
- t_resume_db_shard(),
- t_sleep_based_on_count()
- ]
-}.
-
-
-t_resume_db_shard() ->
- ?_test(begin
- DbName0 = ?tempdb(),
- DbName = <<DbName0/binary, "/_replicator">>,
- ok = fabric:create_db(DbName, [?CTX]),
- Change = {[{<<"id">>, DbName}]},
- scan_changes_cb({change, Change, req}, type, {self(), 1}),
- ResumeMsg = receive Msg -> Msg after 1000 -> timeout end,
- ?assertMatch({'$gen_cast', {resume_scan, <<"shards/", _/binary>>}}, ResumeMsg),
- fabric:delete_db(DbName, [?CTX])
- end).
-
-
-t_sleep_based_on_count() ->
- ?_test(begin
- DbName0 = ?tempdb(),
- DbName = <<DbName0/binary, "/_replicator">>,
- ok = fabric:create_db(DbName, [?CTX]),
- Change = {[{<<"id">>, DbName}]},
- scan_changes_cb({change, Change, req}, type, {self(), 1000}),
- Timeout = receive Msg -> Msg after 100 -> timeout end,
- ?assertEqual(timeout, Timeout),
- fabric:delete_db(DbName, [?CTX])
- end).
-
-
-endif.
[18/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Prevent replicator manager change feeds from getting stuck
Switch them them from `longpoll` to `normal`
This would prevent them being stuck. That could happen if more than one
`resume_scan` message arrives for the same shard. The first time a longpoll
changef feed would finish and end sequence is checkpointed. But if another
resume_scan arrives and database hasn't changed then the longpoll change
feed would hang until db is updated.
The reason there would be multiple `resume_scan` messages is because there
is a race condition between db update handler and scanner component. They are
both started asynchronously roughly at the same. Scanner finds new shard while
db handler notices changes for those shards. If shards are modified quickly
after they are discovered by the scanner both of those components would issue
a resume_scan.
The effect of this would be more pronounced if there are a large number of
_replicator shards and constant db creation/deletion/updates.
COUCHDB-2964
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/d00b9814
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/d00b9814
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/d00b9814
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: d00b981445c03622497088eb872059ab4f48b298
Parents: f63efa7
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Mar 10 01:15:47 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Fri Mar 10 01:15:47 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/d00b9814/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 2bcad69..85dd428 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -375,7 +375,7 @@ changes_reader({Server, Epoch}, DbName, Since) ->
#changes_args{
include_docs = true,
since = Since,
- feed = "longpoll",
+ feed = "normal",
timeout = infinity
},
{json_req, null},
[06/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Merge branch 'couchdb-3291-use-infinity'
Closes #55
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/648e465f
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/648e465f
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/648e465f
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 648e465f54f538a133fb31c9b1e3b487a6f2ca7c
Parents: 1644696 46f70c7
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Feb 8 13:11:47 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Feb 8 13:11:47 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 9 +++++++--
test/couch_replicator_id_too_long_tests.erl | 2 +-
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/648e465f/src/couch_replicator_changes_reader.erl
----------------------------------------------------------------------
[19/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Remove public db record
COUCHDB-3288
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a8ac02d3
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a8ac02d3
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a8ac02d3
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: a8ac02d3a423ca5798018edb6bf3690b742cf94c
Parents: d00b981
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 1 16:01:35 2017 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:17:09 2017 -0500
----------------------------------------------------------------------
src/couch_replicator.erl | 33 +++++++++-------
src/couch_replicator_api_wrap.erl | 27 ++++++++------
src/couch_replicator_manager.erl | 8 ++--
src/couch_replicator_utils.erl | 40 ++++++++++++--------
src/couch_replicator_worker.erl | 56 ++++++++++++++++------------
test/couch_replicator_compact_tests.erl | 27 ++++++++------
6 files changed, 111 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 7f0c7ee..1ce2bae 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -477,15 +477,21 @@ handle_call({report_seq_done, Seq, StatsInc}, From,
{noreply, NewState}.
-handle_cast({db_compacted, DbName},
- #rep_state{source = #db{name = DbName} = Source} = State) ->
- {ok, NewSource} = couch_db:reopen(Source),
- {noreply, State#rep_state{source = NewSource}};
-
-handle_cast({db_compacted, DbName},
- #rep_state{target = #db{name = DbName} = Target} = State) ->
- {ok, NewTarget} = couch_db:reopen(Target),
- {noreply, State#rep_state{target = NewTarget}};
+handle_cast({db_compacted, DbName}, State) ->
+ #rep_state{
+ source = Source,
+ target = Target
+ } = State,
+ SourceName = couch_replicator_utils:local_db_name(Source),
+ TargetName = couch_replicator_utils:local_db_name(Target),
+ case DbName of
+ SourceName ->
+ {ok, NewSource} = couch_db:reopen(Source),
+ {noreply, State#rep_state{source = NewSource}};
+ TargetName ->
+ {ok, NewTarget} = couch_db:reopen(Target),
+ {noreply, State#rep_state{target = NewTarget}}
+ end;
handle_cast(checkpoint, State) ->
#rep_state{rep_details = #rep{} = Rep} = State,
@@ -968,10 +974,11 @@ has_session_id(SessionId, [{Props} | Rest]) ->
end.
-db_monitor(#db{} = Db) ->
- couch_db:monitor(Db);
-db_monitor(_HttpDb) ->
- nil.
+db_monitor(#httpdb{}) ->
+ nil;
+db_monitor(Db) ->
+ couch_db:monitor(Db).
+
get_pending_count(St) ->
Rep = St#rep_state.rep_details,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index e5f6253..09f22cb 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -59,11 +59,11 @@
db_uri(#httpdb{url = Url}) ->
couch_util:url_strip_password(Url);
-db_uri(#db{name = Name}) ->
- db_uri(Name);
+db_uri(DbName) when is_binary(DbName) ->
+ ?b2l(DbName);
-db_uri(DbName) ->
- ?b2l(DbName).
+db_uri(Db) ->
+ db_uri(couch_db:name(Db)).
db_open(Db, Options) ->
@@ -148,10 +148,12 @@ get_db_info(#httpdb{} = Db) ->
fun(200, _, {Props}) ->
{ok, Props}
end);
-get_db_info(#db{name = DbName, user_ctx = UserCtx}) ->
- {ok, Db} = couch_db:open(DbName, [{user_ctx, UserCtx}]),
- {ok, Info} = couch_db:get_db_info(Db),
- couch_db:close(Db),
+get_db_info(Db) ->
+ DbName = couch_db:name(Db),
+ UserCtx = couch_db:get_user_ctx(Db),
+ {ok, InfoDb} = couch_db:open(DbName, [{user_ctx, UserCtx}]),
+ {ok, Info} = couch_db:get_db_info(InfoDb),
+ couch_db:close(InfoDb),
{ok, [{couch_util:to_binary(K), V} || {K, V} <- Info]}.
@@ -171,8 +173,10 @@ get_pending_count(#httpdb{} = Db, Seq) ->
send_req(Db, Options, fun(200, _, {Props}) ->
{ok, couch_util:get_value(<<"pending">>, Props, null)}
end);
-get_pending_count(#db{name=DbName}=Db, Seq) when is_number(Seq) ->
- {ok, CountDb} = couch_db:open(DbName, [{user_ctx, Db#db.user_ctx}]),
+get_pending_count(Db, Seq) when is_number(Seq) ->
+ DbName = couch_db:name(Db),
+ UserCtx = couch_db:get_user_ctx(Db),
+ {ok, CountDb} = couch_db:open(DbName, [{user_ctx, UserCtx}]),
Pending = couch_db:count_changes_since(CountDb, Seq),
couch_db:close(CountDb),
{ok, Pending}.
@@ -184,7 +188,8 @@ get_view_info(#httpdb{} = Db, DDocId, ViewName) ->
{VInfo} = couch_util:get_value(<<"view_index">>, Props, {[]}),
{ok, VInfo}
end);
-get_view_info(#db{name = DbName}, DDocId, ViewName) ->
+get_view_info(Db, DDocId, ViewName) ->
+ DbName = couch_db:name(Db),
{ok, VInfo} = couch_mrview:get_view_info(DbName, DDocId, ViewName),
{ok, [{couch_util:to_binary(K), V} || {K, V} <- VInfo]}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 85dd428..48bf7a6 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -879,8 +879,8 @@ state_after_error(#rep_state{retries_left = Left, wait = Wait} = State) ->
before_doc_update(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db) ->
Doc;
-before_doc_update(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
- #user_ctx{roles = Roles, name = Name} = UserCtx,
+before_doc_update(#doc{body = {Body}} = Doc, Db) ->
+ #user_ctx{roles = Roles, name = Name} = couch_db:get_user_ctx(Db),
case lists:member(<<"_replicator">>, Roles) of
true ->
Doc;
@@ -906,8 +906,8 @@ before_doc_update(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
after_doc_read(#doc{id = <<?DESIGN_DOC_PREFIX, _/binary>>} = Doc, _Db) ->
Doc;
-after_doc_read(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
- #user_ctx{name = Name} = UserCtx,
+after_doc_read(#doc{body = {Body}} = Doc, Db) ->
+ #user_ctx{name = Name} = couch_db:get_user_ctx(Db),
case (catch couch_db:check_is_admin(Db)) of
ok ->
Doc;
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index e96d52a..17d3334 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -14,6 +14,7 @@
-export([parse_rep_doc/2]).
-export([open_db/1, close_db/1]).
+-export([local_db_name/1]).
-export([start_db_compaction_notifier/2, stop_db_compaction_notifier/1]).
-export([replication_id/2]).
-export([sum_stats/2, is_deleted/1]).
@@ -410,26 +411,33 @@ ssl_verify_options(false, _OTPVersion) ->
%% New db record has Options field removed here to enable smoother dbcore migration
-open_db(#db{name = Name, user_ctx = UserCtx}) ->
- {ok, Db} = couch_db:open(Name, [{user_ctx, UserCtx} | []]),
- Db;
-open_db(HttpDb) ->
- HttpDb.
-
-
-close_db(#db{} = Db) ->
- couch_db:close(Db);
-close_db(_HttpDb) ->
- ok.
-
+open_db(#httpdb{} = HttpDb) ->
+ HttpDb;
+open_db(Db) ->
+ DbName = couch_db:name(Db),
+ UserCtx = couch_db:get_user_ctx(Db),
+ {ok, NewDb} = couch_db:open(DbName, [{user_ctx, UserCtx}]),
+ NewDb.
+
+close_db(#httpdb{}) ->
+ ok;
+close_db(Db) ->
+ couch_db:close(Db).
-start_db_compaction_notifier(#db{name = DbName}, Server) ->
+start_db_compaction_notifier(#httpdb{}, _) ->
+ nil;
+start_db_compaction_notifier(Db, Server) ->
+ DbName = couch_db:name(Db),
{ok, Pid} = couch_event:link_listener(
?MODULE, handle_db_event, Server, [{dbname, DbName}]
),
- Pid;
-start_db_compaction_notifier(_, _) ->
- nil.
+ Pid.
+
+
+local_db_name(#httpdb{}) ->
+ undefined;
+local_db_name(Db) ->
+ couch_db:name(Db).
stop_db_compaction_notifier(nil) ->
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index ee0c455..1a4e599 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -67,16 +67,16 @@
-start_link(Cp, #db{} = Source, Target, ChangesManager, _MaxConns) ->
+start_link(Cp, #httpdb{} = Source, Target, ChangesManager, MaxConns) ->
+ gen_server:start_link(
+ ?MODULE, {Cp, Source, Target, ChangesManager, MaxConns}, []);
+
+start_link(Cp, Source, Target, ChangesManager, _MaxConns) ->
Pid = spawn_link(fun() ->
erlang:put(last_stats_report, now()),
queue_fetch_loop(Source, Target, Cp, Cp, ChangesManager)
end),
- {ok, Pid};
-
-start_link(Cp, Source, Target, ChangesManager, MaxConns) ->
- gen_server:start_link(
- ?MODULE, {Cp, Source, Target, ChangesManager, MaxConns}, []).
+ {ok, Pid}.
init({Cp, Source, Target, ChangesManager, MaxConns}) ->
@@ -139,15 +139,23 @@ handle_call(flush, {Pid, _} = From,
{noreply, State2#state{flush_waiter = From}}.
-handle_cast({db_compacted, DbName},
- #state{source = #db{name = DbName} = Source} = State) ->
- {ok, NewSource} = couch_db:reopen(Source),
- {noreply, State#state{source = NewSource}};
-
-handle_cast({db_compacted, DbName},
- #state{target = #db{name = DbName} = Target} = State) ->
- {ok, NewTarget} = couch_db:reopen(Target),
- {noreply, State#state{target = NewTarget}};
+handle_cast({db_compacted, DbName} = Msg, #state{} = State) ->
+ #state{
+ source = Source,
+ target = Target
+ } = State,
+ SourceName = couch_replicator_utils:local_db_name(Source),
+ TargetName = couch_replicator_utils:local_db_name(Target),
+ case DbName of
+ SourceName ->
+ {ok, NewSource} = couch_db:reopen(Source),
+ {noreply, State#state{source = NewSource}};
+ TargetName ->
+ {ok, NewTarget} = couch_db:reopen(Target),
+ {noreply, State#state{target = NewTarget}};
+ _Else ->
+ {stop, {unexpected_async_call, Msg}, State}
+ end;
handle_cast(Msg, State) ->
{stop, {unexpected_async_call, Msg}, State}.
@@ -220,15 +228,15 @@ queue_fetch_loop(Source, Target, Parent, Cp, ChangesManager) ->
Target2 = open_db(Target),
{IdRevs, Stats0} = find_missing(Changes, Target2),
case Source of
- #db{} ->
- Source2 = open_db(Source),
- Stats = local_process_batch(
- IdRevs, Cp, Source2, Target2, #batch{}, Stats0),
- close_db(Source2);
#httpdb{} ->
ok = gen_server:call(Parent, {add_stats, Stats0}, infinity),
remote_process_batch(IdRevs, Parent),
- {ok, Stats} = gen_server:call(Parent, flush, infinity)
+ {ok, Stats} = gen_server:call(Parent, flush, infinity);
+ _Db ->
+ Source2 = open_db(Source),
+ Stats = local_process_batch(
+ IdRevs, Cp, Source2, Target2, #batch{}, Stats0),
+ close_db(Source2)
end,
close_db(Target2),
ok = gen_server:call(Cp, {report_seq_done, ReportSeq, Stats}, infinity),
@@ -245,7 +253,7 @@ local_process_batch([], Cp, Source, Target, #batch{docs = Docs, size = Size}, St
case Target of
#httpdb{} ->
couch_log:debug("Worker flushing doc batch of size ~p bytes", [Size]);
- #db{} ->
+ _Db ->
couch_log:debug("Worker flushing doc batch of ~p docs", [Size])
end,
Stats2 = flush_docs(Target, Docs),
@@ -360,7 +368,7 @@ spawn_writer(Target, #batch{docs = DocList, size = Size}) ->
case {Target, Size > 0} of
{#httpdb{}, true} ->
couch_log:debug("Worker flushing doc batch of size ~p bytes", [Size]);
- {#db{}, true} ->
+ {_Db, true} ->
couch_log:debug("Worker flushing doc batch of ~p docs", [Size]);
_ ->
ok
@@ -422,7 +430,7 @@ maybe_flush_docs(#httpdb{} = Target, Batch, Doc) ->
end
end;
-maybe_flush_docs(#db{} = Target, #batch{docs = DocAcc, size = SizeAcc}, Doc) ->
+maybe_flush_docs(Target, #batch{docs = DocAcc, size = SizeAcc}, Doc) ->
case SizeAcc + 1 of
SizeAcc2 when SizeAcc2 >= ?DOC_BUFFER_LEN ->
couch_log:debug("Worker flushing doc batch of ~p docs", [SizeAcc2]),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a8ac02d3/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 7a5a25a..5731ff4 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -82,8 +82,8 @@ should_all_processes_be_alive(RepPid, Source, Target) ->
{ok, SourceDb} = reopen_db(Source),
{ok, TargetDb} = reopen_db(Target),
?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb#db.main_pid)),
- ?assert(is_process_alive(TargetDb#db.main_pid))
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb)))
end).
should_run_replication(RepPid, RepId, Source, Target) ->
@@ -149,12 +149,12 @@ should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
compact_db("source", SourceDb),
?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb#db.main_pid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb))),
wait_for_compaction("source", SourceDb),
compact_db("target", TargetDb),
?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(TargetDb#db.main_pid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb))),
wait_for_compaction("target", TargetDb),
{ok, SourceDb2} = reopen_db(SourceDb),
@@ -165,14 +165,14 @@ should_populate_and_compact(RepPid, Source, Target, BatchSize, Rounds) ->
compact_db("source", SourceDb2),
?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(SourceDb2#db.main_pid)),
+ ?assert(is_process_alive(couch_db:get_pid(SourceDb2))),
pause_writer(Writer),
wait_for_compaction("source", SourceDb2),
resume_writer(Writer),
compact_db("target", TargetDb2),
?assert(is_process_alive(RepPid)),
- ?assert(is_process_alive(TargetDb2#db.main_pid)),
+ ?assert(is_process_alive(couch_db:get_pid(TargetDb2))),
pause_writer(Writer),
wait_for_compaction("target", TargetDb2),
resume_writer(Writer)
@@ -248,14 +248,16 @@ should_compare_databases(Source, Target) ->
reopen_db({remote, Db}) ->
reopen_db(Db);
-reopen_db(#db{name=DbName}) ->
- reopen_db(DbName);
-reopen_db(DbName) ->
+reopen_db(DbName) when is_binary(DbName) ->
{ok, Db} = couch_db:open_int(DbName, []),
ok = couch_db:close(Db),
- {ok, Db}.
+ {ok, Db};
+reopen_db(Db) ->
+ reopen_db(couch_db:name(Db)).
-compact_db(Type, #db{name = Name}) ->
+
+compact_db(Type, Db0) ->
+ Name = couch_db:name(Db0),
{ok, Db} = couch_db:open_int(Name, []),
{ok, CompactPid} = couch_db:start_compact(Db),
MonRef = erlang:monitor(process, CompactPid),
@@ -395,7 +397,8 @@ stop_writer(Pid) ->
{reason, "Timeout stopping source database writer"}]})
end.
-writer_loop(#db{name = DbName}, Parent, Counter) ->
+writer_loop(Db0, Parent, Counter) ->
+ DbName = couch_db:name(Db0),
{ok, Data} = file:read_file(?ATTFILE),
maybe_pause(Parent, Counter),
Doc = couch_doc:from_json_obj({[
[09/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
fix crashes when replicator db is deleted
BugzID: 83663
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7a2b2b68
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7a2b2b68
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7a2b2b68
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 7a2b2b68bf4afa6d20b56a8ae51361f83981412a
Parents: 1e413b8
Author: Robert Newson <rn...@apache.org>
Authored: Wed Mar 1 11:14:02 2017 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Mar 1 11:15:59 2017 +0000
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 26 ++++++++++++++++++--------
1 file changed, 18 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7a2b2b68/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index bdc3b8f..c9c0a68 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -731,11 +731,15 @@ update_rep_doc(RepDbName, #doc{body = {RepDocBody}} = RepDoc, KVs, _Try) ->
end.
open_rep_doc(DbName, DocId) ->
- {ok, Db} = couch_db:open_int(DbName, [?CTX, sys_db]),
- try
- couch_db:open_doc(Db, DocId, [ejson_body])
- after
- couch_db:close(Db)
+ case couch_db:open_int(DbName, [?CTX, sys_db]) of
+ {ok, Db} ->
+ try
+ couch_db:open_doc(Db, DocId, [ejson_body])
+ after
+ couch_db:close(Db)
+ end;
+ Else ->
+ Else
end.
save_rep_doc(DbName, Doc) ->
@@ -791,6 +795,9 @@ ensure_rep_ddoc_exists(RepDb) ->
ensure_rep_ddoc_exists(RepDb, DDocId) ->
case open_rep_doc(RepDb, DDocId) of
+ {not_found, no_db_file} ->
+ %% database was deleted.
+ ok;
{not_found, _Reason} ->
{ok, DDoc} = replication_design_doc(DDocId),
couch_log:notice("creating replicator ddoc", []),
@@ -941,9 +948,12 @@ scan_all_dbs(Server) when is_pid(Server) ->
true ->
ok;
false ->
- [gen_server:cast(Server, {resume_scan, ShardName})
- || ShardName <- replicator_shards(DbName)],
- ok
+ try
+ [gen_server:cast(Server, {resume_scan, ShardName})
+ || ShardName <- replicator_shards(DbName)]
+ catch error:database_does_not_exist ->
+ ok
+ end
end
end;
(_, _) -> ok
[05/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Merge branch 'couchdb-3291-better-formatting'
Closes #56
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/1644696c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/1644696c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/1644696c
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 1644696c8c99622eb40a2ef90876e9ecbf716783
Parents: b18b31c c306fab
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Feb 8 12:50:36 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Feb 8 12:50:36 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
[11/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Merge remote-tracking branch 'cloudant/couchdb-3316'
Closes #59
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/50a88baa
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/50a88baa
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/50a88baa
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 50a88baafc10c4cf34bce5a54d59e15ca15ea60a
Parents: 7a2b2b6 50dcd7d
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Mar 3 10:49:54 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Fri Mar 3 10:49:54 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 55 ++++++++++++++++++-----------------
1 file changed, 29 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
[07/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Restore adding some jitter-ed sleep to shard scanning code.
Otherwise a large cluster will flood replicator manager with potentially
hundreds of thousands of `{resume, Shard}` messages. For each one, it
would try to open a changes feed which can add significant load and has
been seen in production to hit varios system limits.
This brings back the change from before the switch to using mem3 shards
for replicator db scans.
Also adds a few tests.
COUCHDB-3311
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/45d739af
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/45d739af
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/45d739af
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 45d739af3fcf8b4f8e3ccca152cb3c2d781dc2fc
Parents: 648e465
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Feb 28 14:00:22 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Feb 28 14:00:22 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 74 ++++++++++++++++++++++++++++-------
1 file changed, 60 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/45d739af/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index bdc3b8f..4e5073e 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -934,22 +934,30 @@ scan_all_dbs(Server) when is_pid(Server) ->
{ok, Db} = mem3_util:ensure_exists(
config:get("mem3", "shards_db", "_dbs")),
ChangesFun = couch_changes:handle_changes(#changes_args{}, nil, Db, nil),
- ChangesFun(fun({change, {Change}, _}, _) ->
- DbName = couch_util:get_value(<<"id">>, Change),
- case DbName of <<"_design/", _/binary>> -> ok; _Else ->
- case couch_replicator_utils:is_deleted(Change) of
- true ->
- ok;
- false ->
- [gen_server:cast(Server, {resume_scan, ShardName})
- || ShardName <- replicator_shards(DbName)],
- ok
- end
- end;
- (_, _) -> ok
- end),
+ ChangesFun({fun scan_changes_cb/3, {Server, 1}}),
couch_db:close(Db).
+scan_changes_cb({change, {Change}, _}, _, {Server, AccCount}) ->
+ DbName = couch_util:get_value(<<"id">>, Change),
+ case DbName of <<"_design/", _/binary>> -> {Server, AccCount}; _Else ->
+ case couch_replicator_utils:is_deleted(Change) of
+ true ->
+ {Server, AccCount};
+ false ->
+ UpdatedCount = lists:foldl(fun(ShardName, Count) ->
+ spawn_link(fun() ->
+ timer:sleep(jitter(Count)),
+ gen_server:cast(Server, {resume_scan, ShardName})
+ end),
+ Count + 1
+ end, AccCount, replicator_shards(DbName)),
+ {Server, UpdatedCount}
+ end
+ end;
+
+scan_changes_cb(_, _, {Server, AccCount}) ->
+ {Server, AccCount}.
+
replicator_shards(DbName) ->
case is_replicator_db(DbName) of
@@ -1027,4 +1035,42 @@ t_fail_non_replicator_shard() ->
end).
+scan_dbs_test_() ->
+{
+ foreach,
+ fun() -> test_util:start_couch([mem3, fabric]) end,
+ fun(Ctx) -> test_util:stop_couch(Ctx) end,
+ [
+ t_resume_db_shard(),
+ t_sleep_based_on_count()
+ ]
+}.
+
+
+t_resume_db_shard() ->
+ ?_test(begin
+ DbName0 = ?tempdb(),
+ DbName = <<DbName0/binary, "/_replicator">>,
+ ok = fabric:create_db(DbName, [?CTX]),
+ Change = {[{<<"id">>, DbName}]},
+ scan_changes_cb({change, Change, req}, type, {self(), 1}),
+ ResumeMsg = receive Msg -> Msg after 1000 -> timeout end,
+ ?assertMatch({'$gen_cast', {resume_scan, <<"shards/", _/binary>>}}, ResumeMsg),
+ fabric:delete_db(DbName, [?CTX])
+ end).
+
+
+t_sleep_based_on_count() ->
+ ?_test(begin
+ DbName0 = ?tempdb(),
+ DbName = <<DbName0/binary, "/_replicator">>,
+ ok = fabric:create_db(DbName, [?CTX]),
+ Change = {[{<<"id">>, DbName}]},
+ scan_changes_cb({change, Change, req}, type, {self(), 1000}),
+ Timeout = receive Msg -> Msg after 100 -> timeout end,
+ ?assertEqual(timeout, Timeout),
+ fabric:delete_db(DbName, [?CTX])
+ end).
+
+
-endif.
[14/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Fix unit test after renaming max_document_size config parameter
`couchdb.max_document_size` was renamed to `httpd.max_http_request_size`
The unit tests was testing how replicator behaves when faced with reduced
request size configuration on the target.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/64958096
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/64958096
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/64958096
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 64958096d4f9a940c01cbc472da5265f349c9545
Parents: 1166759
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 16:42:43 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 16:42:43 2017 -0500
----------------------------------------------------------------------
test/couch_replicator_small_max_request_size_target.erl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/64958096/test/couch_replicator_small_max_request_size_target.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_small_max_request_size_target.erl b/test/couch_replicator_small_max_request_size_target.erl
index c46619d..b7d9fbe 100644
--- a/test/couch_replicator_small_max_request_size_target.erl
+++ b/test/couch_replicator_small_max_request_size_target.erl
@@ -27,7 +27,7 @@ setup(remote) ->
setup({A, B}) ->
Ctx = test_util:start_couch([couch_replicator]),
- config:set("couchdb", "max_document_size", "10000", false),
+ config:set("httpd", "max_http_request_size", "10000", false),
Source = setup(A),
Target = setup(B),
{Ctx, {Source, Target}}.
@@ -49,7 +49,7 @@ teardown(_, {Ctx, {Source, Target}}) ->
reduce_max_request_size_test_() ->
Pairs = [{local, remote}, {remote, remote}],
{
- "Replicate docs when target has a small max_document_size",
+ "Replicate docs when target has a small max_http_request_size",
{
foreachx,
fun setup/1, fun teardown/2,
@@ -66,8 +66,8 @@ reduce_max_request_size_test_() ->
}.
-% Test documents which are below max_document_size but when batched, batch size
-% will be greater than max_document_size. Replicator could automatically split
+% Test documents which are below max_http_request_size but when batched, batch size
+% will be greater than max_http_request_size. Replicator could automatically split
% the batch into smaller batches and POST those separately.
should_replicate_all_docs({From, To}, {_Ctx, {Source, Target}}) ->
{lists:flatten(io_lib:format("~p -> ~p", [From, To])),
[17/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Revert "Don't scan empty replicator databases"
This reverts commit 46aa27fa674a4c1e590aeecd76123e4f91d78fd5.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/f63efa78
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/f63efa78
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/f63efa78
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: f63efa786fec5cc40e6c6193470399da99385abf
Parents: 46aa27f
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Fri Mar 10 01:13:28 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Fri Mar 10 01:13:28 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 46 +++--------------------------------
1 file changed, 3 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/f63efa78/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 9f17087..2bcad69 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -266,17 +266,9 @@ handle_cast({resume_scan, DbName}, State) ->
end,
true = ets:insert(?DB_TO_SEQ, {DbName, Since, false}),
ensure_rep_ddoc_exists(DbName),
- case has_replication_docs(DbName) of
- false ->
- %% the database is empty save for the rep_ddoc,
- %% don't bother scanning it.
- couch_log:debug("ignoring empty ~s", [DbName]),
- Pids;
- true ->
- Pid = start_changes_reader(DbName, Since, State#state.epoch),
- couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
- [{DbName, Pid} | Pids]
- end
+ Pid = start_changes_reader(DbName, Since, State#state.epoch),
+ couch_log:debug("Scanning ~s from update_seq ~p", [DbName, Since]),
+ [{DbName, Pid} | Pids]
end,
{noreply, State#state{rep_start_pids = NewPids}};
@@ -1013,38 +1005,6 @@ get_json_value(Key, Props, Default) when is_binary(Key) ->
end.
-has_replication_docs(DbName) ->
- {ok, Db} = couch_db:open(DbName, []),
- try
- case couch_db:get_doc_count(Db) of
- {ok, 0} ->
- false;
- {ok, 1} ->
- case first_doc_id(Db) of
- <<"_design/_replicator">> ->
- false;
- _Else ->
- true
- end;
- _Else ->
- true
- end
- after
- couch_db:close(Db)
- end.
-
-
-first_doc_id(#db{} = Db) ->
- Fun = fun
- (#full_doc_info{deleted = true}, _Reds, Acc) ->
- {ok, Acc};
- (FDI, _Reds, _Acc) ->
- {stop, FDI#full_doc_info.id}
- end,
- {ok, _, Id} = couch_btree:fold(Db#db.id_tree, Fun, nil, []),
- Id.
-
-
-ifdef(TEST).
-include_lib("couch/include/couch_eunit.hrl").
[15/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Merge branch '64229-add-new-request-parameter'
Closes #61
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ffe9697e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ffe9697e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ffe9697e
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: ffe9697e728c4725f5ff8915cf592d2d4d0074cc
Parents: 1166759 6495809
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 19:17:26 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 19:17:26 2017 -0500
----------------------------------------------------------------------
test/couch_replicator_small_max_request_size_target.erl | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
[13/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Merge remote-tracking branch 'cloudant/couchdb-2992-remove-dead-code'
Closes #60
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/11667591
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/11667591
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/11667591
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 11667591c1211ef2817945393ec22e0361d900ad
Parents: 50a88ba 30915e3
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 14:45:59 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 14:45:59 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_utils.erl | 30 ------------------------------
1 file changed, 30 deletions(-)
----------------------------------------------------------------------
[20/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Update tests to use pluggable storage engine API
COUCHDB-3287
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/4e45eab6
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/4e45eab6
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/4e45eab6
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 4e45eab609aede8f17ff44b89044c34b0f4ab6a1
Parents: a8ac02d
Author: Paul J. Davis <pa...@gmail.com>
Authored: Wed Feb 10 20:20:57 2016 -0600
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Tue Apr 4 16:17:54 2017 -0500
----------------------------------------------------------------------
test/couch_replicator_compact_tests.erl | 4 ++--
test/couch_replicator_filtered_tests.erl | 4 ++--
test/couch_replicator_missing_stubs_tests.erl | 4 ++--
test/couch_replicator_selector_tests.erl | 4 ++--
test/couch_replicator_test_helper.erl | 4 ++--
test/couch_replicator_use_checkpoints_tests.erl | 4 ++--
6 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_compact_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_compact_tests.erl b/test/couch_replicator_compact_tests.erl
index 5731ff4..3ac35a1 100644
--- a/test/couch_replicator_compact_tests.erl
+++ b/test/couch_replicator_compact_tests.erl
@@ -221,7 +221,7 @@ should_compare_databases(Source, Target) ->
{timeout, 35, ?_test(begin
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, _, Acc) ->
+ Fun = fun(FullDocInfo, Acc) ->
{ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
{Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
DocId = couch_util:get_value(<<"_id">>, Props),
@@ -240,7 +240,7 @@ should_compare_databases(Source, Target) ->
?assertEqual(DocJson, DocTargetJson),
{ok, Acc}
end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb)
end)}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_filtered_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_filtered_tests.erl b/test/couch_replicator_filtered_tests.erl
index 03cf44c..d34e9f0 100644
--- a/test/couch_replicator_filtered_tests.erl
+++ b/test/couch_replicator_filtered_tests.erl
@@ -169,7 +169,7 @@ compare_dbs(Source, Target, FilterFun) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
{ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, _, Acc) ->
+ Fun = fun(FullDocInfo, Acc) ->
{ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
TargetReply = read_doc(TargetDb, DocId),
case FilterFun(DocId, SourceDoc) of
@@ -181,7 +181,7 @@ compare_dbs(Source, Target, FilterFun) ->
{ok, [ValidReply|Acc]}
end
end,
- {ok, _, AllReplies} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb),
{ok, TargetDbInfo, AllReplies}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_missing_stubs_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_missing_stubs_tests.erl b/test/couch_replicator_missing_stubs_tests.erl
index e8ccd64..f8d231d 100644
--- a/test/couch_replicator_missing_stubs_tests.erl
+++ b/test/couch_replicator_missing_stubs_tests.erl
@@ -131,9 +131,9 @@ populate_db(DbName) ->
update_db_docs(DbName, Times) ->
{ok, Db} = couch_db:open_int(DbName, []),
- {ok, _, _} = couch_db:enum_docs(
+ {ok, _} = couch_db:fold_docs(
Db,
- fun(FDI, _, Acc) -> db_fold_fun(FDI, Acc) end,
+ fun(FDI, Acc) -> db_fold_fun(FDI, Acc) end,
{DbName, Times},
[]),
ok = couch_db:close(Db).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_selector_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_selector_tests.erl b/test/couch_replicator_selector_tests.erl
index 98c6099..a7f4c5d 100644
--- a/test/couch_replicator_selector_tests.erl
+++ b/test/couch_replicator_selector_tests.erl
@@ -65,7 +65,7 @@ compare_dbs(Source, Target, FilterFun) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
{ok, TargetDbInfo} = couch_db:get_db_info(TargetDb),
- Fun = fun(FullDocInfo, _, Acc) ->
+ Fun = fun(FullDocInfo, Acc) ->
{ok, DocId, SourceDoc} = read_doc(SourceDb, FullDocInfo),
TargetReply = read_doc(TargetDb, DocId),
case FilterFun(DocId, SourceDoc) of
@@ -77,7 +77,7 @@ compare_dbs(Source, Target, FilterFun) ->
{ok, [ValidReply|Acc]}
end
end,
- {ok, _, AllReplies} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ {ok, AllReplies} = couch_db:fold_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb),
{ok, TargetDbInfo, AllReplies}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_test_helper.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_test_helper.erl b/test/couch_replicator_test_helper.erl
index 398b27b..ae27745 100644
--- a/test/couch_replicator_test_helper.erl
+++ b/test/couch_replicator_test_helper.erl
@@ -14,7 +14,7 @@ compare_dbs(Source, Target, ExceptIds) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, _, Acc) ->
+ Fun = fun(FullDocInfo, Acc) ->
{ok, DocSource} = couch_db:open_doc(SourceDb, FullDocInfo),
Id = DocSource#doc.id,
case lists:member(Id, ExceptIds) of
@@ -27,7 +27,7 @@ compare_dbs(Source, Target, ExceptIds) ->
{ok, Acc}
end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4e45eab6/test/couch_replicator_use_checkpoints_tests.erl
----------------------------------------------------------------------
diff --git a/test/couch_replicator_use_checkpoints_tests.erl b/test/couch_replicator_use_checkpoints_tests.erl
index e04488e..9dea8af 100644
--- a/test/couch_replicator_use_checkpoints_tests.erl
+++ b/test/couch_replicator_use_checkpoints_tests.erl
@@ -144,7 +144,7 @@ populate_db(DbName, DocCount) ->
compare_dbs(Source, Target) ->
{ok, SourceDb} = couch_db:open_int(Source, []),
{ok, TargetDb} = couch_db:open_int(Target, []),
- Fun = fun(FullDocInfo, _, Acc) ->
+ Fun = fun(FullDocInfo, Acc) ->
{ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
{Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
DocId = couch_util:get_value(<<"_id">>, Props),
@@ -163,7 +163,7 @@ compare_dbs(Source, Target) ->
?assertEqual(DocJson, DocTargetJson),
{ok, Acc}
end,
- {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ {ok, _} = couch_db:fold_docs(SourceDb, Fun, [], []),
ok = couch_db:close(SourceDb),
ok = couch_db:close(TargetDb).
[04/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Use string formatting to shorten document ID during logging.
Previously used an explicit lists:sublist call but value was never used
anywhere besides the log message.
COUCHDB-3291
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/c306fab2
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/c306fab2
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/c306fab2
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: c306fab27dbd88d8ecc8f60fb5ec04e7911fd786
Parents: b18b31c
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Wed Feb 8 12:02:34 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Wed Feb 8 12:02:34 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_changes_reader.erl | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/c306fab2/src/couch_replicator_changes_reader.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_changes_reader.erl b/src/couch_replicator_changes_reader.erl
index f9d9097..497de93 100644
--- a/src/couch_replicator_changes_reader.erl
+++ b/src/couch_replicator_changes_reader.erl
@@ -92,10 +92,9 @@ process_change(#doc_info{id = <<>>} = DocInfo, {_, Db, _, _, _}) ->
process_change(#doc_info{id = Id} = DocInfo, {Parent, Db, ChangesQueue, _, _}) ->
case is_doc_id_too_long(byte_size(Id)) of
true ->
- ShortId = lists:sublist(binary_to_list(Id), 64),
SourceDb = couch_replicator_api_wrap:db_uri(Db),
couch_log:error("Replicator: document id `~s...` from source db "
- " `~s` is too long, ignoring.", [ShortId, SourceDb]),
+ " `~64s` is too long, ignoring.", [Id, SourceDb]),
Stats = couch_replicator_stats:new([{doc_write_failures, 1}]),
ok = gen_server:call(Parent, {add_stats, Stats}, infinity);
false ->
[12/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Remove unused mp_parse_doc function from replicator
It was left accidentally when merging Cloudant's dbcore work.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/30915e33
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/30915e33
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/30915e33
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 30915e3309fb30c2164e668d33dbd393e77925c0
Parents: 50a88ba
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Tue Mar 7 14:38:29 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Tue Mar 7 14:38:29 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_utils.erl | 30 ------------------------------
1 file changed, 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/30915e33/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 223bac8..e96d52a 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -17,7 +17,6 @@
-export([start_db_compaction_notifier/2, stop_db_compaction_notifier/1]).
-export([replication_id/2]).
-export([sum_stats/2, is_deleted/1]).
--export([mp_parse_doc/2]).
-export([handle_db_event/3]).
@@ -449,35 +448,6 @@ handle_db_event(_DbName, _Event, Server) ->
sum_stats(S1, S2) ->
couch_replicator_stats:sum_stats(S1, S2).
-mp_parse_doc({headers, H}, []) ->
- case couch_util:get_value("content-type", H) of
- {"application/json", _} ->
- fun (Next) ->
- mp_parse_doc(Next, [])
- end
- end;
-mp_parse_doc({body, Bytes}, AccBytes) ->
- fun (Next) ->
- mp_parse_doc(Next, [Bytes | AccBytes])
- end;
-mp_parse_doc(body_end, AccBytes) ->
- receive {get_doc_bytes, Ref, From} ->
- From ! {doc_bytes, Ref, lists:reverse(AccBytes)}
- end,
- fun mp_parse_atts/1.
-
-mp_parse_atts(eof) ->
- ok;
-mp_parse_atts({headers, _H}) ->
- fun mp_parse_atts/1;
-mp_parse_atts({body, Bytes}) ->
- receive {get_bytes, Ref, From} ->
- From ! {bytes, Ref, Bytes}
- end,
- fun mp_parse_atts/1;
-mp_parse_atts(body_end) ->
- fun mp_parse_atts/1.
-
is_deleted(Change) ->
case couch_util:get_value(<<"deleted">>, Change) of
undefined ->
[10/20] couch-replicator commit: updated
refs/heads/COUCHDB-3287-pluggable-storage-engines to 4e45eab
Posted by da...@apache.org.
Make sure to log db as well as doc in replicator logs.
COUCHDB-3316
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/50dcd7d7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/50dcd7d7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/50dcd7d7
Branch: refs/heads/COUCHDB-3287-pluggable-storage-engines
Commit: 50dcd7d7c5f7ce003e8e2fc84646c1aa9931ebaa
Parents: 7a2b2b6
Author: Nick Vatamaniuc <va...@apache.org>
Authored: Thu Mar 2 19:12:47 2017 -0500
Committer: Nick Vatamaniuc <va...@apache.org>
Committed: Thu Mar 2 19:12:47 2017 -0500
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 55 ++++++++++++++++++-----------------
1 file changed, 29 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/50dcd7d7/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index c9c0a68..2bcad69 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -93,8 +93,8 @@ replication_started(#rep{id = {BaseId, _} = RepId}) ->
{<<"_replication_id">>, ?l2b(BaseId)},
{<<"_replication_stats">>, undefined}]),
ok = gen_server:call(?MODULE, {rep_started, RepId}, infinity),
- couch_log:notice("Document `~s` triggered replication `~s`",
- [DocId, pp_rep_id(RepId)])
+ couch_log:notice("Document `~s` from `~s` triggered replication `~s`",
+ [DocId, DbName, pp_rep_id(RepId)])
end.
@@ -108,8 +108,8 @@ replication_completed(#rep{id = RepId}, Stats) ->
{<<"_replication_state_reason">>, undefined},
{<<"_replication_stats">>, {Stats}}]),
ok = gen_server:call(?MODULE, {rep_complete, RepId}, infinity),
- couch_log:notice("Replication `~s` finished (triggered by document `~s`)",
- [pp_rep_id(RepId), DocId])
+ couch_log:notice("Replication `~s` finished (triggered by document `~s`"
+ " from `~s`)", [pp_rep_id(RepId), DocId, DbName])
end.
@@ -117,10 +117,10 @@ replication_usurped(#rep{id = RepId}, By) ->
case rep_state(RepId) of
nil ->
ok;
- #rep_state{rep = #rep{doc_id = DocId}} ->
+ #rep_state{rep = #rep{db_name = DbName, doc_id = DocId}} ->
ok = gen_server:call(?MODULE, {rep_complete, RepId}, infinity),
- couch_log:notice("Replication `~s` usurped by ~s (triggered by document `~s`)",
- [pp_rep_id(RepId), By, DocId])
+ couch_log:notice("Replication `~s` usurped by ~s (triggered by document"
+ " `~s` from `~s`)", [pp_rep_id(RepId), By, DocId, DbName])
end.
@@ -461,10 +461,11 @@ process_update(State, DbName, {Change}) ->
rep_doc_deleted(DbName, DocId),
State;
{Owner, false} when Owner /= node() ->
- couch_log:notice("Not starting '~s' as owner is ~s.", [DocId, Owner]),
+ couch_log:notice("Not starting '~s' from '~s' as owner is ~s.",
+ [DocId, DbName, Owner]),
State;
{_Owner, false} ->
- couch_log:notice("Maybe starting '~s' as I'm the owner", [DocId]),
+ couch_log:notice("Maybe starting '~s' from '~s' as I'm the owner", [DocId, DbName]),
case get_json_value(<<"_replication_state">>, RepProps) of
undefined ->
maybe_start_replication(State, DbName, DocId, JsonRepDoc);
@@ -497,8 +498,8 @@ rep_db_update_error(Error, DbName, DocId) ->
_ ->
Reason = to_binary(Error)
end,
- couch_log:error("Replication manager, error processing document `~s`: ~s",
- [DocId, Reason]),
+ couch_log:error("Replication manager, error processing document `~s`"
+ " from `~s`: ~s", [DocId, DbName, Reason]),
update_rep_doc(DbName, DocId, [{<<"_replication_state">>, <<"error">>},
{<<"_replication_state_reason">>, Reason}]).
@@ -528,8 +529,8 @@ maybe_start_replication(State, DbName, DocId, RepDoc) ->
},
true = ets:insert(?REP_TO_STATE, {RepId, RepState}),
true = ets:insert(?DOC_TO_REP, {{DbName, DocId}, RepId}),
- couch_log:notice("Attempting to start replication `~s` (document `~s`).",
- [pp_rep_id(RepId), DocId]),
+ couch_log:notice("Attempting to start replication `~s` (document `~s`"
+ " from `~s`).", [pp_rep_id(RepId), DocId, DbName]),
StartDelaySecs = erlang:max(0,
config:get_integer("replicator", "start_delay", 10)),
StartSplaySecs = erlang:max(1,
@@ -544,13 +545,15 @@ maybe_start_replication(State, DbName, DocId, RepDoc) ->
#rep_state{rep = #rep{doc_id = DocId}} ->
State;
#rep_state{starting = false, rep = #rep{db_name = DbName, doc_id = OtherDocId}} ->
- couch_log:notice("The replication specified by the document `~s` was already"
- " triggered by the document `~s`", [DocId, OtherDocId]),
+ couch_log:notice("The replication specified by the document `~s` from"
+ " `~s` was already triggered by the document `~s`",
+ [DocId, DbName, OtherDocId]),
maybe_tag_rep_doc(DbName, DocId, RepDoc, ?l2b(BaseId)),
State;
#rep_state{starting = true, rep = #rep{db_name = DbName, doc_id = OtherDocId}} ->
- couch_log:notice("The replication specified by the document `~s` is already"
- " being triggered by the document `~s`", [DocId, OtherDocId]),
+ couch_log:notice("The replication specified by the document `~s` from"
+ " `~s` is already being triggered by the document `~s`",
+ [DocId, DbName, OtherDocId]),
maybe_tag_rep_doc(DbName, DocId, RepDoc, ?l2b(BaseId)),
State
end.
@@ -618,8 +621,8 @@ rep_doc_deleted(DbName, DocId) ->
couch_replicator:cancel_replication(RepId),
true = ets:delete(?REP_TO_STATE, RepId),
true = ets:delete(?DOC_TO_REP, {DbName, DocId}),
- couch_log:notice("Stopped replication `~s` because replication document `~s`"
- " was deleted", [pp_rep_id(RepId), DocId]);
+ couch_log:notice("Stopped replication `~s` because replication document"
+ " `~s` from `~s` was deleted", [pp_rep_id(RepId), DocId, DbName]);
[] ->
ok
end.
@@ -641,20 +644,20 @@ maybe_retry_replication(#rep_state{retries_left = 0} = RepState, Error, State) -
couch_replicator:cancel_replication(RepId),
true = ets:delete(?REP_TO_STATE, RepId),
true = ets:delete(?DOC_TO_REP, {DbName, DocId}),
- couch_log:error("Error in replication `~s` (triggered by document `~s`): ~s"
- "~nReached maximum retry attempts (~p).",
- [pp_rep_id(RepId), DocId, to_binary(error_reason(Error)), MaxRetries]),
+ couch_log:error("Error in replication `~s` (triggered by document `~s` from"
+ " `~s` ): ~s~nReached maximum retry attempts (~p).", [pp_rep_id(RepId),
+ DocId, DbName, to_binary(error_reason(Error)), MaxRetries]),
State;
maybe_retry_replication(RepState, Error, State) ->
#rep_state{
- rep = #rep{id = RepId, doc_id = DocId} = Rep
+ rep = #rep{id = RepId, doc_id = DocId, db_name = DbName} = Rep
} = RepState,
#rep_state{wait = Wait} = NewRepState = state_after_error(RepState),
true = ets:insert(?REP_TO_STATE, {RepId, NewRepState}),
- couch_log:error("Error in replication `~s` (triggered by document `~s`): ~s"
- "~nRestarting replication in ~p seconds.",
- [pp_rep_id(RepId), DocId, to_binary(error_reason(Error)), Wait]),
+ couch_log:error("Error in replication `~s` (triggered by document `~s` from"
+ " `~s` ): ~s~nRestarting replication in ~p seconds.", [pp_rep_id(RepId),
+ DocId, DbName, to_binary(error_reason(Error)), Wait]),
Pid = spawn_link(?MODULE, start_replication, [Rep, Wait]),
State#state{
rep_start_pids = [{rep_start, Pid} | State#state.rep_start_pids]