You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by da...@apache.org on 2014/02/06 18:18:59 UTC
[01/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Updated Branches:
refs/heads/import-rcouch [created] 589d9589c
Unknown errors on _replicate POST are 500's not 404's
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/01e49b42
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/01e49b42
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/01e49b42
Branch: refs/heads/import-rcouch
Commit: 01e49b42f2fb3a7f70b55b9ab785d8b7bfe7b71a
Parents: 7089f60
Author: Robert Newson <rn...@apache.org>
Authored: Mon Apr 29 19:47:20 2013 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Mon Apr 29 19:47:20 2013 +0100
----------------------------------------------------------------------
src/couch_replicator_httpd.erl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/01e49b42/src/couch_replicator_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpd.erl b/src/couch_replicator_httpd.erl
index eccf885..c0fe004 100644
--- a/src/couch_replicator_httpd.erl
+++ b/src/couch_replicator_httpd.erl
@@ -35,7 +35,7 @@ handle_req(#httpd{method = 'POST', user_ctx = UserCtx} = Req) ->
case couch_replicator:replicate(Rep) of
{error, {Error, Reason}} ->
send_json(
- Req, 404,
+ Req, 500,
{[{error, to_binary(Error)}, {reason, to_binary(Reason)}]});
{error, not_found} ->
% Tried to cancel a replication that didn't exist.
[22/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
add `make test` target . all erlang test pass
note: from time to time there is a timing issue on 200- test that need
to be fixed. It is most probably due to the way the indexer is
supervised.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/2a8500c0
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/2a8500c0
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/2a8500c0
Branch: refs/heads/import-rcouch
Commit: 2a8500c0f4b72e9ecbb57ff76f322132dcb7a93a
Parents: ac076e9
Author: benoitc <be...@apache.org>
Authored: Thu Jan 9 16:24:32 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
test/01-load.t | 1 +
test/02-httpc-pool.t | 2 ++
test/03-replication-compact.t | 6 +++++-
test/04-replication-large-atts.t | 4 ++++
test/05-replication-many-leaves.t | 2 ++
test/06-doc-missing-stubs.t | 3 +++
test/07-use-checkpoints.t | 3 +++
7 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/01-load.t
----------------------------------------------------------------------
diff --git a/test/01-load.t b/test/01-load.t
index 8bd82dd..d901ee8 100644
--- a/test/01-load.t
+++ b/test/01-load.t
@@ -1,5 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index a7bde6c..aac2eb5 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -1,5 +1,7 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 7c4d38c..3e52b37 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -1,5 +1,7 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
@@ -83,6 +85,8 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link(),
+
ibrowse:start(),
Pairs = [
@@ -484,5 +488,5 @@ cancel_replication(RepId, RepPid) ->
att_data() ->
{ok, Data} = file:read_file(
- test_util:source_file("share/www/image/logo.png")),
+ test_util:build_file("share/www/image/logo.png")),
Data.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index a7063c7..da927e9 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -1,5 +1,7 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
@@ -64,6 +66,8 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link(),
+
application:start(ibrowse),
application:start(crypto),
couch_config:set("attachments", "compressible_types", "text/*", false),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index 212ee99..8a5008b 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -76,6 +76,8 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link(),
+
ibrowse:start(),
crypto:start(),
couch_config:set("replicator", "connection_timeout", "90000", false),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index e17efc9..cee19ca 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -1,5 +1,7 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
@@ -68,6 +70,7 @@ main(_) ->
% Test motivated by COUCHDB-1365.
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link(),
ibrowse:start(),
Pairs = [
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2a8500c0/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
index cefc1a7..af7ec44 100755
--- a/test/07-use-checkpoints.t
+++ b/test/07-use-checkpoints.t
@@ -1,4 +1,6 @@
#!/usr/bin/env escript
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
%% -*- erlang -*-
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
@@ -92,6 +94,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link(),
ibrowse:start(),
test_use_checkpoints(false),
[09/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Be a bit nicer about logging
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/eee70d84
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/eee70d84
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/eee70d84
Branch: refs/heads/import-rcouch
Commit: eee70d84d780019cb0a921f0b83f0c904cb07f6a
Parents: b1bf3fb
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Oct 2 15:25:57 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 15:35:54 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 10 +++++++++-
1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/eee70d84/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index f779747..eb7d845 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -222,7 +222,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
#httpdb{retries = Retries, wait = Wait0} = HttpDb,
Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT),
?LOG_INFO("Retrying GET to ~s in ~p seconds due to error ~p",
- [Url, Wait / 1000, Else]
+ [Url, Wait / 1000, error_reason(Else)]
),
ok = timer:sleep(Wait),
RetryDb = HttpDb#httpdb{
@@ -235,6 +235,14 @@ open_doc_revs(Db, Id, Revs, Options, Fun, Acc) ->
{ok, Results} = couch_db:open_doc_revs(Db, Id, Revs, Options),
{ok, lists:foldl(fun(R, A) -> {_, A2} = Fun(R, A), A2 end, Acc, Results)}.
+error_reason({http_request_failed, "GET", _Url, {error, timeout}}) ->
+ timeout;
+error_reason({http_request_failed, "GET", _Url, {error, {_, req_timedout}}}) ->
+ req_timedout;
+error_reason({http_request_failed, "GET", _Url, Error}) ->
+ Error;
+error_reason(Else) ->
+ Else.
open_doc(#httpdb{} = Db, Id, Options) ->
send_req(
[26/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
fix replication tests
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/62b62ed7
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/62b62ed7
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/62b62ed7
Branch: refs/heads/import-rcouch
Commit: 62b62ed7c500cc6c9092cebaf46b46cbcb005202
Parents: 5610fe8
Author: benoitc <be...@apache.org>
Authored: Thu Jan 9 23:41:26 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
test/05-replication-many-leaves.t | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/62b62ed7/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index 8a5008b..d10419f 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -1,5 +1,7 @@
#!/usr/bin/env escript
%% -*- erlang -*-
+%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
% the License at
[02/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
COUCHDB-1788 - tolerate undefined source/target when reading _replicator docs
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7c9704c4
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7c9704c4
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7c9704c4
Branch: refs/heads/import-rcouch
Commit: 7c9704c449a1883843491ab68c34a1b7b92f28c9
Parents: 01e49b4
Author: Robert Newson <rn...@apache.org>
Authored: Mon May 6 10:34:04 2013 +0100
Committer: Robert Newson <rn...@apache.org>
Committed: Mon May 6 10:34:04 2013 +0100
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7c9704c4/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index fec44df..bbb0e11 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -692,6 +692,8 @@ after_doc_read(#doc{body = {Body}} = Doc, #db{user_ctx=UserCtx} = Db) ->
end.
+strip_credentials(undefined) ->
+ undefined;
strip_credentials(Url) when is_binary(Url) ->
re:replace(Url,
"http(s)?://(?:[^:]+):[^@]+@(.*)$",
[19/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
remove Makefile.am from apps/ and fix couchspawnkillable
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ac076e96
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ac076e96
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ac076e96
Branch: refs/heads/import-rcouch
Commit: ac076e9669dd053b58ac102e382f69bf91a93b49
Parents: 7ffb923
Author: benoitc <be...@apache.org>
Authored: Tue Jan 7 19:15:43 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
Makefile.am | 78 --------------------------------------------------------
1 file changed, 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ac076e96/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
deleted file mode 100644
index 2dcd47d..0000000
--- a/Makefile.am
+++ /dev/null
@@ -1,78 +0,0 @@
-## Licensed under the Apache License, Version 2.0 (the "License"); you may not
-## use this file except in compliance with the License. You may obtain a copy of
-## the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-## WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-## License for the specific language governing permissions and limitations under
-## the License.
-
-couch_replicatorlibdir = $(localerlanglibdir)/couch_replicator-0.1
-couch_replicatorincludedir = $(couch_replicatorlibdir)/include
-couch_replicatorebindir = $(couch_replicatorlibdir)/ebin
-
-couch_replicatorinclude_DATA = $(include_files)
-couch_replicatorebin_DATA = $(compiled_files)
-
-include_files = \
- src/couch_replicator_api_wrap.hrl \
- src/couch_replicator.hrl \
- src/couch_replicator_js_functions.hrl
-
-source_files = \
- src/couch_replicator_api_wrap.erl \
- src/couch_replicator_httpc_pool.erl \
- src/couch_replicator_httpc.erl \
- src/couch_replicator_httpd.erl \
- src/couch_replicator_job_sup.erl \
- src/couch_replicator_notifier.erl \
- src/couch_replicator_manager.erl \
- src/couch_replicator_utils.erl \
- src/couch_replicator_worker.erl \
- src/couch_replicator.app.src \
- src/couch_replicator.erl
-
-test_files = \
- test/01-load.t \
- test/02-httpc-pool.t \
- test/03-replication-compact.t \
- test/04-replication-large-atts.t \
- test/05-replication-many-leaves.t \
- test/06-doc-missing-stubs.t \
- test/07-use-checkpoints.t
-
-compiled_files = \
- ebin/couch_replicator_api_wrap.beam \
- ebin/couch_replicator_httpc_pool.beam \
- ebin/couch_replicator_httpc.beam \
- ebin/couch_replicator_httpd.beam \
- ebin/couch_replicator_job_sup.beam \
- ebin/couch_replicator_notifier.beam \
- ebin/couch_replicator_manager.beam \
- ebin/couch_replicator_utils.beam \
- ebin/couch_replicator_worker.beam \
- ebin/couch_replicator.app \
- ebin/couch_replicator.beam
-
-EXTRA_DIST = $(include_files) $(source_files) $(test_files)
-CLEANFILES = $(compiled_files)
-
-check:
-if TESTS
- $(abs_top_builddir)/test/etap/run $(abs_top_srcdir)/src/couch_replicator/test
-endif
-
-ebin/%.app: src/%.app.src
- @mkdir -p ebin/
- sed -e "s|%version%|@version@|g" \
- < $< > $@
-
-ebin/%.beam: src/%.erl $(include_files)
- @mkdir -p ebin/
- $(ERLC) -Wall -I$(top_srcdir)/src -I$(top_srcdir)/src/couchdb \
- -o ebin/ $(ERLC_FLAGS) ${TEST} $<;
-
-
[20/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
make couch_httpd a full couch application
With this change, the HTTP API is now handled by its own erlang
application and supervision. It also improved the way you can reload the
configuration and the modules. Now upgrading a listener or its
configuration is first removing it from the supervsion then start the
new process with the new configuration. This behaviour is similar to the
one you have in nginx.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/6d095edc
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/6d095edc
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/6d095edc
Branch: refs/heads/import-rcouch
Commit: 6d095edc068775b31daa38a0dadeb54bcb9deee9
Parents: 62b62ed
Author: benoitc <be...@apache.org>
Authored: Sat Jan 11 11:20:27 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
test/02-httpc-pool.t | 1 +
test/03-replication-compact.t | 3 ++-
test/04-replication-large-atts.t | 3 ++-
test/05-replication-many-leaves.t | 3 ++-
test/06-doc-missing-stubs.t | 3 ++-
test/07-use-checkpoints.t | 3 ++-
6 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index aac2eb5..50f40db 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -30,6 +30,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
ibrowse:start(),
test_pool_full(),
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 801b173..d42049a 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -85,6 +85,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
couch_replicator_sup:start_link(),
ibrowse:start(),
@@ -440,7 +441,7 @@ maybe_pause(Parent, Counter) ->
db_url(DbName) ->
iolist_to_binary([
"http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_http, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index da927e9..eb750e6 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -66,6 +66,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
couch_replicator_sup:start_link(),
application:start(ibrowse),
@@ -232,7 +233,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
"http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_http, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index d10419f..5c2aa27 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -78,6 +78,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
couch_replicator_sup:start_link(),
ibrowse:start(),
@@ -259,7 +260,7 @@ add_attachments(SourceDb, [{DocId, RevList} | Rest], NumAtts, IdRevsAcc) ->
db_url(DbName) ->
iolist_to_binary([
"http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_http, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index cee19ca..2e37d59 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -70,6 +70,7 @@ main(_) ->
% Test motivated by COUCHDB-1365.
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
couch_replicator_sup:start_link(),
ibrowse:start(),
@@ -268,7 +269,7 @@ att_decoded_md5(Att) ->
db_url(DbName) ->
iolist_to_binary([
"http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_http, port)),
"/", DbName
]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/6d095edc/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
index af7ec44..1631328 100755
--- a/test/07-use-checkpoints.t
+++ b/test/07-use-checkpoints.t
@@ -94,6 +94,7 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
+ couch_httpd_sup:start_link(),
couch_replicator_sup:start_link(),
ibrowse:start(),
@@ -210,7 +211,7 @@ compare_dbs(#db{name = SourceName}, #db{name = TargetName}) ->
db_url(DbName) ->
iolist_to_binary([
"http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
- ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_http, port)),
"/", DbName
]).
[17/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
inital move to rebar compilation
- move src/apps
- download dependencies using rebar
- replace ejson by jiffy
- replace couch_drv & couch_ejson_compare by couch_collate
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/abaeaaba
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/abaeaaba
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/abaeaaba
Branch: refs/heads/import-rcouch
Commit: abaeaaba870e7007dc80673b1b98a78a2959ba2a
Parents: b58ffb9
Author: benoitc <be...@apache.org>
Authored: Mon Jan 6 21:12:45 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:53 2014 -0600
----------------------------------------------------------------------
src/couch_replicator.erl | 2 +-
src/couch_replicator_api_wrap.erl | 2 +-
src/couch_replicator_httpc.erl | 4 ++--
src/couch_replicator_httpc_pool.erl | 2 +-
src/couch_replicator_httpd.erl | 2 +-
src/couch_replicator_job_sup.erl | 2 +-
src/couch_replicator_manager.erl | 2 +-
src/couch_replicator_notifier.erl | 2 +-
src/couch_replicator_utils.erl | 4 ++--
src/couch_replicator_worker.erl | 2 +-
10 files changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 91039b6..d470c8a 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -24,7 +24,7 @@
-export([init/1, terminate/2, code_change/3]).
-export([handle_call/3, handle_cast/2, handle_info/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator_api_wrap.hrl").
-include("couch_replicator.hrl").
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 52e15b7..311025b 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -18,7 +18,7 @@
% Notes:
% Many options and apis aren't yet supported here, they are added as needed.
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator_api_wrap.hrl").
-export([
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_httpc.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc.erl b/src/couch_replicator_httpc.erl
index b8fb31b..c13bf18 100644
--- a/src/couch_replicator_httpc.erl
+++ b/src/couch_replicator_httpc.erl
@@ -12,9 +12,9 @@
-module(couch_replicator_httpc).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator_api_wrap.hrl").
--include("../ibrowse/ibrowse.hrl").
+-include_lib("ibrowse/include/ibrowse.hrl").
-export([setup/1]).
-export([send_req/3]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_httpc_pool.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpc_pool.erl b/src/couch_replicator_httpc_pool.erl
index b065b7c..a82a2df 100644
--- a/src/couch_replicator_httpc_pool.erl
+++ b/src/couch_replicator_httpc_pool.erl
@@ -21,7 +21,7 @@
-export([init/1, handle_call/3, handle_info/2, handle_cast/2]).
-export([code_change/3, terminate/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-import(couch_util, [
get_value/2,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_httpd.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_httpd.erl b/src/couch_replicator_httpd.erl
index c0fe004..0a21d52 100644
--- a/src/couch_replicator_httpd.erl
+++ b/src/couch_replicator_httpd.erl
@@ -12,7 +12,7 @@
-module(couch_replicator_httpd).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-import(couch_httpd, [
send_json/2,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_job_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_job_sup.erl b/src/couch_replicator_job_sup.erl
index e8a7b96..484cc1a 100644
--- a/src/couch_replicator_job_sup.erl
+++ b/src/couch_replicator_job_sup.erl
@@ -14,7 +14,7 @@
-behaviour(supervisor).
-export([init/1, start_link/0]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
start_link() ->
supervisor:start_link({local,?MODULE}, ?MODULE, []).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index 4891c4c..8055727 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -22,7 +22,7 @@
-export([start_link/0, init/1, handle_call/3, handle_info/2, handle_cast/2]).
-export([code_change/3, terminate/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator.hrl").
-include("couch_replicator_js_functions.hrl").
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_notifier.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_notifier.erl b/src/couch_replicator_notifier.erl
index 3b48b71..39fd68b 100644
--- a/src/couch_replicator_notifier.erl
+++ b/src/couch_replicator_notifier.erl
@@ -21,7 +21,7 @@
-export([init/1, terminate/2, code_change/3]).
-export([handle_event/2, handle_call/2, handle_info/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
start_link(FunAcc) ->
couch_event_sup:start_link(couch_replication,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 240d7d4..0baddc2 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -18,10 +18,10 @@
-export([replication_id/2]).
-export([sum_stats/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator_api_wrap.hrl").
-include("couch_replicator.hrl").
--include("../ibrowse/ibrowse.hrl").
+-include_lib("ibrowse/include/ibrowse.hrl").
-import(couch_util, [
get_value/2,
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/abaeaaba/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index d66d478..0f65900 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -20,7 +20,7 @@
-export([init/1, terminate/2, code_change/3]).
-export([handle_call/3, handle_cast/2, handle_info/2]).
--include("couch_db.hrl").
+-include_lib("couch/include/couch_db.hrl").
-include("couch_replicator_api_wrap.hrl").
-include("couch_replicator.hrl").
[03/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Use latest=true when retrieving revisions
This allows the replicator to keep up with revisions that may have
occured after missing_revs was called.
BugzID: 14241
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/27dbba31
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/27dbba31
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/27dbba31
Branch: refs/heads/import-rcouch
Commit: 27dbba3162c47261f8acca907b32fa6f70f8716a
Parents: 7c9704c
Author: Bob Dionne <bo...@cloudant.com>
Authored: Tue Aug 7 12:42:36 2012 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 11:59:02 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 ++
src/couch_replicator_worker.erl | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/27dbba31/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 90cfa8e..cd69e59 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -465,6 +465,8 @@ options_to_query_args([revs | Rest], Acc) ->
options_to_query_args(Rest, [{"revs", "true"} | Acc]);
options_to_query_args([{open_revs, all} | Rest], Acc) ->
options_to_query_args(Rest, [{"open_revs", "all"} | Acc]);
+options_to_query_args([latest | Rest], Acc) ->
+ options_to_query_args(Rest, [{"latest", "true"} | Acc]);
options_to_query_args([{open_revs, Revs} | Rest], Acc) ->
JsonRevs = ?b2l(?JSON_ENCODE(couch_doc:revs_to_strs(Revs))),
options_to_query_args(Rest, [{"open_revs", JsonRevs} | Acc]).
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/27dbba31/src/couch_replicator_worker.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_worker.erl b/src/couch_replicator_worker.erl
index e8a3570..d66d478 100644
--- a/src/couch_replicator_worker.erl
+++ b/src/couch_replicator_worker.erl
@@ -296,13 +296,13 @@ spawn_doc_reader(Source, Target, FetchParams) ->
fetch_doc(Source, {Id, Revs, PAs}, DocHandler, Acc) ->
try
couch_replicator_api_wrap:open_doc_revs(
- Source, Id, Revs, [{atts_since, PAs}], DocHandler, Acc)
+ Source, Id, Revs, [{atts_since, PAs}, latest], DocHandler, Acc)
catch
throw:{missing_stub, _} ->
?LOG_ERROR("Retrying fetch and update of document `~s` due to out of "
"sync attachment stubs. Missing revisions are: ~s",
[Id, couch_doc:revs_to_strs(Revs)]),
- couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [], DocHandler, Acc)
+ couch_replicator_api_wrap:open_doc_revs(Source, Id, Revs, [latest], DocHandler, Acc)
end.
[24/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
fix tests
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/5610fe8a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/5610fe8a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/5610fe8a
Branch: refs/heads/import-rcouch
Commit: 5610fe8a5be9bacb640cdf22c831fd7e34d574d9
Parents: 2a8500c
Author: benoitc <be...@apache.org>
Authored: Thu Jan 9 20:56:15 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
src/couch_replicator_sup.erl | 6 +++++-
test/03-replication-compact.t | 5 ++++-
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/5610fe8a/src/couch_replicator_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_sup.erl b/src/couch_replicator_sup.erl
index 905e2cf..7ae5367 100644
--- a/src/couch_replicator_sup.erl
+++ b/src/couch_replicator_sup.erl
@@ -17,6 +17,7 @@
%% API
-export([start_link/0]).
+-export([stop/0]).
%% Supervisor callbacks
-export([init/1]).
@@ -31,6 +32,9 @@
start_link() ->
supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+stop() ->
+ catch exit(whereis(couch_replicator_sup), normal).
+
%% ===================================================================
%% Supervisor callbacks
%% ===================================================================
@@ -56,4 +60,4 @@ init([]) ->
supervisor,
[couch_replicator_manager_sup]}
],
- {ok, { {one_for_one, 10, 3600}, Children} }.
+ {ok, { {one_for_one, 100, 3600}, Children} }.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/5610fe8a/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index 3e52b37..801b173 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -119,8 +119,11 @@ test() ->
delete_db(SourceDb),
delete_db(TargetDb),
couch_server_sup:stop(),
+ couch_replicator_sup:stop(),
+
ok = timer:sleep(1000),
- couch_server_sup:start_link(test_util:config_files())
+ couch_server_sup:start_link(test_util:config_files()),
+ couch_replicator_sup:start_link()
end,
Pairs),
[10/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Simplify doc_streamer initialization
The unlink at the end is a noop, so we can simplify this to use
spawn_link/3 and the fully-exported function call instead of
spawn_link/1 and a closure.
BugzID: 20822
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b1bf3fb4
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b1bf3fb4
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b1bf3fb4
Branch: refs/heads/import-rcouch
Commit: b1bf3fb40d9b8c40698bd5b12830d161feac5754
Parents: 1c501b8
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Thu Sep 12 12:56:50 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 15:35:54 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b1bf3fb4/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index fccb759..f779747 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -221,7 +221,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
),
#httpdb{retries = Retries, wait = Wait0} = HttpDb,
Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT),
- twig:log(notice,"Retrying GET to ~s in ~p seconds due to error ~p",
+ ?LOG_INFO("Retrying GET to ~s in ~p seconds due to error ~p",
[Url, Wait / 1000, Else]
),
ok = timer:sleep(Wait),
@@ -834,6 +834,12 @@ rev_to_str({_Pos, _Id} = Rev) ->
rev_to_str(Rev) ->
Rev.
+write_fun() ->
+ fun(Data) ->
+ receive {get_data, Ref, From} ->
+ From ! {data, Ref, Data}
+ end
+ end.
stream_doc({JsonBytes, Atts, Boundary, Len}) ->
case erlang:erase({doc_streamer, Boundary}) of
@@ -843,17 +849,11 @@ stream_doc({JsonBytes, Atts, Boundary, Len}) ->
_ ->
ok
end,
- Self = self(),
- DocStreamer = spawn_link(fun() ->
- couch_doc:doc_to_multi_part_stream(
- Boundary, JsonBytes, Atts,
- fun(Data) ->
- receive {get_data, Ref, From} ->
- From ! {data, Ref, Data}
- end
- end, true),
- unlink(Self)
- end),
+ DocStreamer = spawn_link(
+ couch_doc,
+ doc_to_multi_part_stream,
+ [Boundary, JsonBytes, Atts, write_fun(), true]
+ ),
erlang:put({doc_streamer, Boundary}, DocStreamer),
{ok, <<>>, {Len, Boundary}};
stream_doc({0, Id}) ->
[25/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
start and stop the application correctly
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/589d9589
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/589d9589
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/589d9589
Branch: refs/heads/import-rcouch
Commit: 589d9589cdd682a56707a89326597ac6fc3e8739
Parents: 22ca019
Author: benoitc <be...@apache.org>
Authored: Mon Jan 13 22:30:39 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
test/02-httpc-pool.t | 8 ++------
test/03-replication-compact.t | 21 ++++++---------------
test/04-replication-large-atts.t | 10 ++--------
test/05-replication-many-leaves.t | 10 +++-------
test/06-doc-missing-stubs.t | 9 ++-------
test/07-use-checkpoints.t | 9 ++-------
6 files changed, 17 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index 9446c9b..560b6a1 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -15,8 +15,6 @@
% the License.
main(_) ->
- test_util:init_code_path(),
-
etap:plan(55),
case (catch test()) of
ok ->
@@ -29,15 +27,13 @@ main(_) ->
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- ibrowse:start(),
+ test_util:start_couch(),
test_pool_full(),
test_worker_dead_pool_non_full(),
test_worker_dead_pool_full(),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index d89a539..699f698 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -69,10 +69,8 @@ source_db_name() -> <<"couch_test_rep_db_a">>.
target_db_name() -> <<"couch_test_rep_db_b">>.
-main(_) ->
- test_util:init_code_path(),
-
- etap:plan(376),
+main(_) ->
+etap:plan(376),
case (catch test()) of
ok ->
etap:end_tests();
@@ -84,11 +82,7 @@ main(_) ->
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- couch_replicator_sup:start_link(),
-
- ibrowse:start(),
+ test_util:start_couch(),
Pairs = [
{source_db_name(), target_db_name()},
@@ -119,16 +113,13 @@ test() ->
delete_db(SourceDb),
delete_db(TargetDb),
- couch_server_sup:stop(),
- couch_replicator_sup:stop(),
-
+ test_util:stop_couch(),
ok = timer:sleep(1000),
- couch_server_sup:start_link(test_util:config_files()),
- couch_replicator_sup:start_link()
+ test_util:start_couch()
end,
Pairs),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index 04e7861..c001f3e 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -51,8 +51,6 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
main(_) ->
- test_util:init_code_path(),
-
etap:plan(1192),
case (catch test()) of
ok ->
@@ -65,12 +63,8 @@ main(_) ->
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- couch_replicator_sup:start_link(),
+ test_util:start_couch(),
- application:start(ibrowse),
- application:start(crypto),
couch_config:set("attachments", "compressible_types", "text/*", false),
Pairs = [
@@ -103,7 +97,7 @@ test() ->
Pairs),
delete_db(SourceDb),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index fbb9d95..c4c0007 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -77,12 +77,8 @@ main(_) ->
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- couch_replicator_sup:start_link(),
-
- ibrowse:start(),
- crypto:start(),
+ test_util:start_couch(),
+
couch_config:set("replicator", "connection_timeout", "90000", false),
Pairs = [
@@ -130,7 +126,7 @@ test() ->
end,
Pairs),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index 5fbb7f3..5db5830 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -54,8 +54,6 @@ target_revs_limit() -> 3.
main(_) ->
- test_util:init_code_path(),
-
etap:plan(128),
case (catch test()) of
ok ->
@@ -69,10 +67,7 @@ main(_) ->
% Test motivated by COUCHDB-1365.
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- couch_replicator_sup:start_link(),
- ibrowse:start(),
+ test_util:start_couch(),
Pairs = [
{source_db_name(), target_db_name()},
@@ -113,7 +108,7 @@ test() ->
end,
Pairs),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/589d9589/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
index 193b17d..9f975e2 100755
--- a/test/07-use-checkpoints.t
+++ b/test/07-use-checkpoints.t
@@ -79,8 +79,6 @@ target_db_name() -> <<"couch_test_rep_db_b">>.
main(_) ->
- test_util:init_code_path(),
-
etap:plan(16),
case (catch test()) of
ok ->
@@ -93,15 +91,12 @@ main(_) ->
test() ->
- couch_server_sup:start_link(test_util:config_files()),
- couch_httpd_sup:start_link(),
- couch_replicator_sup:start_link(),
- ibrowse:start(),
+ test_util:start_couch(),
test_use_checkpoints(false),
test_use_checkpoints(true),
- couch_server_sup:stop(),
+ test_util:stop_couch(),
ok.
[23/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
working release
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/7ffb923c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/7ffb923c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/7ffb923c
Branch: refs/heads/import-rcouch
Commit: 7ffb923c5096e7329a74d2069e3fe128c45daedf
Parents: 9953fe5
Author: benoitc <be...@apache.org>
Authored: Tue Jan 7 16:14:56 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
src/couch_replicator.app.src | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/7ffb923c/src/couch_replicator.app.src
----------------------------------------------------------------------
diff --git a/src/couch_replicator.app.src b/src/couch_replicator.app.src
index 1e62edf..3e71b3b 100644
--- a/src/couch_replicator.app.src
+++ b/src/couch_replicator.app.src
@@ -12,7 +12,7 @@
{application, couch_replicator, [
{description, "CouchDB replicator"},
- {vsn, "%version%"},
+ {vsn, "1.6.1"},
{modules, []},
{registered, [
couch_replicator_manager_sup,
[21/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
move src/* to the root
There is no need of an src/ folder except for an aesthetic reason, we
are obviously distributing a source. This layout is alos more common in
the Erlang world, and most editor and package manager understand it.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/22ca0195
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/22ca0195
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/22ca0195
Branch: refs/heads/import-rcouch
Commit: 22ca01957225611d847da73a9f0a7579e5db25dd
Parents: 6d095ed
Author: benoitc <be...@apache.org>
Authored: Sun Jan 12 10:17:31 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:54 2014 -0600
----------------------------------------------------------------------
test/01-load.t | 2 +-
test/02-httpc-pool.t | 2 +-
test/03-replication-compact.t | 2 +-
test/04-replication-large-atts.t | 2 +-
test/05-replication-many-leaves.t | 2 +-
test/06-doc-missing-stubs.t | 2 +-
test/07-use-checkpoints.t | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/01-load.t
----------------------------------------------------------------------
diff --git a/test/01-load.t b/test/01-load.t
index d901ee8..b900297 100644
--- a/test/01-load.t
+++ b/test/01-load.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/02-httpc-pool.t
----------------------------------------------------------------------
diff --git a/test/02-httpc-pool.t b/test/02-httpc-pool.t
index 50f40db..9446c9b 100755
--- a/test/02-httpc-pool.t
+++ b/test/02-httpc-pool.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/03-replication-compact.t
----------------------------------------------------------------------
diff --git a/test/03-replication-compact.t b/test/03-replication-compact.t
index d42049a..d89a539 100755
--- a/test/03-replication-compact.t
+++ b/test/03-replication-compact.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index eb750e6..04e7861 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/05-replication-many-leaves.t
----------------------------------------------------------------------
diff --git a/test/05-replication-many-leaves.t b/test/05-replication-many-leaves.t
index 5c2aa27..fbb9d95 100755
--- a/test/05-replication-many-leaves.t
+++ b/test/05-replication-many-leaves.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/06-doc-missing-stubs.t
----------------------------------------------------------------------
diff --git a/test/06-doc-missing-stubs.t b/test/06-doc-missing-stubs.t
index 2e37d59..5fbb7f3 100755
--- a/test/06-doc-missing-stubs.t
+++ b/test/06-doc-missing-stubs.t
@@ -1,6 +1,6 @@
#!/usr/bin/env escript
%% -*- erlang -*-
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
% use this file except in compliance with the License. You may obtain a copy of
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/22ca0195/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
index 1631328..193b17d 100755
--- a/test/07-use-checkpoints.t
+++ b/test/07-use-checkpoints.t
@@ -1,5 +1,5 @@
#!/usr/bin/env escript
-%%! -pa ./src/deps/*/ebin -pa ./src/apps/*/ebin -pa ./src/test/etap
+%%! -pa ./deps/*/ebin -pa ./apps/*/ebin -pa ./test/etap
%% -*- erlang -*-
% Licensed under the Apache License, Version 2.0 (the "License"); you may not
[13/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
configurable checkpoint interval
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/ee27edf1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/ee27edf1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/ee27edf1
Branch: refs/heads/import-rcouch
Commit: ee27edf10f3ddc2eb8619a154b92c617de4b6b02
Parents: a6d8389
Author: Robert Newson <rn...@apache.org>
Authored: Wed Nov 20 14:59:00 2013 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Nov 20 14:59:00 2013 +0000
----------------------------------------------------------------------
src/couch_replicator.erl | 25 ++++++++++++++-----------
src/couch_replicator_utils.erl | 6 +++++-
2 files changed, 19 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ee27edf1/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index b00efec..91039b6 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -69,7 +69,8 @@
source_monitor = nil,
target_monitor = nil,
source_seq = nil,
- use_checkpoints = true
+ use_checkpoints = true,
+ checkpoint_interval = 5000
}).
@@ -248,7 +249,8 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
target_name = TargetName,
start_seq = {_Ts, StartSeq},
source_seq = SourceCurSeq,
- committed_seq = {_, CommittedSeq}
+ committed_seq = {_, CommittedSeq},
+ checkpoint_interval = CheckpointInterval
} = State = init_state(Rep),
NumWorkers = get_value(worker_processes, Options),
@@ -289,7 +291,8 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
{doc_write_failures, 0},
{source_seq, SourceCurSeq},
{checkpointed_source_seq, CommittedSeq},
- {progress, 0}
+ {progress, 0},
+ {checkpoint_interval, CheckpointInterval}
]),
couch_task_status:set_update_frequency(1000),
@@ -456,9 +459,11 @@ handle_cast({report_seq, Seq},
{noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}.
-code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 30 ->
- {ok, erlang:append_element(OldState, true)};
-code_change(_OldVsn, State, _Extra) ->
+code_change(OldVsn, OldState, Extra) when tuple_size(OldState) =:= 30 ->
+ code_change(OldVsn, erlang:append_element(OldState, true), Extra);
+code_change(OldVsn, OldState, Extra) when tuple_size(OldState) =:= 31 ->
+ code_change(OldVsn, erlang:append_element(OldState, 5000), Extra);
+code_change(_OldVsn, #rep_state{}=State, _Extra) ->
{ok, State}.
@@ -508,7 +513,7 @@ do_last_checkpoint(#rep_state{seqs_in_progress = [],
start_timer(State) ->
- After = checkpoint_interval(State),
+ After = State#rep_state.checkpoint_interval,
case timer:apply_after(After, gen_server, cast, [self(), checkpoint]) of
{ok, Ref} ->
Ref;
@@ -567,7 +572,8 @@ init_state(Rep) ->
source_monitor = db_monitor(Source),
target_monitor = db_monitor(Target),
source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ),
- use_checkpoints = get_value(use_checkpoints, Options, true)
+ use_checkpoints = get_value(use_checkpoints, Options, true),
+ checkpoint_interval = get_value(checkpoint_interval, Options, 5000)
},
State#rep_state{timer = start_timer(State)}.
@@ -673,9 +679,6 @@ changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) ->
end.
-checkpoint_interval(_State) ->
- 5000.
-
do_checkpoint(#rep_state{use_checkpoints=false} = State) ->
NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} },
{ok, NewState};
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/ee27edf1/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index 1646c69..240d7d4 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -229,6 +229,7 @@ make_options(Props) ->
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
+ DefCheckpointInterval = couch_config:get("replicator", "checkpoint_interval", "5000"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
@@ -239,7 +240,8 @@ make_options(Props) ->
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
{worker_processes, list_to_integer(DefWorkers)},
- {use_checkpoints, list_to_existing_atom(UseCheckpoints)}
+ {use_checkpoints, list_to_existing_atom(UseCheckpoints)},
+ {checkpoint_interval, list_to_integer(DefCheckpointInterval)}
])).
@@ -283,6 +285,8 @@ convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
convert_options([{<<"use_checkpoints">>, V} | R]) ->
[{use_checkpoints, V} | convert_options(R)];
+convert_options([{<<"checkpoint_interval">>, V} | R]) ->
+ [{checkpoint_interval, couch_util:to_integer(V)} | convert_options(R)];
convert_options([_ | R]) -> % skip unknown option
convert_options(R).
[08/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Handle open_revs retries at a higher level
This patch disables the httpc client retries for the request to stream
document revisions to the replicator. The retry logic at that level
could end up jumbling together response body data from different
requests and thoroughly confusing the multipart parser. Moving the
retry logic up a level allows us to start fresh each time.
BugzID: 21367
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/61b0f145
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/61b0f145
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/61b0f145
Branch: refs/heads/import-rcouch
Commit: 61b0f145573d2665dd784196d5f72d22c90bb430
Parents: 1910622
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Tue Sep 10 22:26:14 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 12:11:24 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 98 +++++++++++++++++++++++-----------
1 file changed, 67 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/61b0f145/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 9921560..28f96c0 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -48,6 +48,7 @@
get_value/3
]).
+-define(MAX_WAIT, 5 * 60 * 1000).
db_uri(#httpdb{url = Url}) ->
couch_util:url_strip_password(Url);
@@ -157,43 +158,78 @@ get_missing_revs(Db, IdRevs) ->
+open_doc_revs(#httpdb{retries = 0} = HttpDb, Id, Revs, Options, _Fun, _Acc) ->
+ Path = encode_doc_id(Id),
+ QS = options_to_query_args(HttpDb, Path, [revs, {open_revs, Revs} | Options]),
+ Url = couch_util:url_strip_password(
+ couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}])
+ ),
+ ?LOG_ERROR("Replication crashing because GET ~s failed", [Url]),
+ exit(kaboom);
open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
Path = encode_doc_id(Id),
- QArgs = options_to_query_args(
- HttpDb, Path, [revs, {open_revs, Revs} | Options]),
- Self = self(),
- Streamer = spawn_link(fun() ->
- send_req(
- HttpDb,
- [{path, Path}, {qs, QArgs},
- {ibrowse_options, [{stream_to, {self(), once}}]},
- {headers, [{"Accept", "multipart/mixed"}]}],
- fun(200, Headers, StreamDataFun) ->
- remote_open_doc_revs_streamer_start(Self),
- {<<"--">>, _, _} = couch_httpd:parse_multipart_request(
- get_value("Content-Type", Headers),
- StreamDataFun,
- fun mp_parse_mixed/1)
- end),
- unlink(Self)
+ QS = options_to_query_args(HttpDb, Path, [revs, {open_revs, Revs} | Options]),
+ {Pid, Ref} = spawn_monitor(fun() ->
+ Self = self(),
+ Callback = fun(200, Headers, StreamDataFun) ->
+ remote_open_doc_revs_streamer_start(Self),
+ {<<"--">>, _, _} = couch_httpd:parse_multipart_request(
+ get_value("Content-Type", Headers),
+ StreamDataFun,
+ fun mp_parse_mixed/1
+ )
+ end,
+ Streamer = spawn_link(fun() ->
+ Params = [
+ {path, Path},
+ {qs, QS},
+ {ibrowse_options, [{stream_to, {self(), once}}]},
+ {headers, [{"Accept", "multipart/mixed"}]}
+ ],
+ % We're setting retries to 0 here to avoid the case where the
+ % Streamer retries the request and ends up jumbling together two
+ % different response bodies. Retries are handled explicitly by
+ % open_doc_revs itself.
+ send_req(HttpDb#httpdb{retries = 0}, Params, Callback)
+ end),
+ % If this process dies normally we can leave
+ % the Streamer process hanging around keeping an
+ % HTTP connection open. This is a bit of a
+ % hammer approach to making sure it releases
+ % that connection back to the pool.
+ spawn(fun() ->
+ Ref = erlang:monitor(process, Self),
+ receive
+ {'DOWN', Ref, process, Self, normal} ->
+ exit(Streamer, {streamer_parent_died, Self});
+ {'DOWN', Ref, process, Self, _} ->
+ ok
+ end
end),
- % If this process dies normally we can leave
- % the Streamer process hanging around keeping an
- % HTTP connection open. This is a bit of a
- % hammer approach to making sure it releases
- % that connection back to the pool.
- spawn(fun() ->
- Ref = erlang:monitor(process, Self),
receive
- {'DOWN', Ref, process, Self, normal} ->
- exit(Streamer, {streamer_parent_died, Self});
- {'DOWN', Ref, process, Self, _} ->
- ok
- end
+ {started_open_doc_revs, Ref} ->
+ Ret = receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc),
+ exit({exit_ok, Ret})
+ end
end),
receive
- {started_open_doc_revs, Ref} ->
- receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc)
+ {'DOWN', Ref, process, Pid, {exit_ok, Ret}} ->
+ Ret;
+ {'DOWN', Ref, process, Pid, Else} ->
+ Url = couch_util:url_strip_password(
+ couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}])
+ ),
+ #httpdb{retries = Retries, wait = Wait0} = HttpDb,
+ Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT),
+ twig:log(notice,"Retrying GET to ~s in ~p seconds due to error ~s",
+ [Url, Wait / 1000, Else]
+ ),
+ ok = timer:sleep(Wait),
+ RetryDb = HttpDb#httpdb{
+ retries = Retries - 1,
+ wait = Wait
+ },
+ open_doc_revs(RetryDb, Id, Revs, Options, Fun, Acc)
end;
open_doc_revs(Db, Id, Revs, Options, Fun, Acc) ->
{ok, Results} = couch_db:open_doc_revs(Db, Id, Revs, Options),
[16/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
don't crash if ibrowse is already started
fix #COUCHDB-1986
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/b58ffb9b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/b58ffb9b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/b58ffb9b
Branch: refs/heads/import-rcouch
Commit: b58ffb9b7ad263a185377feb25ff30ad9bab8f62
Parents: 4485a89
Author: benoitc <bc...@gmail.com>
Authored: Sat Dec 14 17:21:26 2013 +0100
Committer: benoitc <bc...@gmail.com>
Committed: Sat Dec 14 17:21:26 2013 +0100
----------------------------------------------------------------------
test/04-replication-large-atts.t | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/b58ffb9b/test/04-replication-large-atts.t
----------------------------------------------------------------------
diff --git a/test/04-replication-large-atts.t b/test/04-replication-large-atts.t
index 5386179..a7063c7 100755
--- a/test/04-replication-large-atts.t
+++ b/test/04-replication-large-atts.t
@@ -64,8 +64,8 @@ main(_) ->
test() ->
couch_server_sup:start_link(test_util:config_files()),
- ibrowse:start(),
- crypto:start(),
+ application:start(ibrowse),
+ application:start(crypto),
couch_config:set("attachments", "compressible_types", "text/*", false),
Pairs = [
[18/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
make the couch_replicator a full Erlang application
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/9953fe54
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/9953fe54
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/9953fe54
Branch: refs/heads/import-rcouch
Commit: 9953fe549003c48ceec241e8ac83a055a641f425
Parents: abaeaab
Author: benoitc <be...@apache.org>
Authored: Tue Jan 7 00:15:26 2014 +0100
Committer: Paul J. Davis <pa...@gmail.com>
Committed: Thu Feb 6 11:17:53 2014 -0600
----------------------------------------------------------------------
src/couch_replicator.app.src | 23 +++++-------
src/couch_replicator_app.erl | 29 +++++++++++++++
src/couch_replicator_manager_sup.erl | 47 ++++++++++++++++++++++++
src/couch_replicator_sup.erl | 59 +++++++++++++++++++++++++++++++
4 files changed, 144 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9953fe54/src/couch_replicator.app.src
----------------------------------------------------------------------
diff --git a/src/couch_replicator.app.src b/src/couch_replicator.app.src
index 750eaad..1e62edf 100644
--- a/src/couch_replicator.app.src
+++ b/src/couch_replicator.app.src
@@ -13,21 +13,16 @@
{application, couch_replicator, [
{description, "CouchDB replicator"},
{vsn, "%version%"},
- {modules, [
- couch_replicator_api_wrap,
- couch_replicator_httpc,
- couch_replicator_httpd,
- couch_replicator_job_sup,
- couch_replicator_notifier,
- couch_replicator_manager,
- couch_replicator_httpc_pool,
- couch_replicator_utils,
- couch_replicator_worker,
- couch_replicator
- ]},
+ {modules, []},
{registered, [
- couch_replicator_job_sup
+ couch_replicator_manager_sup,
+ couch_replicator_job_sup,
+ couch_replicator_sup
]},
- {applications, [kernel, stdlib]}
+ {applications, [kernel, stdlib, crypto, sasl, inets, oauth, ibrowse,
+ couch]},
+
+ {mod, { couch_replicator_app, []}},
+ {env, []}
]}.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9953fe54/src/couch_replicator_app.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_app.erl b/src/couch_replicator_app.erl
new file mode 100644
index 0000000..4083db6
--- /dev/null
+++ b/src/couch_replicator_app.erl
@@ -0,0 +1,29 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_app).
+
+-behaviour(application).
+
+%% Application callbacks
+-export([start/2, stop/1]).
+
+%% ===================================================================
+%% Application callbacks
+%% ===================================================================
+
+start(_StartType, _StartArgs) ->
+ couch_util:start_app_deps(couch_replicator),
+ couch_replicator_sup:start_link().
+
+stop(_State) ->
+ ok.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9953fe54/src/couch_replicator_manager_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager_sup.erl b/src/couch_replicator_manager_sup.erl
new file mode 100644
index 0000000..f292546
--- /dev/null
+++ b/src/couch_replicator_manager_sup.erl
@@ -0,0 +1,47 @@
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_manager_sup).
+
+-behaviour(supervisor).
+
+%% API
+-export([start_link/0]).
+
+%% Supervisor callbacks
+-export([init/1]).
+
+%% Helper macro for declaring children of supervisor
+-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
+
+%% ===================================================================
+%% API functions
+%% ===================================================================
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%% ===================================================================
+%% Supervisor callbacks
+%% ===================================================================
+
+init([]) ->
+ Children = [
+ {couch_replicator_manager,
+ {couch_replicator_manager, start_link, []},
+ permanent,
+ brutal_kill,
+ worker,
+ [couch_replicator_manager]}
+ ],
+ {ok, { {one_for_one, 10, 3600}, Children} }.
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/9953fe54/src/couch_replicator_sup.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_sup.erl b/src/couch_replicator_sup.erl
new file mode 100644
index 0000000..905e2cf
--- /dev/null
+++ b/src/couch_replicator_sup.erl
@@ -0,0 +1,59 @@
+
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(couch_replicator_sup).
+
+-behaviour(supervisor).
+
+%% API
+-export([start_link/0]).
+
+%% Supervisor callbacks
+-export([init/1]).
+
+%% Helper macro for declaring children of supervisor
+-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
+
+%% ===================================================================
+%% API functions
+%% ===================================================================
+
+start_link() ->
+ supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%% ===================================================================
+%% Supervisor callbacks
+%% ===================================================================
+
+init([]) ->
+ Children = [
+ {couch_replication_event,
+ {gen_event, start_link, [{local, couch_replication}]},
+ permanent,
+ brutal_kill,
+ worker,
+ dynamic},
+ {couch_replicator_job_sup,
+ {couch_replicator_job_sup, start_link, []},
+ permanent,
+ infinity,
+ supervisor,
+ [couch_replicator_job_sup]},
+ {couch_replicator_manager_sup,
+ {couch_replicator_manager_sup, start_link, []},
+ permanent,
+ infinity,
+ supervisor,
+ [couch_replicator_manager_sup]}
+ ],
+ {ok, { {one_for_one, 10, 3600}, Children} }.
[11/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Bubble up missing_stub exceptions for proper retry
A missing_stub exception is special in that the replicator retries the
request without the atts_since optimmization.
COUCHDB-1901
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/eb06a31a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/eb06a31a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/eb06a31a
Branch: refs/heads/import-rcouch
Commit: eb06a31ab0731ebe19e329923b9fde668b8543dd
Parents: eee70d8
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Oct 2 17:00:16 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 17:00:16 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/eb06a31a/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index eb7d845..52e15b7 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -215,6 +215,8 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
receive
{'DOWN', Ref, process, Pid, {exit_ok, Ret}} ->
Ret;
+ {'DOWN', Ref, process, Pid, {{nocatch, {missing_stub,_} = Stub}, _}} ->
+ throw(Stub);
{'DOWN', Ref, process, Pid, Else} ->
Url = couch_util:url_strip_password(
couch_replicator_httpc:full_url(HttpDb, [{path,Path}, {qs,QS}])
[14/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Include reason for replication failure in _replicator doc
closes COUCHDB-1647
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/2b666093
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/2b666093
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/2b666093
Branch: refs/heads/import-rcouch
Commit: 2b666093dc7292c8a437a5f5f8151a06261b15c8
Parents: ee27edf
Author: Robert Newson <rn...@apache.org>
Authored: Tue Dec 3 11:15:59 2013 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Tue Dec 3 13:10:49 2013 +0000
----------------------------------------------------------------------
src/couch_replicator_manager.erl | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/2b666093/src/couch_replicator_manager.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_manager.erl b/src/couch_replicator_manager.erl
index bbb0e11..4891c4c 100644
--- a/src/couch_replicator_manager.erl
+++ b/src/couch_replicator_manager.erl
@@ -68,6 +68,7 @@ replication_started(#rep{id = {BaseId, _} = RepId}) ->
#rep_state{rep = #rep{doc_id = DocId}} ->
update_rep_doc(DocId, [
{<<"_replication_state">>, <<"triggered">>},
+ {<<"_replication_state_reason">>, undefined},
{<<"_replication_id">>, ?l2b(BaseId)},
{<<"_replication_stats">>, undefined}]),
ok = gen_server:call(?MODULE, {rep_started, RepId}, infinity),
@@ -83,6 +84,7 @@ replication_completed(#rep{id = RepId}, Stats) ->
#rep_state{rep = #rep{doc_id = DocId}} ->
update_rep_doc(DocId, [
{<<"_replication_state">>, <<"completed">>},
+ {<<"_replication_state_reason">>, undefined},
{<<"_replication_stats">>, {Stats}}]),
ok = gen_server:call(?MODULE, {rep_complete, RepId}, infinity),
?LOG_INFO("Replication `~s` finished (triggered by document `~s`)",
@@ -95,9 +97,9 @@ replication_error(#rep{id = {BaseId, _} = RepId}, Error) ->
nil ->
ok;
#rep_state{rep = #rep{doc_id = DocId}} ->
- % TODO: maybe add error reason to replication document
update_rep_doc(DocId, [
{<<"_replication_state">>, <<"error">>},
+ {<<"_replication_state_reason">>, to_binary(error_reason(Error))},
{<<"_replication_id">>, ?l2b(BaseId)}]),
ok = gen_server:call(?MODULE, {rep_error, RepId, Error}, infinity)
end.
@@ -344,7 +346,8 @@ rep_db_update_error(Error, DocId) ->
end,
?LOG_ERROR("Replication manager, error processing document `~s`: ~s",
[DocId, Reason]),
- update_rep_doc(DocId, [{<<"_replication_state">>, <<"error">>}]).
+ update_rep_doc(DocId, [{<<"_replication_state">>, <<"error">>},
+ {<<"_replication_state_reason">>, Reason}]).
rep_user_ctx({RepDoc}) ->
@@ -619,6 +622,9 @@ rep_state(RepId) ->
end.
+error_reason({error, {Error, Reason}})
+ when is_atom(Error), is_binary(Reason) ->
+ io_lib:format("~s: ~s", [Error, Reason]);
error_reason({error, Reason}) ->
Reason;
error_reason(Reason) ->
[05/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Fix replication deadlock after HTTP retries
An HTTP failure in the open_doc_revs can lead to a replication deadlock
if the retry happens while parsing the multipart/mime response. The
UserFun callback can end up attempting to PUT a request using an
attachment reader from the failed request while the pid that invoked
open_doc_revs will never see the started_open_doc_revs message which
indicates a retry has happened.
This just spawns a middleman process so that the process that invoked
open_doc_revs can listen for the response from the user function as well
as the started_open_doc_revs message to handle retries appropriately.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/19106225
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/19106225
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/19106225
Branch: refs/heads/import-rcouch
Commit: 19106225ced08ad5970ff626f0969e1cec6279bf
Parents: a28ef18
Author: Paul J. Davis <pa...@gmail.com>
Authored: Tue Apr 2 18:05:30 2013 -0500
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 11:59:31 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 36 +++++++++++++++++++++++++++++++---
1 file changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/19106225/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 4aabe15..9921560 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -535,7 +535,7 @@ receive_docs(Streamer, UserFun, Ref, UserAcc) ->
fun() -> receive_doc_data(Streamer, Ref) end,
Ref) of
{ok, Doc, Parser} ->
- case UserFun({ok, Doc}, UserAcc) of
+ case run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref) of
{ok, UserAcc2} ->
ok;
{skip, UserAcc2} ->
@@ -546,13 +546,13 @@ receive_docs(Streamer, UserFun, Ref, UserAcc) ->
{"application/json", []} ->
Doc = couch_doc:from_json_obj(
?JSON_DECODE(receive_all(Streamer, Ref, []))),
- {_, UserAcc2} = UserFun({ok, Doc}, UserAcc),
+ {_, UserAcc2} = run_user_fun(UserFun, {ok, Doc}, UserAcc, Ref),
receive_docs(Streamer, UserFun, Ref, UserAcc2);
{"application/json", [{"error","true"}]} ->
{ErrorProps} = ?JSON_DECODE(receive_all(Streamer, Ref, [])),
Rev = get_value(<<"missing">>, ErrorProps),
Result = {{not_found, missing}, couch_doc:parse_rev(Rev)},
- {_, UserAcc2} = UserFun(Result, UserAcc),
+ {_, UserAcc2} = run_user_fun(UserFun, Result, UserAcc, Ref),
receive_docs(Streamer, UserFun, Ref, UserAcc2)
end;
{done, Ref} ->
@@ -560,6 +560,36 @@ receive_docs(Streamer, UserFun, Ref, UserAcc) ->
end.
+run_user_fun(UserFun, Arg, UserAcc, OldRef) ->
+ {Pid, Ref} = spawn_monitor(fun() ->
+ try UserFun(Arg, UserAcc) of
+ Resp ->
+ exit({exit_ok, Resp})
+ catch
+ throw:Reason ->
+ exit({exit_throw, Reason});
+ error:Reason ->
+ exit({exit_error, Reason});
+ exit:Reason ->
+ exit({exit_exit, Reason})
+ end
+ end),
+ receive
+ {started_open_doc_revs, NewRef} ->
+ erlang:demonitor(Ref, [flush]),
+ exit(Pid, kill),
+ restart_remote_open_doc_revs(OldRef, NewRef);
+ {'DOWN', Ref, process, Pid, {exit_ok, Ret}} ->
+ Ret;
+ {'DOWN', Ref, process, Pid, {exit_throw, Reason}} ->
+ throw(Reason);
+ {'DOWN', Ref, process, Pid, {exit_error, Reason}} ->
+ erlang:error(Reason);
+ {'DOWN', Ref, process, Pid, {exit_exit, Reason}} ->
+ erlang:exit(Reason)
+ end.
+
+
restart_remote_open_doc_revs(Ref, NewRef) ->
receive
{body_bytes, Ref, _} ->
[04/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Avoid deadlocking the httpc pool
The multipart mime parsing code was getting into a state where it would
orphan parsers that held onto pool connections. This just adds a monitor
to make sure that the parser process dies when the parent dies normally
without finishing reading the rest of the parser.
BugzID: 16751
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a28ef188
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a28ef188
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a28ef188
Branch: refs/heads/import-rcouch
Commit: a28ef188ed8a8c3556d245c9b71675f333018995
Parents: 27dbba3
Author: Paul J. Davis <pa...@gmail.com>
Authored: Fri Feb 22 13:44:20 2013 -0600
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 11:59:20 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a28ef188/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index cd69e59..4aabe15 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -177,6 +177,20 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
end),
unlink(Self)
end),
+ % If this process dies normally we can leave
+ % the Streamer process hanging around keeping an
+ % HTTP connection open. This is a bit of a
+ % hammer approach to making sure it releases
+ % that connection back to the pool.
+ spawn(fun() ->
+ Ref = erlang:monitor(process, Self),
+ receive
+ {'DOWN', Ref, process, Self, normal} ->
+ exit(Streamer, {streamer_parent_died, Self});
+ {'DOWN', Ref, process, Self, _} ->
+ ok
+ end
+ end),
receive
{started_open_doc_revs, Ref} ->
receive_docs_loop(Streamer, Fun, Id, Revs, Ref, Acc)
[12/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Allow the use of checkpoint to be optional
A new parameter use_checkpoints which defaults to be true can be either used in the replicator section of
ini files and/or used in the replication command or doc added to the replicator db. When this option is set
to false no checkpoints are taken. It should be used with caution and only for small dbs.
BugzID:14327
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/a6d8389c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/a6d8389c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/a6d8389c
Branch: refs/heads/import-rcouch
Commit: a6d8389c67c5b1452e342b2f865beb13e813a2d3
Parents: eb06a31
Author: Bob Dionne <bo...@cloudant.com>
Authored: Sun Aug 19 11:38:28 2012 -0400
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Nov 20 14:56:47 2013 +0000
----------------------------------------------------------------------
src/couch_replicator.erl | 11 +++++++++--
src/couch_replicator_utils.erl | 6 +++++-
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a6d8389c/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator.erl b/src/couch_replicator.erl
index 1ce2cf5..b00efec 100644
--- a/src/couch_replicator.erl
+++ b/src/couch_replicator.erl
@@ -68,7 +68,8 @@
target_db_compaction_notifier = nil,
source_monitor = nil,
target_monitor = nil,
- source_seq = nil
+ source_seq = nil,
+ use_checkpoints = true
}).
@@ -455,6 +456,8 @@ handle_cast({report_seq, Seq},
{noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}.
+code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 30 ->
+ {ok, erlang:append_element(OldState, true)};
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@@ -563,7 +566,8 @@ init_state(Rep) ->
start_db_compaction_notifier(Target, self()),
source_monitor = db_monitor(Source),
target_monitor = db_monitor(Target),
- source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ)
+ source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ),
+ use_checkpoints = get_value(use_checkpoints, Options, true)
},
State#rep_state{timer = start_timer(State)}.
@@ -672,6 +676,9 @@ changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) ->
checkpoint_interval(_State) ->
5000.
+do_checkpoint(#rep_state{use_checkpoints=false} = State) ->
+ NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} },
+ {ok, NewState};
do_checkpoint(#rep_state{current_through_seq=Seq, committed_seq=Seq} = State) ->
SourceCurSeq = source_cur_seq(State),
NewState = State#rep_state{source_seq = SourceCurSeq},
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/a6d8389c/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_utils.erl b/src/couch_replicator_utils.erl
index d7778db..1646c69 100644
--- a/src/couch_replicator_utils.erl
+++ b/src/couch_replicator_utils.erl
@@ -228,6 +228,7 @@ make_options(Props) ->
DefConns = couch_config:get("replicator", "http_connections", "20"),
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
+ UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
@@ -237,7 +238,8 @@ make_options(Props) ->
{http_connections, list_to_integer(DefConns)},
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
- {worker_processes, list_to_integer(DefWorkers)}
+ {worker_processes, list_to_integer(DefWorkers)},
+ {use_checkpoints, list_to_existing_atom(UseCheckpoints)}
])).
@@ -279,6 +281,8 @@ convert_options([{<<"socket_options">>, V} | R]) ->
[{socket_options, SocketOptions} | convert_options(R)];
convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
+convert_options([{<<"use_checkpoints">>, V} | R]) ->
+ [{use_checkpoints, V} | convert_options(R)];
convert_options([_ | R]) -> % skip unknown option
convert_options(R).
[07/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Ensure the att reader learns of a Parser death
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/81d6526c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/81d6526c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/81d6526c
Branch: refs/heads/import-rcouch
Commit: 81d6526c2e9ad7c75cbf95737fcda80313b02e24
Parents: 61b0f14
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Thu Sep 12 13:30:14 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 12:11:24 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/81d6526c/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index 28f96c0..d8be12a 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -713,6 +713,7 @@ doc_from_multi_part_stream(ContentType, DataFun, Ref) ->
{doc_bytes, Ref, DocBytes} ->
Doc = couch_doc:from_json_obj(?JSON_DECODE(DocBytes)),
ReadAttachmentDataFun = fun() ->
+ link(Parser),
Parser ! {get_bytes, Ref, self()},
receive
{started_open_doc_revs, NewRef} ->
[06/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Disable automatic retries of streaming writes
An automatic retry makes no sense here, as we've already streamed part
of the response body from the GET. Disabling the retries bubbles the
error up to run_user_fun inside open_doc_revs allows us to retry the
whole GET + PUT correctly.
BugzID: 20822
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/1c501b87
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/1c501b87
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/1c501b87
Branch: refs/heads/import-rcouch
Commit: 1c501b874ced48b36442cec6c9c0bf988e1f91d9
Parents: 81d6526
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Thu Sep 12 17:13:07 2013 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Oct 2 12:11:24 2013 -0400
----------------------------------------------------------------------
src/couch_replicator_api_wrap.erl | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/1c501b87/src/couch_replicator_api_wrap.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator_api_wrap.erl b/src/couch_replicator_api_wrap.erl
index d8be12a..fccb759 100644
--- a/src/couch_replicator_api_wrap.erl
+++ b/src/couch_replicator_api_wrap.erl
@@ -221,7 +221,7 @@ open_doc_revs(#httpdb{} = HttpDb, Id, Revs, Options, Fun, Acc) ->
),
#httpdb{retries = Retries, wait = Wait0} = HttpDb,
Wait = 2 * erlang:min(Wait0 * 2, ?MAX_WAIT),
- twig:log(notice,"Retrying GET to ~s in ~p seconds due to error ~s",
+ twig:log(notice,"Retrying GET to ~s in ~p seconds due to error ~p",
[Url, Wait / 1000, Else]
),
ok = timer:sleep(Wait),
@@ -278,7 +278,11 @@ update_doc(#httpdb{} = HttpDb, #doc{id = DocId} = Doc, Options, Type) ->
end ++ [{"Content-Type", ?b2l(ContentType)}, {"Content-Length", Len}],
Body = {fun stream_doc/1, {JsonBytes, Doc#doc.atts, Boundary, Len}},
send_req(
- HttpDb,
+ % A crash here bubbles all the way back up to run_user_fun inside
+ % open_doc_revs, which will retry the whole thing. That's the
+ % appropriate course of action, since we've already started streaming
+ % the response body from the GET request.
+ HttpDb#httpdb{retries = 0},
[{method, put}, {path, encode_doc_id(DocId)},
{qs, QArgs}, {headers, Headers}, {body, Body}],
fun(Code, _, {Props}) when Code =:= 200 orelse Code =:= 201 ->
[15/26] couch-replicator commit: updated refs/heads/import-rcouch to
589d958
Posted by da...@apache.org.
Tests for replicator use_checkpoints option
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/commit/4485a89c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/tree/4485a89c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/diff/4485a89c
Branch: refs/heads/import-rcouch
Commit: 4485a89cad80c2bdae649417c3cb68526258da25
Parents: 2b66609
Author: Volker Mische <vo...@gmail.com>
Authored: Tue Jul 16 00:26:03 2013 +0200
Committer: Robert Newson <rn...@apache.org>
Committed: Sat Dec 14 11:31:32 2013 +0100
----------------------------------------------------------------------
Makefile.am | 3 +-
test/07-use-checkpoints.t | 256 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 258 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4485a89c/Makefile.am
----------------------------------------------------------------------
diff --git a/Makefile.am b/Makefile.am
index 71d3702..2dcd47d 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -41,7 +41,8 @@ test_files = \
test/03-replication-compact.t \
test/04-replication-large-atts.t \
test/05-replication-many-leaves.t \
- test/06-doc-missing-stubs.t
+ test/06-doc-missing-stubs.t \
+ test/07-use-checkpoints.t
compiled_files = \
ebin/couch_replicator_api_wrap.beam \
http://git-wip-us.apache.org/repos/asf/couchdb-couch-replicator/blob/4485a89c/test/07-use-checkpoints.t
----------------------------------------------------------------------
diff --git a/test/07-use-checkpoints.t b/test/07-use-checkpoints.t
new file mode 100755
index 0000000..cefc1a7
--- /dev/null
+++ b/test/07-use-checkpoints.t
@@ -0,0 +1,256 @@
+#!/usr/bin/env escript
+%% -*- erlang -*-
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+% Verify that compacting databases that are being used as the source or
+% target of a replication doesn't affect the replication and that the
+% replication doesn't hold their reference counters forever.
+
+-define(b2l(B), binary_to_list(B)).
+
+-record(user_ctx, {
+ name = null,
+ roles = [],
+ handler
+}).
+
+-record(doc, {
+ id = <<"">>,
+ revs = {0, []},
+ body = {[]},
+ atts = [],
+ deleted = false,
+ meta = []
+}).
+
+-record(db, {
+ main_pid = nil,
+ update_pid = nil,
+ compactor_pid = nil,
+ instance_start_time, % number of microsecs since jan 1 1970 as a binary string
+ fd,
+ updater_fd,
+ fd_ref_counter,
+ header = nil,
+ committed_update_seq,
+ fulldocinfo_by_id_btree,
+ docinfo_by_seq_btree,
+ local_docs_btree,
+ update_seq,
+ name,
+ filepath,
+ validate_doc_funs = [],
+ security = [],
+ security_ptr = nil,
+ user_ctx = #user_ctx{},
+ waiting_delayed_commit = nil,
+ revs_limit = 1000,
+ fsync_options = [],
+ options = [],
+ compression,
+ before_doc_update,
+ after_doc_read
+}).
+
+-record(rep, {
+ id,
+ source,
+ target,
+ options,
+ user_ctx,
+ doc_id
+}).
+
+
+source_db_name() -> <<"couch_test_rep_db_a">>.
+target_db_name() -> <<"couch_test_rep_db_b">>.
+
+
+main(_) ->
+ test_util:init_code_path(),
+
+ etap:plan(16),
+ case (catch test()) of
+ ok ->
+ etap:end_tests();
+ Other ->
+ etap:diag(io_lib:format("Test died abnormally: ~p", [Other])),
+ etap:bail(Other)
+ end,
+ ok.
+
+
+test() ->
+ couch_server_sup:start_link(test_util:config_files()),
+ ibrowse:start(),
+
+ test_use_checkpoints(false),
+ test_use_checkpoints(true),
+
+ couch_server_sup:stop(),
+ ok.
+
+
+test_use_checkpoints(UseCheckpoints) ->
+ Pairs = [
+ {source_db_name(), target_db_name()},
+ {{remote, source_db_name()}, target_db_name()},
+ {source_db_name(), {remote, target_db_name()}},
+ {{remote, source_db_name()}, {remote, (target_db_name())}}
+ ],
+
+ ListenerFun = case UseCheckpoints of
+ false ->
+ fun({finished, _, {CheckpointHistory}}) ->
+ etap:is(CheckpointHistory,
+ [{<<"use_checkpoints">>,false}],
+ "No checkpoints found");
+ (_) ->
+ ok
+ end;
+ true ->
+ fun({finished, _, {CheckpointHistory}}) ->
+ SessionId = lists:keyfind(
+ <<"session_id">>, 1, CheckpointHistory),
+ etap:isnt(SessionId, false, "There's a checkpoint");
+ (_) ->
+ ok
+ end
+ end,
+ {ok, Listener} = couch_replicator_notifier:start_link(ListenerFun),
+
+ lists:foreach(
+ fun({Source, Target}) ->
+ {ok, SourceDb} = create_db(source_db_name()),
+ etap:diag("Populating source database"),
+ populate_db(SourceDb, 100),
+ ok = couch_db:close(SourceDb),
+
+ etap:diag("Creating target database"),
+ {ok, TargetDb} = create_db(target_db_name()),
+ ok = couch_db:close(TargetDb),
+
+ etap:diag("Setup replicator notifier listener"),
+
+ etap:diag("Triggering replication"),
+ replicate(Source, Target, UseCheckpoints),
+
+ etap:diag("Replication finished, comparing source and target databases"),
+ compare_dbs(SourceDb, TargetDb),
+
+ etap:diag("Deleting databases"),
+ delete_db(TargetDb),
+ delete_db(SourceDb),
+
+ ok = timer:sleep(1000)
+ end,
+ Pairs),
+
+ couch_replicator_notifier:stop(Listener).
+
+
+populate_db(Db, DocCount) ->
+ Docs = lists:foldl(
+ fun(DocIdCounter, Acc) ->
+ Id = iolist_to_binary(["doc", integer_to_list(DocIdCounter)]),
+ Value = iolist_to_binary(["val", integer_to_list(DocIdCounter)]),
+ Doc = #doc{
+ id = Id,
+ body = {[ {<<"value">>, Value} ]}
+ },
+ [Doc | Acc]
+ end,
+ [], lists:seq(1, DocCount)),
+ {ok, _} = couch_db:update_docs(Db, Docs, []).
+
+
+compare_dbs(#db{name = SourceName}, #db{name = TargetName}) ->
+ {ok, SourceDb} = couch_db:open_int(SourceName, []),
+ {ok, TargetDb} = couch_db:open_int(TargetName, []),
+ Fun = fun(FullDocInfo, _, Acc) ->
+ {ok, Doc} = couch_db:open_doc(SourceDb, FullDocInfo),
+ {Props} = DocJson = couch_doc:to_json_obj(Doc, [attachments]),
+ DocId = couch_util:get_value(<<"_id">>, Props),
+ DocTarget = case couch_db:open_doc(TargetDb, DocId) of
+ {ok, DocT} ->
+ DocT;
+ Error ->
+ etap:bail("Error opening document '" ++ ?b2l(DocId) ++
+ "' from target: " ++ couch_util:to_list(Error))
+ end,
+ DocTargetJson = couch_doc:to_json_obj(DocTarget, [attachments]),
+ case DocTargetJson of
+ DocJson ->
+ ok;
+ _ ->
+ etap:bail("Content from document '" ++ ?b2l(DocId) ++
+ "' differs in target database")
+ end,
+ {ok, Acc}
+ end,
+ {ok, _, _} = couch_db:enum_docs(SourceDb, Fun, [], []),
+ etap:diag("Target database has the same documents as the source database"),
+ ok = couch_db:close(SourceDb),
+ ok = couch_db:close(TargetDb).
+
+
+db_url(DbName) ->
+ iolist_to_binary([
+ "http://", couch_config:get("httpd", "bind_address", "127.0.0.1"),
+ ":", integer_to_list(mochiweb_socket_server:get(couch_httpd, port)),
+ "/", DbName
+ ]).
+
+
+create_db(DbName) ->
+ {ok, Db} = couch_db:create(
+ DbName,
+ [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}, overwrite]),
+ couch_db:close(Db),
+ {ok, Db}.
+
+
+delete_db(#db{name = DbName, main_pid = Pid}) ->
+ ok = couch_server:delete(
+ DbName, [{user_ctx, #user_ctx{roles = [<<"_admin">>]}}]),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, _Reason} ->
+ ok
+ after 30000 ->
+ etap:bail("Timeout deleting database")
+ end.
+
+
+replicate({remote, Db}, Target, UseCheckpoints) ->
+ replicate(db_url(Db), Target, UseCheckpoints);
+
+replicate(Source, {remote, Db}, UseCheckpoints) ->
+ replicate(Source, db_url(Db), UseCheckpoints);
+
+replicate(Source, Target, UseCheckpoints) ->
+ RepObject = {[
+ {<<"source">>, Source},
+ {<<"target">>, Target},
+ {<<"use_checkpoints">>, UseCheckpoints}
+ ]},
+ {ok, Rep} = couch_replicator_utils:parse_rep_doc(
+ RepObject, #user_ctx{roles = [<<"_admin">>]}),
+ {ok, Pid} = couch_replicator:async_replicate(Rep),
+ MonRef = erlang:monitor(process, Pid),
+ receive
+ {'DOWN', MonRef, process, Pid, Reason} ->
+ etap:is(Reason, normal, "Replication finished successfully")
+ after 300000 ->
+ etap:bail("Timeout waiting for replication to finish")
+ end.