You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2013/11/20 16:25:18 UTC
[1/2] git commit: updated refs/heads/master to 0693f98
Updated Branches:
refs/heads/master 6cd153de4 -> 0693f98e2
Allow the use of checkpoint to be optional
A new parameter use_checkpoints which defaults to be true can be either used in the replicator section of
ini files and/or used in the replication command or doc added to the replicator db. When this option is set
to false no checkpoints are taken. It should be used with caution and only for small dbs.
BugzID:14327
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/1d5fe2aa
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/1d5fe2aa
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/1d5fe2aa
Branch: refs/heads/master
Commit: 1d5fe2aa22cf5c947224d587108ec8bd47270e07
Parents: 6cd153d
Author: Bob Dionne <bo...@cloudant.com>
Authored: Sun Aug 19 11:38:28 2012 -0400
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Nov 20 14:56:47 2013 +0000
----------------------------------------------------------------------
src/couch_replicator/src/couch_replicator.erl | 11 +++++++++--
src/couch_replicator/src/couch_replicator_utils.erl | 6 +++++-
2 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/1d5fe2aa/src/couch_replicator/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/src/couch_replicator.erl b/src/couch_replicator/src/couch_replicator.erl
index 1ce2cf5..b00efec 100644
--- a/src/couch_replicator/src/couch_replicator.erl
+++ b/src/couch_replicator/src/couch_replicator.erl
@@ -68,7 +68,8 @@
target_db_compaction_notifier = nil,
source_monitor = nil,
target_monitor = nil,
- source_seq = nil
+ source_seq = nil,
+ use_checkpoints = true
}).
@@ -455,6 +456,8 @@ handle_cast({report_seq, Seq},
{noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}.
+code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 30 ->
+ {ok, erlang:append_element(OldState, true)};
code_change(_OldVsn, State, _Extra) ->
{ok, State}.
@@ -563,7 +566,8 @@ init_state(Rep) ->
start_db_compaction_notifier(Target, self()),
source_monitor = db_monitor(Source),
target_monitor = db_monitor(Target),
- source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ)
+ source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ),
+ use_checkpoints = get_value(use_checkpoints, Options, true)
},
State#rep_state{timer = start_timer(State)}.
@@ -672,6 +676,9 @@ changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) ->
checkpoint_interval(_State) ->
5000.
+do_checkpoint(#rep_state{use_checkpoints=false} = State) ->
+ NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} },
+ {ok, NewState};
do_checkpoint(#rep_state{current_through_seq=Seq, committed_seq=Seq} = State) ->
SourceCurSeq = source_cur_seq(State),
NewState = State#rep_state{source_seq = SourceCurSeq},
http://git-wip-us.apache.org/repos/asf/couchdb/blob/1d5fe2aa/src/couch_replicator/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl
index d7778db..1646c69 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -228,6 +228,7 @@ make_options(Props) ->
DefConns = couch_config:get("replicator", "http_connections", "20"),
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
+ UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
@@ -237,7 +238,8 @@ make_options(Props) ->
{http_connections, list_to_integer(DefConns)},
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
- {worker_processes, list_to_integer(DefWorkers)}
+ {worker_processes, list_to_integer(DefWorkers)},
+ {use_checkpoints, list_to_existing_atom(UseCheckpoints)}
])).
@@ -279,6 +281,8 @@ convert_options([{<<"socket_options">>, V} | R]) ->
[{socket_options, SocketOptions} | convert_options(R)];
convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
+convert_options([{<<"use_checkpoints">>, V} | R]) ->
+ [{use_checkpoints, V} | convert_options(R)];
convert_options([_ | R]) -> % skip unknown option
convert_options(R).
[2/2] git commit: updated refs/heads/master to 0693f98
Posted by rn...@apache.org.
configurable checkpoint interval
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/0693f98e
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/0693f98e
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/0693f98e
Branch: refs/heads/master
Commit: 0693f98e2678017f382e1f047532469d2098175f
Parents: 1d5fe2a
Author: Robert Newson <rn...@apache.org>
Authored: Wed Nov 20 14:59:00 2013 +0000
Committer: Robert Newson <rn...@apache.org>
Committed: Wed Nov 20 14:59:00 2013 +0000
----------------------------------------------------------------------
src/couch_replicator/src/couch_replicator.erl | 25 +++++++++++---------
.../src/couch_replicator_utils.erl | 6 ++++-
2 files changed, 19 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/0693f98e/src/couch_replicator/src/couch_replicator.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/src/couch_replicator.erl b/src/couch_replicator/src/couch_replicator.erl
index b00efec..91039b6 100644
--- a/src/couch_replicator/src/couch_replicator.erl
+++ b/src/couch_replicator/src/couch_replicator.erl
@@ -69,7 +69,8 @@
source_monitor = nil,
target_monitor = nil,
source_seq = nil,
- use_checkpoints = true
+ use_checkpoints = true,
+ checkpoint_interval = 5000
}).
@@ -248,7 +249,8 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
target_name = TargetName,
start_seq = {_Ts, StartSeq},
source_seq = SourceCurSeq,
- committed_seq = {_, CommittedSeq}
+ committed_seq = {_, CommittedSeq},
+ checkpoint_interval = CheckpointInterval
} = State = init_state(Rep),
NumWorkers = get_value(worker_processes, Options),
@@ -289,7 +291,8 @@ do_init(#rep{options = Options, id = {BaseId, Ext}} = Rep) ->
{doc_write_failures, 0},
{source_seq, SourceCurSeq},
{checkpointed_source_seq, CommittedSeq},
- {progress, 0}
+ {progress, 0},
+ {checkpoint_interval, CheckpointInterval}
]),
couch_task_status:set_update_frequency(1000),
@@ -456,9 +459,11 @@ handle_cast({report_seq, Seq},
{noreply, State#rep_state{seqs_in_progress = NewSeqsInProgress}}.
-code_change(_OldVsn, OldState, _Extra) when tuple_size(OldState) =:= 30 ->
- {ok, erlang:append_element(OldState, true)};
-code_change(_OldVsn, State, _Extra) ->
+code_change(OldVsn, OldState, Extra) when tuple_size(OldState) =:= 30 ->
+ code_change(OldVsn, erlang:append_element(OldState, true), Extra);
+code_change(OldVsn, OldState, Extra) when tuple_size(OldState) =:= 31 ->
+ code_change(OldVsn, erlang:append_element(OldState, 5000), Extra);
+code_change(_OldVsn, #rep_state{}=State, _Extra) ->
{ok, State}.
@@ -508,7 +513,7 @@ do_last_checkpoint(#rep_state{seqs_in_progress = [],
start_timer(State) ->
- After = checkpoint_interval(State),
+ After = State#rep_state.checkpoint_interval,
case timer:apply_after(After, gen_server, cast, [self(), checkpoint]) of
{ok, Ref} ->
Ref;
@@ -567,7 +572,8 @@ init_state(Rep) ->
source_monitor = db_monitor(Source),
target_monitor = db_monitor(Target),
source_seq = get_value(<<"update_seq">>, SourceInfo, ?LOWEST_SEQ),
- use_checkpoints = get_value(use_checkpoints, Options, true)
+ use_checkpoints = get_value(use_checkpoints, Options, true),
+ checkpoint_interval = get_value(checkpoint_interval, Options, 5000)
},
State#rep_state{timer = start_timer(State)}.
@@ -673,9 +679,6 @@ changes_manager_loop_open(Parent, ChangesQueue, BatchSize, Ts) ->
end.
-checkpoint_interval(_State) ->
- 5000.
-
do_checkpoint(#rep_state{use_checkpoints=false} = State) ->
NewState = State#rep_state{checkpoint_history = {[{<<"use_checkpoints">>, false}]} },
{ok, NewState};
http://git-wip-us.apache.org/repos/asf/couchdb/blob/0693f98e/src/couch_replicator/src/couch_replicator_utils.erl
----------------------------------------------------------------------
diff --git a/src/couch_replicator/src/couch_replicator_utils.erl b/src/couch_replicator/src/couch_replicator_utils.erl
index 1646c69..240d7d4 100644
--- a/src/couch_replicator/src/couch_replicator_utils.erl
+++ b/src/couch_replicator/src/couch_replicator_utils.erl
@@ -229,6 +229,7 @@ make_options(Props) ->
DefTimeout = couch_config:get("replicator", "connection_timeout", "30000"),
DefRetries = couch_config:get("replicator", "retries_per_request", "10"),
UseCheckpoints = couch_config:get("replicator", "use_checkpoints", "true"),
+ DefCheckpointInterval = couch_config:get("replicator", "checkpoint_interval", "5000"),
{ok, DefSocketOptions} = couch_util:parse_term(
couch_config:get("replicator", "socket_options",
"[{keepalive, true}, {nodelay, false}]")),
@@ -239,7 +240,8 @@ make_options(Props) ->
{socket_options, DefSocketOptions},
{worker_batch_size, list_to_integer(DefBatchSize)},
{worker_processes, list_to_integer(DefWorkers)},
- {use_checkpoints, list_to_existing_atom(UseCheckpoints)}
+ {use_checkpoints, list_to_existing_atom(UseCheckpoints)},
+ {checkpoint_interval, list_to_integer(DefCheckpointInterval)}
])).
@@ -283,6 +285,8 @@ convert_options([{<<"since_seq">>, V} | R]) ->
[{since_seq, V} | convert_options(R)];
convert_options([{<<"use_checkpoints">>, V} | R]) ->
[{use_checkpoints, V} | convert_options(R)];
+convert_options([{<<"checkpoint_interval">>, V} | R]) ->
+ [{checkpoint_interval, couch_util:to_integer(V)} | convert_options(R)];
convert_options([_ | R]) -> % skip unknown option
convert_options(R).