svn commit: r1004758 - in /couchdb/branches/new_replicator: etc/couchdb/default.ini.tpl.in src/couchdb/couch_replicator.erl src/couchdb/couch_replicator_doc_copiers.erl src/couchdb/couch_replicator_rev_finders.erl

[fd...@apache.org]

Author: fdmanana
Date: Tue Oct  5 18:25:07 2010
New Revision: 1004758

URL: http://svn.apache.org/viewvc?rev=1004758&view=rev
Log:
New replicator: add better default values and .ini option "buffer_size".

Modified:
    couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in
    couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
    couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
    couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl

Modified: couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in?rev=1004758&r1=1004757&r2=1004758&view=diff
==============================================================================
--- couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/branches/new_replicator/etc/couchdb/default.ini.tpl.in Tue Oct  5 18:25:07 2010
@@ -118,3 +118,5 @@ compressible_types = text/*, application
 [replicator]
 ; should be at least 2
 worker_processes = 10
+; integer value in bytes
+buffer_size = 1048576
\ No newline at end of file

Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl?rev=1004758&r1=1004757&r2=1004758&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicator.erl Tue Oct  5 18:25:07 2010
@@ -217,8 +217,10 @@ do_init(#rep{options = Options} = Rep) -
         start_seq = StartSeq
     } = State = init_state(Rep),
 
+    QueueSize = round(
+        (?l2i(couch_config:get("replicator", "buffer_size", "1048576"))) / 2),
     {ok, MissingRevsQueue} = couch_work_queue:new(
-        [{max_size, 100000}, {max_items, 2000}, {multi_workers, true}]),
+        [{max_size, QueueSize}, {multi_workers, true}]),
 
     {RevFindersCount, CopiersCount} = case ?l2i(
         couch_config:get("replicator", "worker_processes", "10")) of
@@ -227,13 +229,13 @@ do_init(#rep{options = Options} = Rep) -
             "should be at least 2", []),
         {1, 1};
     N ->
-        {N div 2, (N div 2) + (N rem 2)}
+        {lists:max([N div 3, 1]), lists:max([2 * (N div 3) + (N rem 3), 1])}
     end,
 
     case get_value(doc_ids, Options) of
     undefined ->
         {ok, ChangesQueue} = couch_work_queue:new(
-            [{max_size, 100000}, {max_items, 500}, {multi_workers, true}]),
+            [{max_size, QueueSize}, {multi_workers, true}]),
 
         % This starts the _changes reader process. It adds the changes from
         % the source db to the ChangesQueue.

Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl?rev=1004758&r1=1004757&r2=1004758&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicator_doc_copiers.erl Tue Oct  5 18:25:07 2010
@@ -17,7 +17,7 @@
 -include("couch_db.hrl").
 -include("couch_api_wrap.hrl").
 
--define(DOC_BATCH_SIZE, 50).
+-define(DOC_BATCH_SIZE, 1000).
 
 
 

Modified: couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl
URL: http://svn.apache.org/viewvc/couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl?rev=1004758&r1=1004757&r2=1004758&view=diff
==============================================================================
--- couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl (original)
+++ couchdb/branches/new_replicator/src/couchdb/couch_replicator_rev_finders.erl Tue Oct  5 18:25:07 2010
@@ -16,7 +16,7 @@
 
 -include("couch_db.hrl").
 
--define(REV_BATCH_SIZE, 100).
+-define(REV_BATCH_SIZE, 3000).