[couchdb] branch persist-stats-across-job-removals updated: Properly account for replication stats when splitting bulk docs batches

[va...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

vatamane pushed a commit to branch persist-stats-across-job-removals
in repository https://gitbox.apache.org/repos/asf/couchdb.git


The following commit(s) were added to refs/heads/persist-stats-across-job-removals by this push:
     new f0f72f6  Properly account for replication stats when splitting bulk docs batches
f0f72f6 is described below

commit f0f72f6c7c9740e57a0d4b992297f0c0798695fd
Author: Nick Vatamaniuc <va...@apache.org>
AuthorDate: Mon Jan 13 18:39:31 2020 -0500

    Properly account for replication stats when splitting bulk docs batches
    
    Previously if batch of bulk docs had to be bisected in order to fit a lower max
    request size limit on the target, we only counted stats for the second batch.
    So it was possibly we might have missed some `doc_write_failures` updates which
    can be perceived as a data loss to the customer.
    
    So we use the handy-dandy `sum_stats/2` function to sum the return stats from
    both batches and return that.
    
    Issue: https://github.com/apache/couchdb/issues/2414
---
 src/couch_replicator/src/couch_replicator_worker.erl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/couch_replicator/src/couch_replicator_worker.erl b/src/couch_replicator/src/couch_replicator_worker.erl
index 3d80f58..f37af8f 100644
--- a/src/couch_replicator/src/couch_replicator_worker.erl
+++ b/src/couch_replicator/src/couch_replicator_worker.erl
@@ -372,8 +372,9 @@ handle_flush_docs_result({error, request_body_too_large}, Target, DocList) ->
         " request body is too large. Splitting batch into 2 separate batches of"
         " sizes ~p and ~p", [Len, couch_replicator_api_wrap:db_uri(Target),
         length(DocList1), length(DocList2)]),
-    flush_docs(Target, DocList1),
-    flush_docs(Target, DocList2);
+    Stats1 = flush_docs(Target, DocList1),
+    Stats2 = flush_docs(Target, DocList2),
+    couch_replicator_stats:sum_stats(Stats1, Stats2);
 handle_flush_docs_result({ok, Errors}, Target, DocList) ->
     DbUri = couch_replicator_api_wrap:db_uri(Target),
     lists:foreach(