[couchdb] branch fix/attachment-writers-larger-n updated (8a4d45b -> 261b90a)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jan pushed a change to branch fix/attachment-writers-larger-n
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


 discard 8a4d45b  fix: avoid dropping attachment chunks on quorum writes
     new 261b90a  fix: avoid dropping attachment chunks on quorum writes

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (8a4d45b)
            \
             N -- N -- N   refs/heads/fix/attachment-writers-larger-n (261b90a)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:

[couchdb] 01/01: fix: avoid dropping attachment chunks on quorum writes

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch fix/attachment-writers-larger-n
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 261b90abd445e6a60e5f1ad8ba5e02492e56e8c8
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Thu Aug 26 16:41:58 2021 +0200

    fix: avoid dropping attachment chunks on quorum writes
    
    This only applies to databases that have an n > [cluster] n.
    
    Our `middleman()` function that proxies attachment streams from
    the incoming HTTP socket on the coordinating node to the target
    shard-bearing nodes used the server config to determine whether
    it should start dropping chunks from the stream.
    
    If a database was created with a larger `n`, the `middleman()`
    function could have started to drop attachment chunks before
    all attached nodes had a chance to receive it.
    
    This fix uses a database’s concrete `n` value rather than the
    server config default value.
    
    Co-Authored-By: James Coglan <jc...@gmail.com>
    Co-Authored-By: Robert Newson <rn...@apache.org>
---
 src/chttpd/src/chttpd_db.erl       |  2 +-
 src/fabric/src/fabric.erl          |  8 ++++----
 src/fabric/src/fabric_doc_atts.erl | 26 +++++++++++++-------------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ffdab2a..31ada8f 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -1571,7 +1571,7 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
                 undefined -> <<"application/octet-stream">>;
                 CType -> list_to_binary(CType)
             end,
-            Data = fabric:att_receiver(Req, chttpd:body_length(Req)),
+            Data = fabric:att_receiver(Req, couch_db:name(Db), chttpd:body_length(Req)),
             ContentLen = case couch_httpd:header_value(Req,"Content-Length") of
                 undefined -> undefined;
                 Length -> list_to_integer(Length)
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 27fa8c0..34b967d 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -28,7 +28,7 @@
 % Documents
 -export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
     get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3,
-    purge_docs/3, att_receiver/2]).
+    purge_docs/3, att_receiver/3]).
 
 % Views
 -export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
@@ -324,11 +324,11 @@ purge_docs(DbName, IdsRevs, Options) when is_list(IdsRevs) ->
 %%      returns a fabric attachment receiver context tuple
 %%      with the spawned middleman process, an empty binary,
 %%      or exits with an error tuple {Error, Arg}
--spec att_receiver(#httpd{}, Length :: undefined | chunked | pos_integer() |
+-spec att_receiver(#httpd{}, dbname(), Length :: undefined | chunked | pos_integer() |
         {unknown_transfer_encoding, any()}) ->
     {fabric_attachment_receiver, pid(), chunked | pos_integer()} | binary().
-att_receiver(Req, Length) ->
-    fabric_doc_atts:receiver(Req, Length).
+att_receiver(Req, DbName, Length) ->
+    fabric_doc_atts:receiver(Req, DbName, Length).
 
 %% @equiv all_docs(DbName, [], Callback, Acc0, QueryArgs)
 all_docs(DbName, Callback, Acc, QueryArgs) ->
diff --git a/src/fabric/src/fabric_doc_atts.erl b/src/fabric/src/fabric_doc_atts.erl
index 65ba65f..bd0687b 100644
--- a/src/fabric/src/fabric_doc_atts.erl
+++ b/src/fabric/src/fabric_doc_atts.erl
@@ -18,25 +18,25 @@
 -include_lib("couch/include/couch_db.hrl").
 
 -export([
-    receiver/2,
+    receiver/3,
     receiver_callback/2
 ]).
 
 
-receiver(_Req, undefined) ->
+receiver(_Req, _DbName, undefined) ->
     <<"">>;
-receiver(_Req, {unknown_transfer_encoding, Unknown}) ->
+receiver(_Req, _DbName, {unknown_transfer_encoding, Unknown}) ->
     exit({unknown_transfer_encoding, Unknown});
-receiver(Req, chunked) ->
-    MiddleMan = spawn(fun() -> middleman(Req, chunked) end),
+receiver(Req, DbName, chunked) ->
+    MiddleMan = spawn(fun() -> middleman(Req, DbName, chunked) end),
     {fabric_attachment_receiver, MiddleMan, chunked};
-receiver(_Req, 0) ->
+receiver(_Req, _DbName, 0) ->
     <<"">>;
-receiver(Req, Length) when is_integer(Length) ->
+receiver(Req, DbName, Length) when is_integer(Length) ->
     maybe_send_continue(Req),
-    Middleman = spawn(fun() -> middleman(Req, Length) end),
+    Middleman = spawn(fun() -> middleman(Req, DbName, Length) end),
     {fabric_attachment_receiver, Middleman, Length};
-receiver(_Req, Length) ->
+receiver(_Req, _DbName, Length) ->
     exit({length_not_integer, Length}).
 
 
@@ -108,7 +108,7 @@ receive_unchunked_attachment(Req, Length) ->
     end,
     receive_unchunked_attachment(Req, Length - size(Data)).
 
-middleman(Req, chunked) ->
+middleman(Req, DbName, chunked) ->
     % spawn a process to actually receive the uploaded data
     RcvFun = fun(ChunkRecord, ok) ->
         receive {From, go} -> From ! {self(), ChunkRecord} end, ok
@@ -116,13 +116,13 @@ middleman(Req, chunked) ->
     Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req,4096,RcvFun,ok) end),
 
     % take requests from the DB writers and get data from the receiver
-    N = config:get_integer("cluster", "n", 3),
+    N = mem3:n(DbName),
     Timeout = fabric_util:attachments_timeout(),
     middleman_loop(Receiver, N, [], [], Timeout);
 
-middleman(Req, Length) ->
+middleman(Req, DbName, Length) ->
     Receiver = spawn(fun() -> receive_unchunked_attachment(Req, Length) end),
-    N = config:get_integer("cluster", "n", 3),
+    N = mem3:n(DbName),
     Timeout = fabric_util:attachments_timeout(),
     middleman_loop(Receiver, N, [], [], Timeout).