[couchdb] branch fix/attachment-writers-larger-n created (now 8a4d45b)

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jan pushed a change to branch fix/attachment-writers-larger-n
in repository https://gitbox.apache.org/repos/asf/couchdb.git.


      at 8a4d45b  fix: avoid dropping attachment chunks on quorum writes

This branch includes the following new commits:

     new 8a4d45b  fix: avoid dropping attachment chunks on quorum writes

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[couchdb] 01/01: fix: avoid dropping attachment chunks on quorum writes

[ja...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jan pushed a commit to branch fix/attachment-writers-larger-n
in repository https://gitbox.apache.org/repos/asf/couchdb.git

commit 8a4d45bd198ee59af3300dfb953cea8723d56432
Author: Jan Lehnardt <ja...@apache.org>
AuthorDate: Thu Aug 26 16:41:58 2021 +0200

    fix: avoid dropping attachment chunks on quorum writes
    
    This only applies to databases that have an n > [cluster] n.
    
    Our `middleman()` function that proxies attachment streams from
    the incoming HTTP socket on the coordinating node to the target
    shard-bearing nodes used the server config to determine whether
    it should start dropping chunks from the stream.
    
    If a database was created with a larger `n`, the `middleman()`
    function could have started to drop attachment chunks before
    all attached nodes had a chance to receive it.
    
    This fix uses a database’s concrete `n` value rather than the
    server config default value.
    
    Co-Authored-By: James Coglan <jc...@gmail.com>
    Co-AUthored-By: Robert Newson <rn...@apache.org>
---
 src/chttpd/src/chttpd_db.erl       |  2 +-
 src/fabric/src/fabric.erl          |  8 ++++----
 src/fabric/src/fabric_doc_atts.erl | 26 +++++++++++++-------------
 3 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl
index ffdab2a..31ada8f 100644
--- a/src/chttpd/src/chttpd_db.erl
+++ b/src/chttpd/src/chttpd_db.erl
@@ -1571,7 +1571,7 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa
                 undefined -> <<"application/octet-stream">>;
                 CType -> list_to_binary(CType)
             end,
-            Data = fabric:att_receiver(Req, chttpd:body_length(Req)),
+            Data = fabric:att_receiver(Req, couch_db:name(Db), chttpd:body_length(Req)),
             ContentLen = case couch_httpd:header_value(Req,"Content-Length") of
                 undefined -> undefined;
                 Length -> list_to_integer(Length)
diff --git a/src/fabric/src/fabric.erl b/src/fabric/src/fabric.erl
index 27fa8c0..34b967d 100644
--- a/src/fabric/src/fabric.erl
+++ b/src/fabric/src/fabric.erl
@@ -28,7 +28,7 @@
 % Documents
 -export([open_doc/3, open_revs/4, get_doc_info/3, get_full_doc_info/3,
     get_missing_revs/2, get_missing_revs/3, update_doc/3, update_docs/3,
-    purge_docs/3, att_receiver/2]).
+    purge_docs/3, att_receiver/3]).
 
 % Views
 -export([all_docs/4, all_docs/5, changes/4, query_view/3, query_view/4,
@@ -324,11 +324,11 @@ purge_docs(DbName, IdsRevs, Options) when is_list(IdsRevs) ->
 %%      returns a fabric attachment receiver context tuple
 %%      with the spawned middleman process, an empty binary,
 %%      or exits with an error tuple {Error, Arg}
--spec att_receiver(#httpd{}, Length :: undefined | chunked | pos_integer() |
+-spec att_receiver(#httpd{}, dbname(), Length :: undefined | chunked | pos_integer() |
         {unknown_transfer_encoding, any()}) ->
     {fabric_attachment_receiver, pid(), chunked | pos_integer()} | binary().
-att_receiver(Req, Length) ->
-    fabric_doc_atts:receiver(Req, Length).
+att_receiver(Req, DbName, Length) ->
+    fabric_doc_atts:receiver(Req, DbName, Length).
 
 %% @equiv all_docs(DbName, [], Callback, Acc0, QueryArgs)
 all_docs(DbName, Callback, Acc, QueryArgs) ->
diff --git a/src/fabric/src/fabric_doc_atts.erl b/src/fabric/src/fabric_doc_atts.erl
index 65ba65f..bd0687b 100644
--- a/src/fabric/src/fabric_doc_atts.erl
+++ b/src/fabric/src/fabric_doc_atts.erl
@@ -18,25 +18,25 @@
 -include_lib("couch/include/couch_db.hrl").
 
 -export([
-    receiver/2,
+    receiver/3,
     receiver_callback/2
 ]).
 
 
-receiver(_Req, undefined) ->
+receiver(_Req, _DbName, undefined) ->
     <<"">>;
-receiver(_Req, {unknown_transfer_encoding, Unknown}) ->
+receiver(_Req, _DbName, {unknown_transfer_encoding, Unknown}) ->
     exit({unknown_transfer_encoding, Unknown});
-receiver(Req, chunked) ->
-    MiddleMan = spawn(fun() -> middleman(Req, chunked) end),
+receiver(Req, DbName, chunked) ->
+    MiddleMan = spawn(fun() -> middleman(Req, DbName, chunked) end),
     {fabric_attachment_receiver, MiddleMan, chunked};
-receiver(_Req, 0) ->
+receiver(_Req, _DbName, 0) ->
     <<"">>;
-receiver(Req, Length) when is_integer(Length) ->
+receiver(Req, DbName, Length) when is_integer(Length) ->
     maybe_send_continue(Req),
-    Middleman = spawn(fun() -> middleman(Req, Length) end),
+    Middleman = spawn(fun() -> middleman(Req, DbName, Length) end),
     {fabric_attachment_receiver, Middleman, Length};
-receiver(_Req, Length) ->
+receiver(_Req, _DbName, Length) ->
     exit({length_not_integer, Length}).
 
 
@@ -108,7 +108,7 @@ receive_unchunked_attachment(Req, Length) ->
     end,
     receive_unchunked_attachment(Req, Length - size(Data)).
 
-middleman(Req, chunked) ->
+middleman(Req, DbName, chunked) ->
     % spawn a process to actually receive the uploaded data
     RcvFun = fun(ChunkRecord, ok) ->
         receive {From, go} -> From ! {self(), ChunkRecord} end, ok
@@ -116,13 +116,13 @@ middleman(Req, chunked) ->
     Receiver = spawn(fun() -> couch_httpd:recv_chunked(Req,4096,RcvFun,ok) end),
 
     % take requests from the DB writers and get data from the receiver
-    N = config:get_integer("cluster", "n", 3),
+    N = mem3:n(DbName),
     Timeout = fabric_util:attachments_timeout(),
     middleman_loop(Receiver, N, [], [], Timeout);
 
-middleman(Req, Length) ->
+middleman(Req, DbName, Length) ->
     Receiver = spawn(fun() -> receive_unchunked_attachment(Req, Length) end),
-    N = config:get_integer("cluster", "n", 3),
+    N = mem3:n(DbName),
     Timeout = fabric_util:attachments_timeout(),
     middleman_loop(Receiver, N, [], [], Timeout).