You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2020/09/07 13:14:07 UTC
[couchdb] branch prototype/fdb-layer updated: Add option to delay
responses until the end
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch prototype/fdb-layer
in repository https://gitbox.apache.org/repos/asf/couchdb.git
The following commit(s) were added to refs/heads/prototype/fdb-layer by this push:
new f677f1e Add option to delay responses until the end
f677f1e is described below
commit f677f1eeb02a8ccfd6f637d53c00993112aced30
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Sep 4 12:38:11 2020 +0100
Add option to delay responses until the end
When set, every response is sent once fully generated on the server
side. This increases memory usage on the nodes but simplifies error
handling for the client as it eliminates the possibility that the
response will be deliberately terminated midway through due to a
timeout.
The config value can be changed at runtime without impacting any
in-flight responses.
---
rel/overlay/etc/default.ini | 3 ++
src/chttpd/src/chttpd.erl | 58 +++++++++++++++++----
src/chttpd/test/eunit/chttpd_delayed_test.erl | 73 +++++++++++++++++++++++++++
3 files changed, 125 insertions(+), 9 deletions(-)
diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini
index 1c37765..071359a 100644
--- a/rel/overlay/etc/default.ini
+++ b/rel/overlay/etc/default.ini
@@ -145,6 +145,9 @@ prefer_minimal = Cache-Control, Content-Length, Content-Range, Content-Type, ETa
; _dbs_info in a request
max_db_number_for_dbs_info_req = 100
+; set to true to delay the start of a response until the end has been calculated
+;buffer_response = false
+
; authentication handlers
; authentication_handlers = {chttpd_auth, cookie_authentication_handler}, {chttpd_auth, default_authentication_handler}
; uncomment the next line to enable proxy authentication
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index 325369e..fdca5c8 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -52,8 +52,9 @@
req,
code,
headers,
- first_chunk,
- resp=nil
+ chunks,
+ resp=nil,
+ buffer_response=false
}).
start_link() ->
@@ -800,40 +801,54 @@ start_json_response(Req, Code, Headers0) ->
end_json_response(Resp) ->
couch_httpd:end_json_response(Resp).
+
start_delayed_json_response(Req, Code) ->
start_delayed_json_response(Req, Code, []).
+
start_delayed_json_response(Req, Code, Headers) ->
start_delayed_json_response(Req, Code, Headers, "").
+
start_delayed_json_response(Req, Code, Headers, FirstChunk) ->
{ok, #delayed_resp{
start_fun = fun start_json_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ buffer_response = buffer_response(Req)}}.
+
start_delayed_chunked_response(Req, Code, Headers) ->
start_delayed_chunked_response(Req, Code, Headers, "").
+
start_delayed_chunked_response(Req, Code, Headers, FirstChunk) ->
{ok, #delayed_resp{
start_fun = fun start_chunked_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ buffer_response = buffer_response(Req)}}.
+
-send_delayed_chunk(#delayed_resp{}=DelayedResp, Chunk) ->
+send_delayed_chunk(#delayed_resp{buffer_response=false}=DelayedResp, Chunk) ->
{ok, #delayed_resp{resp=Resp}=DelayedResp1} =
start_delayed_response(DelayedResp),
{ok, Resp} = send_chunk(Resp, Chunk),
- {ok, DelayedResp1}.
+ {ok, DelayedResp1};
+
+send_delayed_chunk(#delayed_resp{buffer_response=true}=DelayedResp, Chunk) ->
+ #delayed_resp{chunks = Chunks} = DelayedResp,
+ {ok, DelayedResp#delayed_resp{chunks = [Chunk | Chunks]}}.
+
send_delayed_last_chunk(Req) ->
send_delayed_chunk(Req, []).
+
send_delayed_error(#delayed_resp{req=Req,resp=nil}=DelayedResp, Reason) ->
{Code, ErrorStr, ReasonStr} = error_info(Reason),
{ok, Resp} = send_error(Req, Code, ErrorStr, ReasonStr),
@@ -843,6 +858,7 @@ send_delayed_error(#delayed_resp{resp=Resp, req=Req}, Reason) ->
log_error_with_stack_trace(Reason),
throw({http_abort, Resp, Reason}).
+
close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
% Use a separate chunk to close the streamed array to maintain strict
% compatibility with earlier versions. See COUCHDB-2724
@@ -851,10 +867,22 @@ close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
close_delayed_json_object(Resp, Buffer, Terminator, _Threshold) ->
send_delayed_chunk(Resp, [Buffer | Terminator]).
-end_delayed_json_response(#delayed_resp{}=DelayedResp) ->
+
+end_delayed_json_response(#delayed_resp{buffer_response=false}=DelayedResp) ->
{ok, #delayed_resp{resp=Resp}} =
start_delayed_response(DelayedResp),
- end_json_response(Resp).
+ end_json_response(Resp);
+
+end_delayed_json_response(#delayed_resp{buffer_response=true}=DelayedResp) ->
+ #delayed_resp{
+ req = Req,
+ code = Code,
+ headers = Headers,
+ chunks = Chunks
+ } = DelayedResp,
+ {ok, Resp} = start_response_length(Req, Code, Headers, iolist_size(Chunks)),
+ send(Resp, lists:reverse(Chunks)).
+
get_delayed_req(#delayed_resp{req=#httpd{mochi_req=MochiReq}}) ->
MochiReq;
@@ -867,7 +895,7 @@ start_delayed_response(#delayed_resp{resp=nil}=DelayedResp) ->
req=Req,
code=Code,
headers=Headers,
- first_chunk=FirstChunk
+ chunks=[FirstChunk]
}=DelayedResp,
{ok, Resp} = StartFun(Req, Code, Headers),
case FirstChunk of
@@ -878,6 +906,18 @@ start_delayed_response(#delayed_resp{resp=nil}=DelayedResp) ->
start_delayed_response(#delayed_resp{}=DelayedResp) ->
{ok, DelayedResp}.
+
+buffer_response(Req) ->
+ case chttpd:qs_value(Req, "buffer_response") of
+ "false" ->
+ false;
+ "true" ->
+ true;
+ _ ->
+ config:get_boolean("chttpd", "buffer_response", false)
+ end.
+
+
error_info({Error, Reason}) when is_list(Reason) ->
error_info({Error, couch_util:to_binary(Reason)});
error_info(bad_request) ->
diff --git a/src/chttpd/test/eunit/chttpd_delayed_test.erl b/src/chttpd/test/eunit/chttpd_delayed_test.erl
new file mode 100644
index 0000000..64232dc
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_delayed_test.erl
@@ -0,0 +1,73 @@
+-module(chttpd_delayed_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_view_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+-define(DDOC, "{\"_id\": \"_design/bar\", \"views\": {\"baz\":
+ {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}").
+
+-define(FIXTURE_TXT, ?ABS_PATH(?FILE)).
+-define(i2l(I), integer_to_list(I)).
+-define(TIMEOUT, 60). % seconds
+
+setup() ->
+ Hashed = couch_passwords:hash_admin_password(?PASS),
+ ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false),
+ ok = config:set("chttpd", "buffer_response", "true"),
+ TmpDb = ?tempdb(),
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]),
+ create_db(Url),
+ Url.
+
+teardown(Url) ->
+ delete_db(Url),
+ ok = config:delete("admins", ?USER, _Persist=false).
+
+create_db(Url) ->
+ {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+
+delete_db(Url) ->
+ {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
+
+
+all_test_() ->
+ {
+ "chttpd delay tests",
+ {
+ setup,
+ fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun test_buffer_response_all_docs/1,
+ fun test_buffer_response_changes/1
+ ]
+ }
+ }
+ }.
+
+
+test_buffer_response_all_docs(Url) ->
+ assert_has_content_length(Url ++ "/_all_docs").
+
+
+test_buffer_response_changes(Url) ->
+ assert_has_content_length(Url ++ "/_changes").
+
+
+assert_has_content_length(Url) ->
+ {timeout, ?TIMEOUT, ?_test(begin
+ {ok, Code, Headers, _Body} = test_request:get(Url, [?AUTH]),
+ ?assertEqual(200, Code),
+ ?assert(lists:keymember("Content-Length", 1, Headers))
+ end)}.
+
\ No newline at end of file