You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by rn...@apache.org on 2020/09/04 12:13:57 UTC
[couchdb] branch delay_response_until_end created (now 667cf84)
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a change to branch delay_response_until_end
in repository https://gitbox.apache.org/repos/asf/couchdb.git.
at 667cf84 Add option to delay responses until the end
This branch includes the following new commits:
new 667cf84 Add option to delay responses until the end
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[couchdb] 01/01: Add option to delay responses until the end
Posted by rn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rnewson pushed a commit to branch delay_response_until_end
in repository https://gitbox.apache.org/repos/asf/couchdb.git
commit 667cf842ad7f6fb8bccbd7109666c16e00c807f9
Author: Robert Newson <rn...@apache.org>
AuthorDate: Fri Sep 4 12:38:11 2020 +0100
Add option to delay responses until the end
When set, every response is sent once fully generated on the server
side. This increases memory usage on the nodes but simplifies error
handling for the client as it eliminates the possibility that the
response will be deliberately terminated midway through due to a
timeout.
The config value can be changed at runtime without impacting any
in-flight responses.
---
src/chttpd/src/chttpd.erl | 48 ++++++++++++++----
src/chttpd/test/eunit/chttpd_delayed_test.erl | 73 +++++++++++++++++++++++++++
2 files changed, 112 insertions(+), 9 deletions(-)
diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl
index adde073..dcaca89 100644
--- a/src/chttpd/src/chttpd.erl
+++ b/src/chttpd/src/chttpd.erl
@@ -52,8 +52,9 @@
req,
code,
headers,
- first_chunk,
- resp=nil
+ chunks,
+ resp=nil,
+ delay_until_end=false
}).
start_link() ->
@@ -780,40 +781,56 @@ start_json_response(Req, Code, Headers0) ->
end_json_response(Resp) ->
couch_httpd:end_json_response(Resp).
+
start_delayed_json_response(Req, Code) ->
start_delayed_json_response(Req, Code, []).
+
start_delayed_json_response(Req, Code, Headers) ->
start_delayed_json_response(Req, Code, Headers, "").
+
start_delayed_json_response(Req, Code, Headers, FirstChunk) ->
+ DelayUntilEnd = config:get_boolean("chttpd", "delay_until_end", false),
{ok, #delayed_resp{
start_fun = fun start_json_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ delay_until_end = DelayUntilEnd}}.
+
start_delayed_chunked_response(Req, Code, Headers) ->
start_delayed_chunked_response(Req, Code, Headers, "").
+
start_delayed_chunked_response(Req, Code, Headers, FirstChunk) ->
+ DelayUntilEnd = config:get_boolean("chttpd", "delay_until_end", false),
{ok, #delayed_resp{
start_fun = fun start_chunked_response/3,
req = Req,
code = Code,
headers = Headers,
- first_chunk = FirstChunk}}.
+ chunks = [FirstChunk],
+ delay_until_end = DelayUntilEnd}}.
-send_delayed_chunk(#delayed_resp{}=DelayedResp, Chunk) ->
+
+send_delayed_chunk(#delayed_resp{delay_until_end=false}=DelayedResp, Chunk) ->
{ok, #delayed_resp{resp=Resp}=DelayedResp1} =
start_delayed_response(DelayedResp),
{ok, Resp} = send_chunk(Resp, Chunk),
- {ok, DelayedResp1}.
+ {ok, DelayedResp1};
+
+send_delayed_chunk(#delayed_resp{delay_until_end=true}=DelayedResp, Chunk) ->
+ #delayed_resp{chunks = Chunks} = DelayedResp,
+ {ok, DelayedResp#delayed_resp{chunks = [Chunk | Chunks]}}.
+
send_delayed_last_chunk(Req) ->
send_delayed_chunk(Req, []).
+
send_delayed_error(#delayed_resp{req=Req,resp=nil}=DelayedResp, Reason) ->
{Code, ErrorStr, ReasonStr} = error_info(Reason),
{ok, Resp} = send_error(Req, Code, ErrorStr, ReasonStr),
@@ -823,6 +840,7 @@ send_delayed_error(#delayed_resp{resp=Resp, req=Req}, Reason) ->
log_error_with_stack_trace(Reason),
throw({http_abort, Resp, Reason}).
+
close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
% Use a separate chunk to close the streamed array to maintain strict
% compatibility with earlier versions. See COUCHDB-2724
@@ -831,10 +849,22 @@ close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
close_delayed_json_object(Resp, Buffer, Terminator, _Threshold) ->
send_delayed_chunk(Resp, [Buffer | Terminator]).
-end_delayed_json_response(#delayed_resp{}=DelayedResp) ->
+
+end_delayed_json_response(#delayed_resp{delay_until_end=false}=DelayedResp) ->
{ok, #delayed_resp{resp=Resp}} =
start_delayed_response(DelayedResp),
- end_json_response(Resp).
+ end_json_response(Resp);
+
+end_delayed_json_response(#delayed_resp{delay_until_end=true}=DelayedResp) ->
+ #delayed_resp{
+ req = Req,
+ code = Code,
+ headers = Headers,
+ chunks = Chunks
+ } = DelayedResp,
+ {ok, Resp} = start_response_length(Req, Code, Headers, iolist_size(Chunks)),
+ send(Resp, lists:reverse(Chunks)).
+
get_delayed_req(#delayed_resp{req=#httpd{mochi_req=MochiReq}}) ->
MochiReq;
@@ -847,7 +877,7 @@ start_delayed_response(#delayed_resp{resp=nil}=DelayedResp) ->
req=Req,
code=Code,
headers=Headers,
- first_chunk=FirstChunk
+ chunks=[FirstChunk]
}=DelayedResp,
{ok, Resp} = StartFun(Req, Code, Headers),
case FirstChunk of
diff --git a/src/chttpd/test/eunit/chttpd_delayed_test.erl b/src/chttpd/test/eunit/chttpd_delayed_test.erl
new file mode 100644
index 0000000..724ef4d
--- /dev/null
+++ b/src/chttpd/test/eunit/chttpd_delayed_test.erl
@@ -0,0 +1,73 @@
+-module(chttpd_delayed_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_view_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+-define(DDOC, "{\"_id\": \"_design/bar\", \"views\": {\"baz\":
+ {\"map\": \"function(doc) {emit(doc._id, doc._id);}\"}}}").
+
+-define(FIXTURE_TXT, ?ABS_PATH(?FILE)).
+-define(i2l(I), integer_to_list(I)).
+-define(TIMEOUT, 60). % seconds
+
+setup() ->
+ Hashed = couch_passwords:hash_admin_password(?PASS),
+ ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist=false),
+ ok = config:set("chttpd", "delay_until_end", "true"),
+ TmpDb = ?tempdb(),
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]),
+ create_db(Url),
+ Url.
+
+teardown(Url) ->
+ delete_db(Url),
+ ok = config:delete("admins", ?USER, _Persist=false).
+
+create_db(Url) ->
+ {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+
+delete_db(Url) ->
+ {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
+
+
+all_test_() ->
+ {
+ "chttpd delay tests",
+ {
+ setup,
+ fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun test_delay_until_end_all_docs/1,
+ fun test_delay_until_end_changes/1
+ ]
+ }
+ }
+ }.
+
+
+test_delay_until_end_all_docs(Url) ->
+ assert_has_content_length(Url ++ "/_all_docs").
+
+
+test_delay_until_end_changes(Url) ->
+ assert_has_content_length(Url ++ "/_changes").
+
+
+assert_has_content_length(Url) ->
+ {timeout, ?TIMEOUT, ?_test(begin
+ {ok, Code, Headers, _Body} = test_request:get(Url, [?AUTH]),
+ ?assertEqual(200, Code),
+ ?assert(lists:keymember("Content-Length", 1, Headers))
+ end)}.
+
\ No newline at end of file