You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2015/07/20 22:27:02 UTC
[1/3] couch-mrview commit: updated refs/heads/2724-chunked-buffering
to 2f489b9
Repository: couchdb-couch-mrview
Updated Branches:
refs/heads/2724-chunked-buffering [created] 2f489b9cd
Avoid gratuitous list flattening
Mochiweb and gen_tcp are perfectly happy with iolists.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/4038d9eb
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/4038d9eb
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/4038d9eb
Branch: refs/heads/2724-chunked-buffering
Commit: 4038d9eb0cb9b1c291fdcfafa5fcd12ad1b85e3c
Parents: b743459
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jul 20 15:53:18 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jul 20 15:53:18 2015 -0400
----------------------------------------------------------------------
src/couch_mrview_http.erl | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/4038d9eb/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index ccc8dc1..d0da69e 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -343,8 +343,7 @@ view_cb({meta, Meta}, #vacc{resp=Resp}=Acc) ->
undefined -> [];
UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
end ++ ["\"rows\":["],
- Prepend = prepend_val(Acc),
- Chunk = lists:flatten(Prepend ++ "{" ++ string:join(Parts, ",") ++ "\r\n"),
+ Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"],
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
{ok, Acc#vacc{resp=Resp1, prepend=""}};
view_cb({row, Row}, Acc) ->
[2/3] couch-mrview commit: updated refs/heads/2724-chunked-buffering
to 2f489b9
Posted by ko...@apache.org.
Buffer rows to reduce number of chunks
This patch reduces the number of chunks in an HTTP chunked response body
by coalescing multiple rows into a single transmission. The default
value is chosen to fill a standard Ethernet frame but will be made
configurable.
COUCHDB-2724
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/bb94b2a5
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/bb94b2a5
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/bb94b2a5
Branch: refs/heads/2724-chunked-buffering
Commit: bb94b2a528ae9410517d57de1d3fb553556a4ce1
Parents: 4038d9e
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jul 20 16:10:24 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jul 20 16:20:16 2015 -0400
----------------------------------------------------------------------
include/couch_mrview.hrl | 5 ++++-
src/couch_mrview_http.erl | 28 ++++++++++++++++++++--------
2 files changed, 24 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/bb94b2a5/include/couch_mrview.hrl
----------------------------------------------------------------------
diff --git a/include/couch_mrview.hrl b/include/couch_mrview.hrl
index 1a0cdea..c176839 100644
--- a/include/couch_mrview.hrl
+++ b/include/couch_mrview.hrl
@@ -94,7 +94,10 @@
resp,
prepend,
etag,
- should_close = false
+ should_close = false,
+ buffer = [],
+ bufsize = 0,
+ threshold = 1490
}).
-record(lacc, {
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/bb94b2a5/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index d0da69e..9149689 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -331,7 +331,7 @@ view_cb({meta, Meta}, #vacc{resp=undefined}=Acc) ->
Headers = [{"ETag", Acc#vacc.etag}],
{ok, Resp} = chttpd:start_delayed_json_response(Acc#vacc.req, 200, Headers),
view_cb({meta, Meta}, Acc#vacc{resp=Resp, should_close=true});
-view_cb({meta, Meta}, #vacc{resp=Resp}=Acc) ->
+view_cb({meta, Meta}, #vacc{}=Acc) ->
% Sending metadata
Parts = case couch_util:get_value(total, Meta) of
undefined -> [];
@@ -344,26 +344,25 @@ view_cb({meta, Meta}, #vacc{resp=Resp}=Acc) ->
UpdateSeq -> [io_lib:format("\"update_seq\":~p", [UpdateSeq])]
end ++ ["\"rows\":["],
Chunk = [prepend_val(Acc), "{", string:join(Parts, ","), "\r\n"],
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, Acc#vacc{resp=Resp1, prepend=""}};
+ {ok, AccOut} = maybe_flush_response(Acc, Chunk, iolist_size(Chunk)),
+ {ok, AccOut#vacc{prepend=""}};
view_cb({row, Row}, Acc) ->
% Adding another row
Chunk = [prepend_val(Acc), row_to_json(Row)],
- {ok, Resp1} = chttpd:send_delayed_chunk(Acc#vacc.resp, Chunk),
- {ok, Acc#vacc{prepend=",\r\n", resp=Resp1}};
+ maybe_flush_response(Acc, Chunk, iolist_size(Chunk));
view_cb(complete, #vacc{resp=undefined}=Acc) ->
% Nothing in view
{ok, Resp} = chttpd:send_json(Acc#vacc.req, 200, {[{rows, []}]}),
{ok, Acc#vacc{resp=Resp}};
-view_cb(complete, #vacc{resp=Resp}=Acc) ->
+view_cb(complete, #vacc{resp=Resp, buffer=Buf}=Acc) ->
% Finish view output and possibly end the response
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, "\r\n]}"),
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [Buf | "\r\n]}"]),
case Acc#vacc.should_close of
true ->
{ok, Resp2} = chttpd:end_delayed_json_response(Resp1),
{ok, Acc#vacc{resp=Resp2}};
_ ->
- {ok, Acc#vacc{resp=Resp1, prepend=",\r\n"}}
+ {ok, Acc#vacc{resp=Resp1, prepend=",\r\n", buffer=[], bufsize=0}}
end;
view_cb({error, Reason}, #vacc{resp=undefined}=Acc) ->
{ok, Resp} = chttpd:send_error(Acc#vacc.req, Reason),
@@ -372,6 +371,19 @@ view_cb({error, Reason}, #vacc{resp=Resp}=Acc) ->
{ok, Resp1} = chttpd:send_delayed_error(Resp, Reason),
{ok, Acc#vacc{resp=Resp1}}.
+maybe_flush_response(#vacc{bufsize=Size, threshold=Max} = Acc, Data, Len)
+ when (Size + Len) > Max ->
+ #vacc{buffer = Buffer, resp = Resp} = Acc,
+ {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
+ {ok, Acc#vacc{prepend = ",\r\n", buffer = Data, bufsize = Len, resp = R1}};
+maybe_flush_response(Acc0, Data, Len) ->
+ #vacc{buffer = Buf, bufsize = Size} = Acc0,
+ Acc = Acc0#vacc{
+ prepend = ",\r\n",
+ buffer = [Buf | Data],
+ bufsize = Size + Len
+ },
+ {ok, Acc}.
prepend_val(#vacc{prepend=Prepend}) ->
case Prepend of
[3/3] couch-mrview commit: updated refs/heads/2724-chunked-buffering
to 2f489b9
Posted by ko...@apache.org.
Read the chunk buffer size from config
Project: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/commit/2f489b9c
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/tree/2f489b9c
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/diff/2f489b9c
Branch: refs/heads/2724-chunked-buffering
Commit: 2f489b9cd0086b3b8daf7e2670bb6ec477983c64
Parents: bb94b2a
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jul 20 16:20:25 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Mon Jul 20 16:20:25 2015 -0400
----------------------------------------------------------------------
src/couch_mrview_http.erl | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-couch-mrview/blob/2f489b9c/src/couch_mrview_http.erl
----------------------------------------------------------------------
diff --git a/src/couch_mrview_http.erl b/src/couch_mrview_http.erl
index 9149689..d7aff3b 100644
--- a/src/couch_mrview_http.erl
+++ b/src/couch_mrview_http.erl
@@ -229,7 +229,8 @@ do_all_docs_req(Req, Db, Keys, NS) ->
end,
Args = Args1#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- VAcc0 = #vacc{db=Db, req=Req},
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
DbName = ?b2l(Db#db.name),
UsersDbName = config:get("couch_httpd_auth",
"authentication_db",
@@ -274,7 +275,8 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end,
Args = Args0#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- VAcc0 = #vacc{db=Db, req=Req},
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
couch_mrview:query_view(Db, DDoc, ViewName, Args, fun view_cb/2, VAcc0)
end),
case is_record(Resp, vacc) of
@@ -291,7 +293,8 @@ multi_query_view(Req, Db, DDoc, ViewName, Queries) ->
couch_mrview_util:validate_args(QueryArg)
end, Queries),
{ok, Resp2} = couch_httpd:etag_maybe(Req, fun() ->
- VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n"},
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ VAcc0 = #vacc{db=Db, req=Req, prepend="\r\n", threshold=Max},
%% TODO: proper calculation of etag
Etag = couch_uuids:new(),
Headers = [{"ETag", Etag}],