You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ko...@apache.org on 2015/07/22 23:01:22 UTC
[1/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Repository: couchdb-chttpd
Updated Branches:
refs/heads/2724-chunked-buffering f7b6f62c1 -> 45bb53436 (forced update)
Buffer rows for normal/longpoll feeds
This patch causes the coordinator to accumulate data in its own buffer
and reduce the number of calls to write data on the socket. The size of
the buffer is configurable:
[httpd]
chunked_response_buffer = 1490
The default is chosen to approximately fill a standard Ethernet frame.
COUCHDB-2724
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/b18805f9
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/b18805f9
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/b18805f9
Branch: refs/heads/2724-chunked-buffering
Commit: b18805f921a7a6f587f1810949e27eb002cb942e
Parents: f45c8b2
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jun 22 21:45:26 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:48 2015 -0400
----------------------------------------------------------------------
src/chttpd_db.erl | 45 +++++++++++++++++++++++++++++++++++++--------
1 file changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/b18805f9/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 71c6cec..3247d83 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -40,7 +40,10 @@
feed,
mochi,
prepend = "",
- responding = false
+ responding = false,
+ buffer = [],
+ bufsize = 0,
+ threshold
}).
-define(IS_ALL_DOCS(T), (
@@ -83,6 +86,8 @@ handle_changes_req1(#httpd{}=Req, Db) ->
ChangesArgs = Args0#changes_args{
filter_fun = couch_changes:configure_filter(Raw, Style, Req, Db)
},
+ % Default to ~filling the payload of a standard Ethernet frame
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
case ChangesArgs#changes_args.feed of
"normal" ->
T0 = os:timestamp(),
@@ -91,12 +96,21 @@ handle_changes_req1(#httpd{}=Req, Db) ->
DeltaT = timer:now_diff(os:timestamp(), T0) / 1000,
couch_stats:update_histogram([couchdb, dbinfo], DeltaT),
chttpd:etag_respond(Req, Etag, fun() ->
- Acc0 = #cacc{feed = normal, etag = Etag, mochi = Req},
+ Acc0 = #cacc{
+ feed = normal,
+ etag = Etag,
+ mochi = Req,
+ threshold = Max
+ },
fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs)
end);
Feed when Feed =:= "continuous"; Feed =:= "longpoll"; Feed =:= "eventsource" ->
couch_stats:increment_counter([couchdb, httpd, clients_requesting_changes]),
- Acc0 = #cacc{feed = list_to_atom(Feed), mochi = Req},
+ Acc0 = #cacc{
+ feed = list_to_atom(Feed),
+ mochi = Req,
+ threshold = Max
+ },
try
fabric:changes(Db, fun changes_callback/2, Acc0, ChangesArgs)
after
@@ -166,16 +180,17 @@ changes_callback(start, Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Req, 200, [], FirstChunk),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, Change}, Acc) ->
- #cacc{prepend = Prepend, mochi = Resp} = Acc,
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [Prepend, ?JSON_ENCODE(Change)]),
- {ok, Acc#cacc{prepend = ",\r\n", mochi = Resp1}};
+ Data = [Acc#cacc.prepend, ?JSON_ENCODE(Change)],
+ Len = iolist_size(Data),
+ maybe_flush_changes_feed(Acc, Data, Len);
changes_callback({stop, EndSeq, Pending}, Acc) ->
- #cacc{mochi = Resp} = Acc,
+ #cacc{buffer = Buf, mochi = Resp} = Acc,
{ok, Resp1} = case is_old_couch(Resp) of
true ->
- chttpd:send_delayed_chunk(Resp, "\n],\n\"last_seq\":0}\n");
+ chttpd:send_delayed_chunk(Resp, [Buf | "\n],\n\"last_seq\":0}\n"]);
false ->
chttpd:send_delayed_chunk(Resp, [
+ Buf,
"\n],\n\"last_seq\":",
?JSON_ENCODE(EndSeq),
",\"pending\":",
@@ -197,6 +212,20 @@ changes_callback({error, Reason}, #cacc{feed = normal, responding = false} = Acc
changes_callback({error, Reason}, Acc) ->
chttpd:send_delayed_error(Acc#cacc.mochi, Reason).
+maybe_flush_changes_feed(#cacc{bufsize=Size, threshold=Max} = Acc, Data, Len)
+ when Size > 0 andalso (Size + Len) > Max ->
+ #cacc{buffer = Buffer, mochi = Resp} = Acc,
+ {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
+ {ok, Acc#cacc{prepend = ",\r\n", buffer = Data, bufsize=Len, mochi = R1}};
+maybe_flush_changes_feed(Acc0, Data, Len) ->
+ #cacc{buffer = Buf, bufsize = Size} = Acc0,
+ Acc = Acc0#cacc{
+ prepend = ",\r\n",
+ buffer = [Buf | Data],
+ bufsize = Size + Len
+ },
+ {ok, Acc}.
+
is_old_couch(Resp) ->
MochiReq = chttpd:get_delayed_req(Resp),
case MochiReq:get_header_value("user-agent") of
[6/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Remove support for CouchDB =< 1.0.0
Thise UA-sniffing code has been in place to ensure interop with clients
running CouchDB versions older than 1.0.1. We should be safe to remove
it now.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/42781159
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/42781159
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/42781159
Branch: refs/heads/2724-chunked-buffering
Commit: 4278115947305a6d505bfe91d098840dd5ba8755
Parents: 5069427
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jul 22 14:10:43 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd_db.erl | 20 ++------------------
1 file changed, 2 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/42781159/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 329217a..b4b4a83 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -128,9 +128,8 @@ changes_callback({change, Change}, #cacc{feed = continuous} = Acc) ->
Data = [?JSON_ENCODE(Change) | "\n"],
Len = iolist_size(Data),
maybe_flush_changes_feed(Acc, Data, Len);
-changes_callback({stop, EndSeq0, Pending}, #cacc{feed = continuous} = Acc) ->
+changes_callback({stop, EndSeq, Pending}, #cacc{feed = continuous} = Acc) ->
#cacc{mochi = Resp, buffer = Buf} = Acc,
- EndSeq = case is_old_couch(Resp) of true -> 0; false -> EndSeq0 end,
Row = {[
{<<"last_seq">>, EndSeq},
{<<"pending">>, Pending}
@@ -185,11 +184,7 @@ changes_callback({change, Change}, Acc) ->
maybe_flush_changes_feed(Acc, Data, Len);
changes_callback({stop, EndSeq, Pending}, Acc) ->
#cacc{buffer = Buf, mochi = Resp} = Acc,
- {ok, Resp1} = case is_old_couch(Resp) of
- true ->
- chttpd:send_delayed_chunk(Resp, [Buf | "\n],\n\"last_seq\":0}\n"]);
- false ->
- chttpd:send_delayed_chunk(Resp, [
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [
Buf,
"\n],\n\"last_seq\":",
?JSON_ENCODE(EndSeq),
@@ -230,17 +225,6 @@ maybe_flush_changes_feed(Acc0, Data, Len) ->
},
{ok, Acc}.
-is_old_couch(Resp) ->
- MochiReq = chttpd:get_delayed_req(Resp),
- case MochiReq:get_header_value("user-agent") of
- undefined ->
- false;
- "CouchDB/1.0.0" ->
- true;
- UserAgent ->
- string:str(UserAgent, "CouchDB/0") > 0
- end.
-
handle_compact_req(Req, _) ->
Msg = <<"Compaction must be triggered on a per-shard basis in CouchDB">>,
couch_httpd:send_error(Req, 403, forbidden, Msg).
[7/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Allow configurable buffer size for all_docs, views
Now that couch_mrview can buffer chunked responses we should pass the
config value for the buffer size to it.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/c0b88502
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/c0b88502
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/c0b88502
Branch: refs/heads/2724-chunked-buffering
Commit: c0b88502a3d80bf0ccedcb6a57abdfd37786eab0
Parents: cf720aa
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Mon Jul 20 17:16:10 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd_db.erl | 3 ++-
src/chttpd_view.erl | 3 ++-
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/c0b88502/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index d8327b1..ea3879f 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -604,7 +604,8 @@ all_docs_view(Req, Db, Keys, OP) ->
Args = Args3#mrargs{preflight_fun=ETagFun},
Options = [{user_ctx, Req#httpd.user_ctx}],
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- VAcc0 = #vacc{db=Db, req=Req},
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc0, Args)
end),
case is_record(Resp, vacc) of
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/c0b88502/src/chttpd_view.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_view.erl b/src/chttpd_view.erl
index fb421a4..009aa8f 100644
--- a/src/chttpd_view.erl
+++ b/src/chttpd_view.erl
@@ -53,7 +53,8 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end,
Args = Args0#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- VAcc0 = #vacc{db=Db, req=Req},
+ Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
fabric:query_view(Db, DDoc, ViewName, fun couch_mrview_http:view_cb/2, VAcc0, Args)
end),
case is_record(Resp, vacc) of
[5/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Execute a callback for every complete DB traversal
This ensures that we don't enter a receive statement waiting for new DB
updates without first flushing the buffer.
COUCHDB-2724
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/cf720aa1
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/cf720aa1
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/cf720aa1
Branch: refs/heads/2724-chunked-buffering
Commit: cf720aa13a0cf23147d85f0c7a7c9d800d151c5f
Parents: b926134
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jun 24 13:56:18 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd_db.erl | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/cf720aa1/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 27e1795..d8327b1 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -201,6 +201,10 @@ changes_callback({stop, EndSeq, Pending}, Acc) ->
end,
chttpd:end_delayed_json_response(Resp1);
+changes_callback(waiting_for_updates, Acc) ->
+ #cacc{buffer = Buf, mochi = Resp} = Acc,
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf),
+ {ok, Acc#cacc{buffer = [], bufsize = 0, mochi = Resp1}};
changes_callback(timeout, Acc) ->
{ok, Resp1} = chttpd:send_delayed_chunk(Acc#cacc.mochi, "\n"),
{ok, Acc#cacc{mochi = Resp1}};
[2/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Match old behavior when buffering is disabled
If the threshold is set to zero we make sure to send a separate chunk to
close the JSON, otherwise we just merge it into the previous chunk.
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/45bb5343
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/45bb5343
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/45bb5343
Branch: refs/heads/2724-chunked-buffering
Commit: 45bb534361d5f6d4deff88fc3a5a621cb01156f1
Parents: 4278115
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jul 22 14:14:48 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd.erl | 11 ++++++++++-
src/chttpd_db.erl | 19 +++++++++----------
2 files changed, 19 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/45bb5343/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index 28e15b6..1cbbd04 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -35,7 +35,8 @@
get_delayed_req/1]).
-export([
- chunked_response_buffer_size/0
+ chunked_response_buffer_size/0,
+ close_delayed_json_object/4
]).
-record(delayed_resp, {
@@ -728,6 +729,14 @@ send_delayed_error(#delayed_resp{resp=Resp}, Reason) ->
log_error_with_stack_trace(Reason),
throw({http_abort, Resp, Reason}).
+close_delayed_json_object(Resp, Buffer, Terminator, 0) ->
+ % Use a separate chunk to close the streamed array to maintain strict
+ % compatibility with earlier versions. See COUCHDB-2724
+ {ok, R1} = chttpd:send_delayed_chunk(Resp, Buffer),
+ send_delayed_chunk(R1, Terminator);
+close_delayed_json_object(Resp, Buffer, Terminator, _Threshold) ->
+ send_delayed_chunk(Resp, [Buffer | Terminator]).
+
end_delayed_json_response(#delayed_resp{}=DelayedResp) ->
{ok, #delayed_resp{resp=Resp}} =
start_delayed_response(DelayedResp),
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/45bb5343/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index b4b4a83..735480c 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -183,16 +183,15 @@ changes_callback({change, Change}, Acc) ->
Len = iolist_size(Data),
maybe_flush_changes_feed(Acc, Data, Len);
changes_callback({stop, EndSeq, Pending}, Acc) ->
- #cacc{buffer = Buf, mochi = Resp} = Acc,
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [
- Buf,
- "\n],\n\"last_seq\":",
- ?JSON_ENCODE(EndSeq),
- ",\"pending\":",
- ?JSON_ENCODE(Pending),
- "}\n"
- ])
- end,
+ #cacc{buffer = Buf, mochi = Resp, threshold = Max} = Acc,
+ Terminator = [
+ "\n],\n\"last_seq\":",
+ ?JSON_ENCODE(EndSeq),
+ ",\"pending\":",
+ ?JSON_ENCODE(Pending),
+ "}\n"
+ ],
+ {ok, Resp1} = chttpd:close_delayed_json_object(Resp, Buf, Terminator, Max),
chttpd:end_delayed_json_response(Resp1);
changes_callback(waiting_for_updates, Acc) ->
[4/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Define an accessor for the chunk buffer config val
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/50694278
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/50694278
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/50694278
Branch: refs/heads/2724-chunked-buffering
Commit: 506942784469d0a084536628e5285867ca29f5e2
Parents: c0b8850
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Tue Jul 21 09:57:24 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd.erl | 16 ++++++++++++++++
src/chttpd_db.erl | 5 ++---
src/chttpd_view.erl | 2 +-
3 files changed, 19 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/50694278/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index f6ce530..28e15b6 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -34,6 +34,10 @@
send_delayed_error/2, end_delayed_json_response/1,
get_delayed_req/1]).
+-export([
+ chunked_response_buffer_size/0
+]).
+
-record(delayed_resp, {
start_fun,
req,
@@ -994,3 +998,15 @@ stack_hash(Stack) ->
with_default(undefined, Default) -> Default;
with_default(Value, _) -> Value.
+
+%% @doc CouchDB uses a chunked transfer-encoding to stream responses to
+%% _all_docs, _changes, _view and other similar requests. This configuration
+%% value sets the maximum size of a chunk; the system will buffer rows in the
+%% response until it reaches this threshold and then send all the rows in one
+%% chunk to improve network efficiency. The default value is chosen so that
+%% the assembled chunk fits into the default Ethernet frame size (some reserved
+%% padding is necessary to accommodate the reporting of the chunk length). Set
+%% this value to 0 to restore the older behavior of sending each row in a
+%% dedicated chunk.
+chunked_response_buffer_size() ->
+ config:get_integer("httpd", "chunked_response_buffer", 1490).
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/50694278/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index ea3879f..329217a 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -86,8 +86,7 @@ handle_changes_req1(#httpd{}=Req, Db) ->
ChangesArgs = Args0#changes_args{
filter_fun = couch_changes:configure_filter(Raw, Style, Req, Db)
},
- % Default to ~filling the payload of a standard Ethernet frame
- Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ Max = chttpd:chunked_response_buffer_size(),
case ChangesArgs#changes_args.feed of
"normal" ->
T0 = os:timestamp(),
@@ -604,7 +603,7 @@ all_docs_view(Req, Db, Keys, OP) ->
Args = Args3#mrargs{preflight_fun=ETagFun},
Options = [{user_ctx, Req#httpd.user_ctx}],
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ Max = chttpd:chunked_response_buffer_size(),
VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
fabric:all_docs(Db, Options, fun couch_mrview_http:view_cb/2, VAcc0, Args)
end),
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/50694278/src/chttpd_view.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_view.erl b/src/chttpd_view.erl
index 009aa8f..bb98fec 100644
--- a/src/chttpd_view.erl
+++ b/src/chttpd_view.erl
@@ -53,7 +53,7 @@ design_doc_view(Req, Db, DDoc, ViewName, Keys) ->
end,
Args = Args0#mrargs{preflight_fun=ETagFun},
{ok, Resp} = couch_httpd:etag_maybe(Req, fun() ->
- Max = config:get_integer("httpd", "chunked_response_buffer", 1490),
+ Max = chttpd:chunked_response_buffer_size(),
VAcc0 = #vacc{db=Db, req=Req, threshold=Max},
fabric:query_view(Db, DDoc, ViewName, fun couch_mrview_http:view_cb/2, VAcc0, Args)
end),
[3/7] chttpd commit: updated refs/heads/2724-chunked-buffering to
45bb534
Posted by ko...@apache.org.
Add basic buffering support for other feed types
With this code it is possible that changes are buffered for a long
period of time and not sent out. Will work on addressing that next.
COUCHDB-2724
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/b9261343
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/b9261343
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/b9261343
Branch: refs/heads/2724-chunked-buffering
Commit: b92613432e9278b144176ddf427c389b9cd5d694
Parents: b18805f
Author: Adam Kocoloski <ad...@cloudant.com>
Authored: Wed Jun 24 13:47:01 2015 -0400
Committer: Adam Kocoloski <ad...@cloudant.com>
Committed: Wed Jul 22 16:57:49 2015 -0400
----------------------------------------------------------------------
src/chttpd_db.erl | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/b9261343/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 3247d83..27e1795 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -126,17 +126,18 @@ changes_callback(start, #cacc{feed = continuous} = Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Acc#cacc.mochi, 200),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, Change}, #cacc{feed = continuous} = Acc) ->
- #cacc{mochi = Resp} = Acc,
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [?JSON_ENCODE(Change) | "\n"]),
- {ok, Acc#cacc{mochi = Resp1}};
+ Data = [?JSON_ENCODE(Change) | "\n"],
+ Len = iolist_size(Data),
+ maybe_flush_changes_feed(Acc, Data, Len);
changes_callback({stop, EndSeq0, Pending}, #cacc{feed = continuous} = Acc) ->
- #cacc{mochi = Resp} = Acc,
+ #cacc{mochi = Resp, buffer = Buf} = Acc,
EndSeq = case is_old_couch(Resp) of true -> 0; false -> EndSeq0 end,
Row = {[
{<<"last_seq">>, EndSeq},
{<<"pending">>, Pending}
]},
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, [?JSON_ENCODE(Row) | "\n"]),
+ Data = [Buf, ?JSON_ENCODE(Row) | "\n"],
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Data),
chttpd:end_delayed_json_response(Resp1);
% callbacks for eventsource feed (newline-delimited eventsource Objects)
@@ -149,23 +150,23 @@ changes_callback(start, #cacc{feed = eventsource} = Acc) ->
{ok, Resp} = chttpd:start_delayed_json_response(Req, 200, Headers),
{ok, Acc#cacc{mochi = Resp, responding = true}};
changes_callback({change, {ChangeProp}=Change}, #cacc{feed = eventsource} = Acc) ->
- #cacc{mochi = Resp} = Acc,
Seq = proplists:get_value(seq, ChangeProp),
Chunk = [
"data: ", ?JSON_ENCODE(Change),
"\n", "id: ", ?JSON_ENCODE(Seq),
"\n\n"
],
- {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
- {ok, Acc#cacc{mochi = Resp1}};
+ Len = iolist_size(Chunk),
+ maybe_flush_changes_feed(Acc, Chunk, Len);
changes_callback(timeout, #cacc{feed = eventsource} = Acc) ->
#cacc{mochi = Resp} = Acc,
Chunk = "event: heartbeat\ndata: \n\n",
{ok, Resp1} = chttpd:send_delayed_chunk(Resp, Chunk),
{ok, {"eventsource", Resp1}};
changes_callback({stop, _EndSeq}, #cacc{feed = eventsource} = Acc) ->
- Resp = Acc#cacc.mochi,
- chttpd:end_delayed_json_response(Resp);
+ #cacc{mochi = Resp, buffer = Buf} = Acc,
+ {ok, Resp1} = chttpd:send_delayed_chunk(Resp, Buf),
+ chttpd:end_delayed_json_response(Resp1);
% callbacks for longpoll and normal (single JSON Object)
changes_callback(start, #cacc{feed = normal} = Acc) ->