You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by jc...@apache.org on 2009/02/13 04:55:37 UTC
svn commit: r743971 - in /couchdb/trunk: etc/couchdb/default.ini.tpl.in
src/couchdb/couch_db.erl src/couchdb/couch_httpd.erl
src/couchdb/couch_httpd_db.erl src/couchdb/couch_stream.erl
src/mochiweb/mochiweb_request.erl
Author: jchris
Date: Fri Feb 13 03:55:36 2009
New Revision: 743971
URL: http://svn.apache.org/viewvc?rev=743971&view=rev
Log:
Accept standalone attachment PUTs with Transfer-Encoding: chunked
Includes a patch to mochiweb_request that is under review for inclusion upstream.
Modified:
couchdb/trunk/etc/couchdb/default.ini.tpl.in
couchdb/trunk/src/couchdb/couch_db.erl
couchdb/trunk/src/couchdb/couch_httpd.erl
couchdb/trunk/src/couchdb/couch_httpd_db.erl
couchdb/trunk/src/couchdb/couch_stream.erl
couchdb/trunk/src/mochiweb/mochiweb_request.erl
Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
+++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Fri Feb 13 03:55:36 2009
@@ -7,6 +7,7 @@
view_index_dir = %localstatelibdir%
util_driver_dir = %couchprivlibdir%
max_document_size = 4294967296 ; 4 GB
+max_attachment_chunk_size = 4294967296 ; 4GB
view_timeout = 5000 ; 5 seconds
max_dbs_open = 100
Modified: couchdb/trunk/src/couchdb/couch_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_db.erl?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_db.erl Fri Feb 13 03:55:36 2009
@@ -24,7 +24,6 @@
-export([start_link/3,make_doc/2,set_admins/2,get_admins/1,ensure_full_commit/1]).
-export([init/1,terminate/2,handle_call/3,handle_cast/2,code_change/3,handle_info/2]).
-
-include("couch_db.hrl").
@@ -400,8 +399,14 @@
% written to a different file
SizeAcc + Len;
{_Key, {_Type, Bin}} when is_binary(Bin) ->
+ % we have a new binary to write
SizeAcc + size(Bin);
+ {_Key, {_Type, {Fun, undefined}}} when is_function(Fun) ->
+ % function without a known length
+ % we'll have to alloc as we go with this one, for now, nothing
+ SizeAcc;
{_Key, {_Type, {Fun, Len}}} when is_function(Fun) ->
+ % function to yield binary data with known length
SizeAcc + Len
end
end,
@@ -409,7 +414,6 @@
{ok, OutputStream} = couch_stream:open(Fd),
ok = couch_stream:ensure_buffer(OutputStream, PreAllocSize),
-
NewBins = lists:map(
fun({Key, {Type, BinValue}}) ->
NewBinValue =
@@ -436,6 +440,18 @@
Bin when is_binary(Bin) ->
{ok, StreamPointer} = couch_stream:write(OutputStream, Bin),
{Fd, StreamPointer, size(Bin)};
+ {StreamFun, undefined} when is_function(StreamFun) ->
+ % we will throw an error if the client
+ % sends a chunk larger than this size
+ MaxChunkSize = list_to_integer(couch_config:get("couchdb",
+ "max_attachment_chunk_size","4294967296")),
+ WriterFun = make_writer_fun(OutputStream),
+ % StreamFun(MaxChunkSize, WriterFun)
+ % will call our WriterFun
+ % once for each chunk of the attachment.
+ {ok, {TotalLength, NewStreamPointer}} =
+ StreamFun(MaxChunkSize, WriterFun, {0, nil}),
+ {Fd, NewStreamPointer, TotalLength};
{Fun, Len} when is_function(Fun) ->
{ok, StreamPointer} =
write_streamed_attachment(OutputStream, Fun, Len, nil),
@@ -445,8 +461,27 @@
end, Bins),
{ok, _FinalPos} = couch_stream:close(OutputStream),
-
Doc#doc{attachments = NewBins}.
+
+
+make_writer_fun(Stream) ->
+ % WriterFun({Length, Binary}, State)
+ % WriterFun({0, _Footers}, State)
+ % Called with Length == 0 on the last time.
+ % WriterFun returns NewState.
+ fun
+ ({0, _Footers}, {FinalLen, SpFin}) ->
+ % last block, return the final tuple
+ {ok, {FinalLen, SpFin}};
+ ({Length, Bin}, {Total, nil}) ->
+ % save StreamPointer
+ {ok, StreamPointer} = couch_stream:write(Stream, Bin),
+ {Total+Length, StreamPointer};
+ ({Length, Bin}, {Total, SpAcc}) ->
+ % write the Bin to disk
+ {ok, _Sp} = couch_stream:write(Stream, Bin),
+ {Total+Length, SpAcc}
+ end.
write_streamed_attachment(_Stream, _F, 0, SpAcc) ->
{ok, SpAcc};
Modified: couchdb/trunk/src/couchdb/couch_httpd.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd.erl?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd.erl Fri Feb 13 03:55:36 2009
@@ -16,7 +16,7 @@
-export([start_link/0, stop/0, handle_request/3]).
-export([header_value/2,header_value/3,qs_value/2,qs_value/3,qs/1,path/1,absolute_uri/2]).
--export([verify_is_server_admin/1,unquote/1,quote/1,recv/2]).
+-export([verify_is_server_admin/1,unquote/1,quote/1,recv/2,recv_chunked/4]).
-export([parse_form/1,json_body/1,body/1,doc_etag/1, make_etag/1, etag_respond/3]).
-export([primary_header_value/2,partition/1,serve_file/3]).
-export([start_chunked_response/3,send_chunk/2]).
@@ -260,6 +260,12 @@
recv(#httpd{mochi_req=MochiReq}, Len) ->
MochiReq:recv(Len).
+recv_chunked(#httpd{mochi_req=MochiReq}, MaxChunkSize, ChunkFun, InitState) ->
+ % Fun is called once with each chunk
+ % Fun({Length, Binary}, State)
+ % called with Length == 0 on the last time.
+ MochiReq:recv_body(MaxChunkSize, ChunkFun, InitState).
+
body(#httpd{mochi_req=MochiReq}) ->
% Maximum size of document PUT request body (4GB)
MaxSize = list_to_integer(
Modified: couchdb/trunk/src/couchdb/couch_httpd_db.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_httpd_db.erl?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_httpd_db.erl (original)
+++ couchdb/trunk/src/couchdb/couch_httpd_db.erl Fri Feb 13 03:55:36 2009
@@ -585,18 +585,31 @@
db_attachment_req(#httpd{method=Method}=Req, Db, DocId, FileNameParts)
when (Method == 'PUT') or (Method == 'DELETE') ->
- FileName = list_to_binary(mochiweb_util:join(lists:map(fun binary_to_list/1, FileNameParts),"/")),
+ FileName = list_to_binary(mochiweb_util:join(lists:map(fun binary_to_list/1,
+ FileNameParts),"/")),
NewAttachment = case Method of
'DELETE' ->
[];
_ ->
+ % see couch_db:doc_flush_binaries for usage of this structure
[{FileName, {
- list_to_binary(couch_httpd:header_value(Req,"Content-Type")),
+ case couch_httpd:header_value(Req,"Content-Type") of
+ undefined ->
+ % We could throw an error here or guess by the FileName.
+ % Currently, just giving it a default.
+ <<"application/octet-stream">>;
+ CType ->
+ list_to_binary(CType)
+ end,
case couch_httpd:header_value(Req,"Content-Length") of
undefined ->
- throw({bad_request, "Attachment uploads must be fixed length"});
+ {fun(MaxChunkSize, ChunkFun, InitState) ->
+ couch_httpd:recv_chunked(Req, MaxChunkSize,
+ ChunkFun, InitState)
+ end, undefined};
Length ->
- {fun() -> couch_httpd:recv(Req, 0) end, list_to_integer(Length)}
+ {fun() -> couch_httpd:recv(Req, 0) end,
+ list_to_integer(Length)}
end
}}]
end,
Modified: couchdb/trunk/src/couchdb/couch_stream.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/couchdb/couch_stream.erl?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/src/couchdb/couch_stream.erl (original)
+++ couchdb/trunk/src/couchdb/couch_stream.erl Fri Feb 13 03:55:36 2009
@@ -136,6 +136,7 @@
{reply, {Pos, BytesRemaining}, Stream};
handle_call({set_min_buffer, MinBuffer}, _From, Stream) ->
{reply, ok, Stream#write_stream{min_alloc = MinBuffer}};
+% set next_alloc if we need more room
handle_call({ensure_buffer, BufferSizeRequested}, _From, Stream) ->
#write_stream{bytes_remaining = BytesRemainingInCurrentBuffer} = Stream,
case BytesRemainingInCurrentBuffer < BufferSizeRequested of
Modified: couchdb/trunk/src/mochiweb/mochiweb_request.erl
URL: http://svn.apache.org/viewvc/couchdb/trunk/src/mochiweb/mochiweb_request.erl?rev=743971&r1=743970&r2=743971&view=diff
==============================================================================
--- couchdb/trunk/src/mochiweb/mochiweb_request.erl (original)
+++ couchdb/trunk/src/mochiweb/mochiweb_request.erl Fri Feb 13 03:55:36 2009
@@ -12,7 +12,7 @@
-define(READ_SIZE, 8192).
-export([get_header_value/1, get_primary_header_value/1, get/1, dump/0]).
--export([send/1, recv/1, recv/2, recv_body/0, recv_body/1]).
+-export([send/1, recv/1, recv/2, recv_body/0, recv_body/1, recv_body/3]).
-export([start_response/1, start_response_length/1, start_raw_response/1]).
-export([respond/1, ok/1]).
-export([not_found/0, not_found/1]).
@@ -171,6 +171,9 @@
%% @doc Receive the body of the HTTP request (defined by Content-Length).
%% Will receive up to MaxBody bytes.
recv_body(MaxBody) ->
+ recv_body(MaxBody, nil, nil).
+
+recv_body(MaxBody, ChunkFun, ChunkAcc) ->
case get_header_value("expect") of
"100-continue" ->
start_raw_response({100, gb_trees:empty()});
@@ -183,7 +186,15 @@
{unknown_transfer_encoding, Unknown} ->
exit({unknown_transfer_encoding, Unknown});
chunked ->
- read_chunked_body(MaxBody, []);
+ case ChunkFun of
+ nil ->
+ read_chunked_body(MaxBody);
+ _StreamFun ->
+ % In this case the MaxBody is actually used to
+ % determine the maximum allowed size of a single
+ % chunk.
+ stream_chunked_body(MaxBody, ChunkFun, ChunkAcc)
+ end;
0 ->
<<>>;
Length when is_integer(Length), Length =< MaxBody ->
@@ -408,15 +419,26 @@
Cached
end.
-read_chunked_body(Max, Acc) ->
+read_chunked_body(MaxBufferSize) ->
+ stream_chunked_body(MaxBufferSize, fun
+ ({0, _}, Acc) ->
+ iolist_to_binary(lists:reverse(Acc));
+ ({_Length, Bin}, Acc) ->
+ [Bin | Acc]
+ end, []).
+
+%% @spec stream_chunked_body(integer(), fun(), term()) -> term()
+%% @doc The function is called for each chunk.
+%% Used internally by read_chunked_body.
+stream_chunked_body(MaxChunkSize, Fun, FunState) ->
case read_chunk_length() of
0 ->
- read_chunk(0),
- iolist_to_binary(lists:reverse(Acc));
- Length when Length > Max ->
+ Fun({0, read_chunk(0)}, FunState);
+ Length when Length > MaxChunkSize ->
exit({body_too_large, chunked});
Length ->
- read_chunked_body(Max - Length, [read_chunk(Length) | Acc])
+ NewState = Fun({Length, read_chunk(Length)}, FunState),
+ stream_chunked_body(MaxChunkSize, Fun, NewState)
end.
%% @spec read_chunk_length() -> integer()
Re: svn commit: r743971 - in /couchdb/trunk: etc/couchdb/default.ini.tpl.in
src/couchdb/couch_db.erl src/couchdb/couch_httpd.erl src/couchdb/couch_httpd_db.erl
src/couchdb/couch_stream.erl src/mochiweb/mochiweb_request.erl
Posted by Chris Anderson <jc...@apache.org>.
On Thu, Feb 12, 2009 at 7:55 PM, <jc...@apache.org> wrote:
> Author: jchris
> Date: Fri Feb 13 03:55:36 2009
> New Revision: 743971
>
> URL: http://svn.apache.org/viewvc?rev=743971&view=rev
> Log:
> Accept standalone attachment PUTs with Transfer-Encoding: chunked
> Includes a patch to mochiweb_request that is under review for inclusion upstream.
>
> Modified: couchdb/trunk/etc/couchdb/default.ini.tpl.in
> URL: http://svn.apache.org/viewvc/couchdb/trunk/etc/couchdb/default.ini.tpl.in?rev=743971&r1=743970&r2=743971&view=diff
> ==============================================================================
> --- couchdb/trunk/etc/couchdb/default.ini.tpl.in (original)
> +++ couchdb/trunk/etc/couchdb/default.ini.tpl.in Fri Feb 13 03:55:36 2009
> @@ -7,6 +7,7 @@
> view_index_dir = %localstatelibdir%
> util_driver_dir = %couchprivlibdir%
> max_document_size = 4294967296 ; 4 GB
> +max_attachment_chunk_size = 4294967296 ; 4GB
> view_timeout = 5000 ; 5 seconds
> max_dbs_open = 100
>
I don't know if there's a better value for max_attachment_chunk_size.
The effect it has would be to throw an error to the client if it tries
to send a chunk bigger than the threshold. Setting it smaller prevents
clients from blowing up CouchDB's memory usage by sending huge chunks.
If anyone has a strong rationale for why it should be a particular
value, please reply.
Chris
--
Chris Anderson
http://jchris.mfdz.com