You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by ni...@apache.org on 2014/02/25 21:43:00 UTC
[1/3] couchdb commit: updated refs/heads/1956-attachment-handling to
9bda4f6
Repository: couchdb
Updated Branches:
refs/heads/1956-attachment-handling [created] 9bda4f667
Enable posting of multipart/related documents
Add case to db_req POST processing for multipart/related documents,
modelled on the PUT equivalent.
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/58989257
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/58989257
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/58989257
Branch: refs/heads/1956-attachment-handling
Commit: 5898925752306e6786dc07df1801342fd3319fce
Parents: b4b6fe1
Author: NickNorth <No...@gmail.com>
Authored: Tue Feb 25 19:51:59 2014 +0000
Committer: NickNorth <No...@gmail.com>
Committed: Tue Feb 25 20:40:09 2014 +0000
----------------------------------------------------------------------
src/couchdb/couch_httpd_db.erl | 42 +++++++++++++++++++++++++++----------
1 file changed, 31 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/58989257/src/couchdb/couch_httpd_db.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl
index 0a7c17c..6a78b13 100644
--- a/src/couchdb/couch_httpd_db.erl
+++ b/src/couchdb/couch_httpd_db.erl
@@ -244,17 +244,29 @@ db_req(#httpd{method='GET',path_parts=[_DbName]}=Req, Db) ->
send_json(Req, {DbInfo});
db_req(#httpd{method='POST',path_parts=[_DbName]}=Req, Db) ->
- couch_httpd:validate_ctype(Req, "application/json"),
- Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)),
- validate_attachment_names(Doc),
- Doc2 = case Doc#doc.id of
- <<"">> ->
- Doc#doc{id=couch_uuids:new(), revs={0, []}};
- _ ->
- Doc
- end,
- DocId = Doc2#doc.id,
- update_doc(Req, Db, DocId, Doc2);
+ case couch_util:to_list(couch_httpd:header_value(Req, "Content-Type")) of
+ ("application/json" ++ _) ->
+ Doc = couch_doc:from_json_obj(couch_httpd:json_body(Req)),
+ validate_attachment_names(Doc),
+ Doc2 = maybe_add_docid(Doc),
+ DocId = Doc2#doc.id,
+ update_doc(Req, Db, DocId, Doc2);
+ ("multipart/related;" ++ _) = ContentType ->
+ {ok, Doc0, WaitFun, Parser} = couch_doc:doc_from_multi_part_stream(
+ ContentType, fun() -> receive_request_data(Req) end),
+ validate_attachment_names(Doc0),
+ Doc2 = maybe_add_docid(Doc0),
+ DocId = Doc2#doc.id,
+ try
+ Result = update_doc(Req, Db, DocId, Doc2),
+ WaitFun(),
+ Result
+ catch throw:Err ->
+ % Document rejected by a validate_doc_update function.
+ couch_doc:abort_multi_part_stream(Parser),
+ throw(Err)
+ end
+ end;
db_req(#httpd{path_parts=[_DbName]}=Req, _Db) ->
send_method_not_allowed(Req, "DELETE,GET,HEAD,POST");
@@ -762,6 +774,14 @@ update_doc(Req, Db, DocId, #doc{deleted=Deleted}=Doc, Headers, UpdateType) ->
{rev, NewRevStr}]})
end.
+maybe_add_docid(Doc) ->
+ case Doc#doc.id of
+ <<"">> ->
+ Doc#doc{id=couch_uuids:new(), revs={0, []}};
+ _ ->
+ Doc
+ end.
+
couch_doc_from_req(Req, DocId, #doc{revs=Revs}=Doc) ->
validate_attachment_names(Doc),
Rev = case couch_httpd:qs_value(Req, "rev") of
[3/3] couchdb commit: updated refs/heads/1956-attachment-handling to
9bda4f6
Posted by ni...@apache.org.
Attachment posting and length tests
Test posting of multipart/related documents.
Test attachments with and without length specified.
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/9bda4f66
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/9bda4f66
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/9bda4f66
Branch: refs/heads/1956-attachment-handling
Commit: 9bda4f6675f3508b390a2e469e981dc774cfe350
Parents: f632df7
Author: NickNorth <No...@gmail.com>
Authored: Tue Feb 25 20:37:02 2014 +0000
Committer: NickNorth <No...@gmail.com>
Committed: Tue Feb 25 20:40:10 2014 +0000
----------------------------------------------------------------------
share/www/script/test/attachments_multipart.js | 204 +++++++++++---------
1 file changed, 109 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/9bda4f66/share/www/script/test/attachments_multipart.js
----------------------------------------------------------------------
diff --git a/share/www/script/test/attachments_multipart.js b/share/www/script/test/attachments_multipart.js
index 6f924a7..53c9d97 100644
--- a/share/www/script/test/attachments_multipart.js
+++ b/share/www/script/test/attachments_multipart.js
@@ -17,105 +17,119 @@ couchTests.attachments_multipart= function(debug) {
if (debug) debugger;
// mime multipart
-
- var xhr = CouchDB.request("PUT", "/test_suite_db/multipart", {
- headers: {"Content-Type": "multipart/related;boundary=\"abc123\""},
- body:
- "--abc123\r\n" +
- "content-type: application/json\r\n" +
- "\r\n" +
- JSON.stringify({
- "body":"This is a body.",
- "_attachments":{
- "foo.txt": {
- "follows":true,
- "content_type":"application/test",
- "length":21
- },
- "bar.txt": {
- "follows":true,
- "content_type":"application/test",
- "length":20
- },
- "baz.txt": {
- "follows":true,
- "content_type":"text/plain",
- "length":19
- }
+ function testMime(docName, includeLength) {
+ var body = {
+ "body":"This is a body.",
+ "_attachments":{
+ "foo.txt": {
+ "follows":true,
+ "content_type":"application/test"
+ },
+ "bar.txt": {
+ "follows":true,
+ "content_type":"application/test"
+ },
+ "baz.txt": {
+ "follows":true,
+ "content_type":"text/plain"
}
- }) +
- "\r\n--abc123\r\n" +
- "\r\n" +
- "this is 21 chars long" +
- "\r\n--abc123\r\n" +
- "\r\n" +
- "this is 20 chars lon" +
- "\r\n--abc123\r\n" +
- "\r\n" +
- "this is 19 chars lo" +
- "\r\n--abc123--epilogue"
- });
+ }
+ };
+ if (includeLength) {
+ body._attachments["foo.txt"].length = 21;
+ body._attachments["bar.txt"].length = 20;
+ body._attachments["baz.txt"].length = 19;
+ }
+ var method = (docName == "") ? "POST" : "PUT";
+ var xhr = CouchDB.request(method, "/test_suite_db/" + docName, {
+ headers: {"Content-Type": "multipart/related;boundary=\"abc123\""},
+ body:
+ "--abc123\r\n" +
+ "content-type: application/json\r\n" +
+ "\r\n" +
+ JSON.stringify(body) +
+ "\r\n--abc123\r\n" +
+ "\r\n" +
+ "this is 21 chars long" +
+ "\r\n--abc123\r\n" +
+ "\r\n" +
+ "this is 20 chars lon" +
+ "\r\n--abc123\r\n" +
+ "\r\n" +
+ "this is 19 chars lo" +
+ "\r\n--abc123--epilogue"
+ });
+
+ var result = JSON.parse(xhr.responseText);
- var result = JSON.parse(xhr.responseText);
-
- T(result.ok);
-
-
+ T(result.ok);
- TEquals(201, xhr.status, "should send 201 Accepted");
-
- xhr = CouchDB.request("GET", "/test_suite_db/multipart/foo.txt");
-
- T(xhr.responseText == "this is 21 chars long");
-
- xhr = CouchDB.request("GET", "/test_suite_db/multipart/bar.txt");
-
- T(xhr.responseText == "this is 20 chars lon");
-
- xhr = CouchDB.request("GET", "/test_suite_db/multipart/baz.txt");
-
- T(xhr.responseText == "this is 19 chars lo");
-
- // now edit an attachment
-
- var doc = db.open("multipart", {att_encoding_info: true});
- var firstrev = doc._rev;
-
- T(doc._attachments["foo.txt"].stub == true);
- T(doc._attachments["bar.txt"].stub == true);
- T(doc._attachments["baz.txt"].stub == true);
- TEquals("undefined", typeof doc._attachments["foo.txt"].encoding);
- TEquals("undefined", typeof doc._attachments["bar.txt"].encoding);
- TEquals("gzip", doc._attachments["baz.txt"].encoding);
-
- //lets change attachment bar
- delete doc._attachments["bar.txt"].stub; // remove stub member (or could set to false)
- delete doc._attachments["bar.txt"].digest; // remove the digest (it's for the gzip form)
- doc._attachments["bar.txt"].length = 18;
- doc._attachments["bar.txt"].follows = true;
- //lets delete attachment baz:
- delete doc._attachments["baz.txt"];
-
- var xhr = CouchDB.request("PUT", "/test_suite_db/multipart", {
- headers: {"Content-Type": "multipart/related;boundary=\"abc123\""},
- body:
- "--abc123\r\n" +
- "content-type: application/json\r\n" +
- "\r\n" +
- JSON.stringify(doc) +
- "\r\n--abc123\r\n" +
- "\r\n" +
- "this is 18 chars l" +
- "\r\n--abc123--"
- });
- TEquals(201, xhr.status);
-
- xhr = CouchDB.request("GET", "/test_suite_db/multipart/bar.txt");
-
- T(xhr.responseText == "this is 18 chars l");
+ docName = result.id;
+
+ TEquals(201, xhr.status, "should send 201 Accepted");
+
+ xhr = CouchDB.request("GET", "/test_suite_db/" + docName + "/foo.txt");
+
+ T(xhr.responseText == "this is 21 chars long");
+
+ xhr = CouchDB.request("GET", "/test_suite_db/" + docName + "/bar.txt");
+
+ T(xhr.responseText == "this is 20 chars lon");
+
+ xhr = CouchDB.request("GET", "/test_suite_db/" + docName + "/baz.txt");
+
+ T(xhr.responseText == "this is 19 chars lo");
+
+ // now edit an attachment
+
+ var doc = db.open(docName, {att_encoding_info: true});
+ var firstrev = doc._rev;
+
+ T(doc._attachments["foo.txt"].stub == true);
+ T(doc._attachments["bar.txt"].stub == true);
+ T(doc._attachments["baz.txt"].stub == true);
+ TEquals("undefined", typeof doc._attachments["foo.txt"].encoding);
+ TEquals("undefined", typeof doc._attachments["bar.txt"].encoding);
+ TEquals("gzip", doc._attachments["baz.txt"].encoding);
+
+ //lets change attachment bar
+ delete doc._attachments["bar.txt"].stub; // remove stub member (or could set to false)
+ delete doc._attachments["bar.txt"].digest; // remove the digest (it's for the gzip form)
+ if (includeLength) {
+ doc._attachments["bar.txt"].length = 18;
+ }
+ doc._attachments["bar.txt"].follows = true;
+ //lets delete attachment baz:
+ delete doc._attachments["baz.txt"];
+
+ var xhr = CouchDB.request("PUT", "/test_suite_db/" + docName, {
+ headers: {"Content-Type": "multipart/related;boundary=\"abc123\""},
+ body:
+ "--abc123\r\n" +
+ "content-type: application/json\r\n" +
+ "\r\n" +
+ JSON.stringify(doc) +
+ "\r\n--abc123\r\n" +
+ "\r\n" +
+ "this is 18 chars l" +
+ "\r\n--abc123--"
+ });
+ TEquals(201, xhr.status);
+
+ xhr = CouchDB.request("GET", "/test_suite_db/" + docName + "/bar.txt");
+
+ T(xhr.responseText == "this is 18 chars l");
+
+ xhr = CouchDB.request("GET", "/test_suite_db/" + docName + "/baz.txt");
+ T(xhr.status == 404);
+ return firstrev;
+ }
- xhr = CouchDB.request("GET", "/test_suite_db/multipart/baz.txt");
- T(xhr.status == 404);
+ // PUT then POST multipart doc with and without attachment lengths
+ var firstrev = testMime("multipart", true);
+ testMime("multipartNoLength", false);
+ testMime("", true);
+ testMime("", false);
// now test receiving multipart docs
[2/3] couchdb commit: updated refs/heads/1956-attachment-handling to
9bda4f6
Posted by ni...@apache.org.
Allow chunked transfer for attachments, and make length optional
Use couch_httpd:recv_chunked to read request data. Spawn a new
process for it, to force waiting between chunks.
Change couch_db:flush_att/2 to write until a body_end token when using
recv_chunked, making attachment length unnecessary.
Simplify couch_db:write_streamed_attachment/3 to remove read_next_chunk
as function arity cannot now be zero.
Tweaks to couch_doc to cope with body_end token.
Project: http://git-wip-us.apache.org/repos/asf/couchdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb/commit/f632df7a
Tree: http://git-wip-us.apache.org/repos/asf/couchdb/tree/f632df7a
Diff: http://git-wip-us.apache.org/repos/asf/couchdb/diff/f632df7a
Branch: refs/heads/1956-attachment-handling
Commit: f632df7a80fe0bf496b80b82e31b73039856d17d
Parents: 5898925
Author: NickNorth <No...@gmail.com>
Authored: Tue Feb 25 20:28:32 2014 +0000
Committer: NickNorth <No...@gmail.com>
Committed: Tue Feb 25 20:40:10 2014 +0000
----------------------------------------------------------------------
src/couchdb/couch_db.erl | 24 ++++++++++++++++--------
src/couchdb/couch_doc.erl | 5 ++++-
src/couchdb/couch_httpd_db.erl | 29 +++++++++++++++++++++--------
3 files changed, 41 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb/blob/f632df7a/src/couchdb/couch_db.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_db.erl b/src/couchdb/couch_db.erl
index 11ea0fd..3bc30ca 100644
--- a/src/couchdb/couch_db.erl
+++ b/src/couchdb/couch_db.erl
@@ -927,7 +927,12 @@ flush_att(Fd, #att{data=Data}=Att) when is_binary(Data) ->
couch_stream:write(OutputStream, Data)
end);
-flush_att(Fd, #att{data=Fun,att_len=undefined}=Att) when is_function(Fun) ->
+flush_att(Fd, #att{data=Fun}=Att) when is_function(Fun, 0) ->
+ with_stream(Fd, Att, fun(OutputStream) ->
+ write_to_body_end(OutputStream, Fun)
+ end);
+
+flush_att(Fd, #att{data=Fun,att_len=undefined}=Att) when is_function(Fun, 3) ->
MaxChunkSize = list_to_integer(
couch_config:get("couchdb", "attachment_stream_buffer_size", "4096")),
with_stream(Fd, Att, fun(OutputStream) ->
@@ -951,7 +956,7 @@ flush_att(Fd, #att{data=Fun,att_len=undefined}=Att) when is_function(Fun) ->
end, ok)
end);
-flush_att(Fd, #att{data=Fun,att_len=AttLen}=Att) when is_function(Fun) ->
+flush_att(Fd, #att{data=Fun,att_len=AttLen}=Att) when is_function(Fun, 1) ->
with_stream(Fd, Att, fun(OutputStream) ->
write_streamed_attachment(OutputStream, Fun, AttLen)
end).
@@ -1038,19 +1043,22 @@ with_stream(Fd, #att{md5=InMd5,type=Type,encoding=Enc}=Att, Fun) ->
encoding=NewEnc
}.
+write_to_body_end(Stream, F) ->
+ case F() of
+ {body_end, _} ->
+ ok;
+ {bytes, Bin} ->
+ ok = couch_stream:write(Stream, Bin),
+ write_to_body_end(Stream, F)
+ end.
write_streamed_attachment(_Stream, _F, 0) ->
ok;
write_streamed_attachment(Stream, F, LenLeft) when LenLeft > 0 ->
- Bin = read_next_chunk(F, LenLeft),
+ Bin = F(lists:min([LenLeft, 16#2000])),
ok = couch_stream:write(Stream, Bin),
write_streamed_attachment(Stream, F, LenLeft - size(Bin)).
-read_next_chunk(F, _) when is_function(F, 0) ->
- F();
-read_next_chunk(F, LenLeft) when is_function(F, 1) ->
- F(lists:min([LenLeft, 16#2000])).
-
enum_docs_since_reduce_to_count(Reds) ->
couch_btree:final_reduce(
fun couch_db_updater:btree_by_seq_reduce/2, Reds).
http://git-wip-us.apache.org/repos/asf/couchdb/blob/f632df7a/src/couchdb/couch_doc.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_doc.erl b/src/couchdb/couch_doc.erl
index 4047370..234f494 100644
--- a/src/couchdb/couch_doc.erl
+++ b/src/couchdb/couch_doc.erl
@@ -581,7 +581,7 @@ doc_from_multi_part_stream(ContentType, DataFun) ->
% replace with function that reads the data from MIME stream.
ReadAttachmentDataFun = fun() ->
Parser ! {get_bytes, Ref, self()},
- receive {bytes, Ref, Bytes} -> Bytes end
+ receive {Kind, Ref, Bytes} -> {Kind, Bytes} end
end,
Atts2 = lists:map(
fun(#att{data=follows}=A) ->
@@ -623,6 +623,9 @@ mp_parse_atts({body, Bytes}) ->
end,
fun mp_parse_atts/1;
mp_parse_atts(body_end) ->
+ receive {get_bytes, Ref, From} ->
+ From ! {body_end, Ref, <<>>}
+ end,
fun mp_parse_atts/1.
http://git-wip-us.apache.org/repos/asf/couchdb/blob/f632df7a/src/couchdb/couch_httpd_db.erl
----------------------------------------------------------------------
diff --git a/src/couchdb/couch_httpd_db.erl b/src/couchdb/couch_httpd_db.erl
index 6a78b13..6067d74 100644
--- a/src/couchdb/couch_httpd_db.erl
+++ b/src/couchdb/couch_httpd_db.erl
@@ -699,14 +699,27 @@ send_ranges_multipart(Req, ContentType, Len, Att, Ranges) ->
{ok, Resp}.
receive_request_data(Req) ->
- receive_request_data(Req, couch_httpd:body_length(Req)).
-
-receive_request_data(Req, LenLeft) when LenLeft > 0 ->
- Len = erlang:min(4096, LenLeft),
- Data = couch_httpd:recv(Req, Len),
- {Data, fun() -> receive_request_data(Req, LenLeft - iolist_size(Data)) end};
-receive_request_data(_Req, _) ->
- throw(<<"expected more data">>).
+ Parent = self(),
+ Ref = make_ref(),
+ Receiver = spawn_link(fun() ->
+ couch_httpd:recv_chunked(Req, 4096, fun
+ ({_Len, Data}, _State) ->
+ Parent ! {chunked_bytes, Ref, Data},
+ receive
+ ok ->
+ null
+ end
+ end, null),
+ unlink(Parent)
+ end),
+ receive_stream_data(Receiver, Ref).
+
+receive_stream_data(Receiver, Ref) ->
+ receive
+ {chunked_bytes, Ref, Data} ->
+ Receiver ! ok,
+ {Data, fun() -> receive_stream_data(Receiver, Ref) end}
+ end.
make_content_range(From, To, Len) ->
io_lib:format("bytes ~B-~B/~B", [From, To, Len]).