You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2016/04/17 23:42:59 UTC
chttpd commit: updated refs/heads/2992-limit-doc-size to f7affd0
[Forced Update!]
Repository: couchdb-chttpd
Updated Branches:
refs/heads/2992-limit-doc-size 5181ec865 -> f7affd0d2 (forced update)
Add support for single_max_doc
This adds support for a new configuration param single_max_doc_size.
Single documents that exceed this document will receive a too_large
error.
COUCHDB-2992
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/f7affd0d
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/f7affd0d
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/f7affd0d
Branch: refs/heads/2992-limit-doc-size
Commit: f7affd0d276e12b29e930666dd36ff7b9fdd8e5d
Parents: be1e959
Author: Tony Sun <to...@cloudant.com>
Authored: Sat Apr 16 19:42:46 2016 -0700
Committer: Tony Sun <to...@cloudant.com>
Committed: Sun Apr 17 14:40:40 2016 -0700
----------------------------------------------------------------------
src/chttpd.erl | 4 ++
src/chttpd_db.erl | 82 +++++++++++++++++++++---------
test/chttpd_db_doc_size_test.erl | 96 +++++++++++++++++++++++++++++++++++
3 files changed, 158 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index ee90e5d..1f8c160 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -858,6 +858,10 @@ error_info({error, {database_name_too_long, DbName}}) ->
<<"At least one path segment of `", DbName/binary, "` is too long.">>};
error_info({missing_stub, Reason}) ->
{412, <<"missing_stub">>, Reason};
+error_info({single_doc_too_large, MaxSize}) ->
+ SizeBin = list_to_binary(integer_to_list(MaxSize)),
+ {413, <<"too_large">>, <<"Document exceeded single_max_doc_size:",
+ SizeBin/binary>>};
error_info(request_entity_too_large) ->
{413, <<"too_large">>, <<"the request entity is too large">>};
error_info({error, security_migration_updates_disabled}) ->
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 9347c1a..2a3fe1b 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -308,7 +308,10 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
Options = [{user_ctx,Ctx}, {w,W}],
- Doc = couch_doc:from_json_obj(chttpd:json_body(Req)),
+ Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
+
+ Doc = couch_doc:from_json_obj(Body),
Doc2 = case Doc#doc.id of
<<"">> ->
Doc#doc{id=couch_uuids:new(), revs={0, []}};
@@ -374,6 +377,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
DocsArray0 ->
DocsArray0
end,
+ {ExceedErrs, DocsArray1} = remove_exceed_docs(DocsArray),
couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)),
W = case couch_util:get_value(<<"w">>, JsonProps) of
Value when is_integer(Value) ->
@@ -401,39 +405,38 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
end,
Doc#doc{id=Id}
end,
- DocsArray),
+ DocsArray1),
Options2 =
case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of
true -> [all_or_nothing|Options];
_ -> Options
end,
- case fabric:update_docs(Db, Docs, Options2) of
- {ok, Results} ->
+ {Code, Results} = case fabric:update_docs(Db, Docs, Options2) of
+ {ok, Results0} ->
% output the results
- DocResults = lists:zipwith(fun update_doc_result_to_json/2,
- Docs, Results),
- send_json(Req, 201, DocResults);
- {accepted, Results} ->
+ {201, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+ Results0)};
+ {accepted, Results0} ->
% output the results
- DocResults = lists:zipwith(fun update_doc_result_to_json/2,
- Docs, Results),
- send_json(Req, 202, DocResults);
+ {202, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+ Results0)};
{aborted, Errors} ->
- ErrorsJson =
- lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 417, ErrorsJson)
- end;
+ {417, lists:map(fun update_doc_result_to_json/1, Errors)}
+ end,
+ FinalResults = lists:keysort(1, ExceedErrs ++ Results),
+ send_json(Req, Code, FinalResults);
false ->
Docs = [couch_doc:from_json_obj(JsonObj) || JsonObj <- DocsArray],
[validate_attachment_names(D) || D <- Docs],
- case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
- {ok, Errors} ->
- ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 201, ErrorsJson);
- {accepted, Errors} ->
- ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 202, ErrorsJson)
- end
+ {Code, Errs1} = case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
+ {ok, Errs0} ->
+ {201, Errs0};
+ {accepted, Errs0} ->
+ {202, Errs0}
+ end,
+ Errs2 = lists:map(fun update_doc_result_to_json/1, Errs1),
+ FinalErrors = lists:keysort(1, ExceedErrs ++ Errs2),
+ send_json(Req, Code, FinalErrors)
end;
db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) ->
@@ -765,7 +768,9 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
case chttpd:qs_value(Req, "batch") of
"ok" ->
% batch
- Doc = couch_doc_from_req(Req, DocId, chttpd:json_body(Req)),
+ Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
+ Doc = couch_doc_from_req(Req, DocId, Body),
spawn(fun() ->
case catch(fabric:update_doc(Db, Doc, Options)) of
@@ -782,6 +787,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
_Normal ->
% normal
Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
Doc = couch_doc_from_req(Req, DocId, Body),
send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType)
end
@@ -1628,6 +1634,34 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) ->
{DocId, Results, Options}
end.
+remove_exceed_docs(DocArray0) ->
+ MaxSize = list_to_integer(config:get("couchdb",
+ "single_max_doc_size", "1048576")),
+ {ExceedDocs, DocsArray} = lists:splitwith(fun (Doc) ->
+ exceed_doc_size(Doc, MaxSize)
+ end, DocArray0),
+ ExceedErrs = lists:map (fun ({Doc}) ->
+ DocId = case couch_util:get_value(<<"_id">>, Doc) of
+ undefined -> couch_uuids:new();
+ Id0 -> Id0
+ end,
+ Reason = lists:concat(["Document exceeded single_max_doc_size:",
+ " of ", MaxSize, " bytes"]),
+ {[{id, DocId}, {error, <<"too_large">>},
+ {reason, ?l2b(Reason)}]}
+ end, ExceedDocs),
+ {ExceedErrs, DocsArray}.
+
+exceed_doc_size(JsonBody, MaxSize) ->
+ size(term_to_binary(JsonBody)) > MaxSize.
+
+verify_doc_size(JsonBody) ->
+ SMaxSize = list_to_integer(config:get("couchdb",
+ "single_max_doc_size", "1048576")),
+ case exceed_doc_size(JsonBody, SMaxSize) of
+ true -> throw({single_doc_too_large, SMaxSize});
+ false -> ok
+ end.
parse_field(<<"id">>, undefined) ->
{ok, undefined};
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/f7affd0d/test/chttpd_db_doc_size_test.erl
----------------------------------------------------------------------
diff --git a/test/chttpd_db_doc_size_test.erl b/test/chttpd_db_doc_size_test.erl
new file mode 100644
index 0000000..02ae3cd
--- /dev/null
+++ b/test/chttpd_db_doc_size_test.erl
@@ -0,0 +1,96 @@
+% Licensed under the Apache License, Version 2.0 (the "License"); you may not
+% use this file except in compliance with the License. You may obtain a copy of
+% the License at
+%
+% http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+% License for the specific language governing permissions and limitations under
+% the License.
+
+-module(chttpd_db_doc_size_test).
+
+-include_lib("couch/include/couch_eunit.hrl").
+-include_lib("couch/include/couch_db.hrl").
+
+-define(USER, "chttpd_db_test_admin").
+-define(PASS, "pass").
+-define(AUTH, {basic_auth, {?USER, ?PASS}}).
+-define(CONTENT_JSON, {"Content-Type", "application/json"}).
+
+setup() ->
+ ok = config:set("admins", ?USER, ?PASS, _Persist=false),
+ ok = config:set("couchdb", "single_max_doc_size", "50"),
+ TmpDb = ?tempdb(),
+ Addr = config:get("chttpd", "bind_address", "127.0.0.1"),
+ Port = mochiweb_socket_server:get(chttpd, port),
+ Url = lists:concat(["http://", Addr, ":", Port, "/", ?b2l(TmpDb)]),
+ create_db(Url),
+ Url.
+
+teardown(Url) ->
+ delete_db(Url),
+ ok = config:delete("admins", ?USER, _Persist=false),
+ ok = config:delete("couchdb", "single_max_doc_size").
+
+create_db(Url) ->
+ {ok, Status, _, _} = test_request:put(Url, [?CONTENT_JSON, ?AUTH], "{}"),
+ ?assert(Status =:= 201 orelse Status =:= 202).
+
+delete_db(Url) ->
+ {ok, 200, _, _} = test_request:delete(Url, [?AUTH]).
+
+all_test_() ->
+ {
+ "chttpd db single doc max size test",
+ {
+ setup,
+ fun chttpd_test_util:start_couch/0, fun chttpd_test_util:stop_couch/1,
+ {
+ foreach,
+ fun setup/0, fun teardown/1,
+ [
+ fun post_single_doc/1,
+ fun put_single_doc/1,
+ fun bulk_doc/1
+ ]
+ }
+ }
+ }.
+
+post_single_doc(Url) ->
+ ?_assertEqual({<<"error">>, <<"too_large">>},
+ begin
+ NewDoc = "{\"post_single_doc\": \"some_doc\",
+ \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}",
+ {ok, _, _, ResultBody} = test_request:post(Url,
+ [?CONTENT_JSON, ?AUTH], NewDoc),
+ {ErrorMsg} = ?JSON_DECODE(ResultBody),
+ lists:nth(1, ErrorMsg)
+ end).
+
+put_single_doc(Url) ->
+ ?_assertEqual({<<"error">>, <<"too_large">>},
+ begin
+ NewDoc = "{\"post_single_doc\": \"some_doc\",
+ \"_id\": \"testdoc\", \"should_be\" : \"too_large\"}",
+ {ok, _, _, ResultBody} = test_request:put(Url ++ "/" ++ "testid",
+ [?CONTENT_JSON, ?AUTH], NewDoc),
+ {ErrorMsg} = ?JSON_DECODE(ResultBody),
+ lists:nth(1, ErrorMsg)
+ end).
+
+bulk_doc(Url) ->
+ NewDoc = "{\"docs\": [{\"doc1\": 1}, {\"errordoc\":
+ \"this_should_be_the_error_document\"}]}",
+ {ok, _, _, ResultBody} = test_request:post(Url ++ "/_bulk_docs/",
+ [?CONTENT_JSON, ?AUTH], NewDoc),
+ ResultJson = ?JSON_DECODE(ResultBody),
+ {InnerJson1} = lists:nth(1, ResultJson),
+ {InnerJson2} = lists:nth(2, ResultJson),
+ Error = couch_util:get_value(<<"error">>, InnerJson1),
+ Msg = couch_util:get_value(<<"error">>, InnerJson2),
+ ?_assertEqual(<<"too_large">>, Error),
+ ?_assertEqual(undefined, Msg).