You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@couchdb.apache.org by to...@apache.org on 2016/04/17 04:44:59 UTC
[1/4] chttpd commit: updated refs/heads/2992-limit-doc-size to
5181ec8 [Forced Update!]
Repository: couchdb-chttpd
Updated Branches:
refs/heads/2992-limit-doc-size acb58c51d -> 5181ec865 (forced update)
Log user name of request initiator
Change the log format from chttpd to include username.
The new format is (space separated):
Nonce, Peer, Host, UserName, Method, RawUri, Code, Status, RequestTime
COUCHDB-2973
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/d2665cec
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/d2665cec
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/d2665cec
Branch: refs/heads/2992-limit-doc-size
Commit: d2665cec222ab447e47cdceff950269553fb6c47
Parents: 02202d3
Author: ILYA Khlopotov <ii...@ca.ibm.com>
Authored: Mon Mar 21 12:23:31 2016 -0700
Committer: ILYA Khlopotov <ii...@ca.ibm.com>
Committed: Fri Apr 15 09:06:37 2016 -0700
----------------------------------------------------------------------
src/chttpd.erl | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/d2665cec/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index 3d2f3d9..e5c6b65 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -371,10 +371,11 @@ maybe_log(#httpd{} = HttpReq, #httpd_resp{should_log = true} = HttpResp) ->
code = Code,
status = Status
} = HttpResp,
+ User = get_user(HttpReq),
Host = MochiReq:get_header_value("Host"),
RawUri = MochiReq:get(raw_path),
RequestTime = timer:now_diff(EndTime, BeginTime) / 1000,
- couch_log:notice("~s ~s ~s ~s ~s ~B ~p ~B", [Nonce, Peer, Host,
+ couch_log:notice("~s ~s ~s ~s ~s ~s ~B ~p ~B", [Nonce, Peer, Host, User,
Method, RawUri, Code, Status, round(RequestTime)]);
maybe_log(_HttpReq, #httpd_resp{should_log = false}) ->
ok.
@@ -1085,6 +1086,10 @@ respond_(#httpd{mochi_req = MochiReq}, Code, Headers, _Args, start_response) ->
respond_(#httpd{mochi_req = MochiReq}, Code, Headers, Args, Type) ->
MochiReq:Type({Code, Headers, Args}).
+get_user(#httpd{user_ctx = #user_ctx{name = User}}) ->
+ couch_util:url_encode(User);
+get_user(#httpd{user_ctx = undefined}) ->
+ "undefined".
-ifdef(TEST).
[2/4] chttpd commit: updated refs/heads/2992-limit-doc-size to 5181ec8
Posted by to...@apache.org.
Add `log_format_test` test case
COUCHDB-2973
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/cd00955b
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/cd00955b
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/cd00955b
Branch: refs/heads/2992-limit-doc-size
Commit: cd00955bbe3f6dcbdd11d78781bb4576f7643f9d
Parents: d2665ce
Author: ILYA Khlopotov <ii...@ca.ibm.com>
Authored: Mon Mar 21 13:06:28 2016 -0700
Committer: ILYA Khlopotov <ii...@ca.ibm.com>
Committed: Fri Apr 15 09:06:46 2016 -0700
----------------------------------------------------------------------
src/chttpd.erl | 42 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 42 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/cd00955b/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index e5c6b65..ee90e5d 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -1145,4 +1145,46 @@ check_url_encoding_fail_test_() ->
check_url_encoding("/dbname%g2"))
].
+log_format_test() ->
+ ?assertEqual(
+ "nonce 127.0.0.1 127.0.0.1:15984 undefined "
+ "GET /_cluster_setup 201 ok 10000",
+ test_log_request("/_cluster_setup", undefined)),
+ ?assertEqual(
+ "nonce 127.0.0.1 127.0.0.1:15984 user_foo "
+ "GET /_all_dbs 201 ok 10000",
+ test_log_request("/_all_dbs", #user_ctx{name = <<"user_foo">>})),
+
+ %% Utf8Name = unicode:characters_to_binary(Something),
+ Utf8User = <<227,130,136,227,129,134,227,129,147,227,129,157>>,
+ ?assertEqual(
+ "nonce 127.0.0.1 127.0.0.1:15984 %E3%82%88%E3%81%86%E3%81%93%E3%81%9D "
+ "GET /_all_dbs 201 ok 10000",
+ test_log_request("/_all_dbs", #user_ctx{name = Utf8User})),
+ ok.
+
+test_log_request(RawPath, UserCtx) ->
+ Headers = mochiweb_headers:make([{"HOST", "127.0.0.1:15984"}]),
+ MochiReq = mochiweb_request:new(socket, [], 'POST', RawPath, version, Headers),
+ Req = #httpd{
+ mochi_req = MochiReq,
+ begin_ts = {1458,588713,124003},
+ original_method = 'GET',
+ peer = "127.0.0.1",
+ nonce = "nonce",
+ user_ctx = UserCtx
+ },
+ Resp = #httpd_resp{
+ end_ts = {1458,588723,124303},
+ code = 201,
+ status = ok
+ },
+ ok = meck:new(couch_log, [passthrough]),
+ ok = meck:expect(couch_log, notice, fun(Format, Args) ->
+ lists:flatten(io_lib:format(Format, Args))
+ end),
+ Message = maybe_log(Req, Resp),
+ ok = meck:unload(couch_log),
+ Message.
+
-endif.
[4/4] chttpd commit: updated refs/heads/2992-limit-doc-size to 5181ec8
Posted by to...@apache.org.
Add support for single_max_doc
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/5181ec86
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/5181ec86
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/5181ec86
Branch: refs/heads/2992-limit-doc-size
Commit: 5181ec865f1c517c54c2a062d7b2cf8eba75a279
Parents: be1e959
Author: Tony Sun <to...@cloudant.com>
Authored: Sat Apr 16 19:42:46 2016 -0700
Committer: Tony Sun <to...@cloudant.com>
Committed: Sat Apr 16 19:45:57 2016 -0700
----------------------------------------------------------------------
src/chttpd.erl | 4 +++
src/chttpd_db.erl | 82 +++++++++++++++++++++++++++++++++++---------------
2 files changed, 62 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/5181ec86/src/chttpd.erl
----------------------------------------------------------------------
diff --git a/src/chttpd.erl b/src/chttpd.erl
index ee90e5d..1f8c160 100644
--- a/src/chttpd.erl
+++ b/src/chttpd.erl
@@ -858,6 +858,10 @@ error_info({error, {database_name_too_long, DbName}}) ->
<<"At least one path segment of `", DbName/binary, "` is too long.">>};
error_info({missing_stub, Reason}) ->
{412, <<"missing_stub">>, Reason};
+error_info({single_doc_too_large, MaxSize}) ->
+ SizeBin = list_to_binary(integer_to_list(MaxSize)),
+ {413, <<"too_large">>, <<"Document exceeded single_max_doc_size:",
+ SizeBin/binary>>};
error_info(request_entity_too_large) ->
{413, <<"too_large">>, <<"the request entity is too large">>};
error_info({error, security_migration_updates_disabled}) ->
http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/blob/5181ec86/src/chttpd_db.erl
----------------------------------------------------------------------
diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl
index 9347c1a..2a3fe1b 100644
--- a/src/chttpd_db.erl
+++ b/src/chttpd_db.erl
@@ -308,7 +308,10 @@ db_req(#httpd{method='POST', path_parts=[DbName], user_ctx=Ctx}=Req, Db) ->
W = chttpd:qs_value(Req, "w", integer_to_list(mem3:quorum(Db))),
Options = [{user_ctx,Ctx}, {w,W}],
- Doc = couch_doc:from_json_obj(chttpd:json_body(Req)),
+ Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
+
+ Doc = couch_doc:from_json_obj(Body),
Doc2 = case Doc#doc.id of
<<"">> ->
Doc#doc{id=couch_uuids:new(), revs={0, []}};
@@ -374,6 +377,7 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
DocsArray0 ->
DocsArray0
end,
+ {ExceedErrs, DocsArray1} = remove_exceed_docs(DocsArray),
couch_stats:update_histogram([couchdb, httpd, bulk_docs], length(DocsArray)),
W = case couch_util:get_value(<<"w">>, JsonProps) of
Value when is_integer(Value) ->
@@ -401,39 +405,38 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req,
end,
Doc#doc{id=Id}
end,
- DocsArray),
+ DocsArray1),
Options2 =
case couch_util:get_value(<<"all_or_nothing">>, JsonProps) of
true -> [all_or_nothing|Options];
_ -> Options
end,
- case fabric:update_docs(Db, Docs, Options2) of
- {ok, Results} ->
+ {Code, Results} = case fabric:update_docs(Db, Docs, Options2) of
+ {ok, Results0} ->
% output the results
- DocResults = lists:zipwith(fun update_doc_result_to_json/2,
- Docs, Results),
- send_json(Req, 201, DocResults);
- {accepted, Results} ->
+ {201, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+ Results0)};
+ {accepted, Results0} ->
% output the results
- DocResults = lists:zipwith(fun update_doc_result_to_json/2,
- Docs, Results),
- send_json(Req, 202, DocResults);
+ {202, lists:zipwith(fun update_doc_result_to_json/2, Docs,
+ Results0)};
{aborted, Errors} ->
- ErrorsJson =
- lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 417, ErrorsJson)
- end;
+ {417, lists:map(fun update_doc_result_to_json/1, Errors)}
+ end,
+ FinalResults = lists:keysort(1, ExceedErrs ++ Results),
+ send_json(Req, Code, FinalResults);
false ->
Docs = [couch_doc:from_json_obj(JsonObj) || JsonObj <- DocsArray],
[validate_attachment_names(D) || D <- Docs],
- case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
- {ok, Errors} ->
- ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 201, ErrorsJson);
- {accepted, Errors} ->
- ErrorsJson = lists:map(fun update_doc_result_to_json/1, Errors),
- send_json(Req, 202, ErrorsJson)
- end
+ {Code, Errs1} = case fabric:update_docs(Db, Docs, [replicated_changes|Options]) of
+ {ok, Errs0} ->
+ {201, Errs0};
+ {accepted, Errs0} ->
+ {202, Errs0}
+ end,
+ Errs2 = lists:map(fun update_doc_result_to_json/1, Errs1),
+ FinalErrors = lists:keysort(1, ExceedErrs ++ Errs2),
+ send_json(Req, Code, FinalErrors)
end;
db_req(#httpd{path_parts=[_,<<"_bulk_docs">>]}=Req, _Db) ->
@@ -765,7 +768,9 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
case chttpd:qs_value(Req, "batch") of
"ok" ->
% batch
- Doc = couch_doc_from_req(Req, DocId, chttpd:json_body(Req)),
+ Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
+ Doc = couch_doc_from_req(Req, DocId, Body),
spawn(fun() ->
case catch(fabric:update_doc(Db, Doc, Options)) of
@@ -782,6 +787,7 @@ db_doc_req(#httpd{method='PUT', user_ctx=Ctx}=Req, Db, DocId) ->
_Normal ->
% normal
Body = chttpd:json_body(Req),
+ ok = verify_doc_size(Body),
Doc = couch_doc_from_req(Req, DocId, Body),
send_updated_doc(Req, Db, DocId, Doc, RespHeaders, UpdateType)
end
@@ -1628,6 +1634,34 @@ bulk_get_open_doc_revs1(Db, Props, _, {DocId, Revs, Options}) ->
{DocId, Results, Options}
end.
+remove_exceed_docs(DocArray0) ->
+ MaxSize = list_to_integer(config:get("couchdb",
+ "single_max_doc_size", "1048576")),
+ {ExceedDocs, DocsArray} = lists:splitwith(fun (Doc) ->
+ exceed_doc_size(Doc, MaxSize)
+ end, DocArray0),
+ ExceedErrs = lists:map (fun ({Doc}) ->
+ DocId = case couch_util:get_value(<<"_id">>, Doc) of
+ undefined -> couch_uuids:new();
+ Id0 -> Id0
+ end,
+ Reason = lists:concat(["Document exceeded single_max_doc_size:",
+ " of ", MaxSize, " bytes"]),
+ {[{id, DocId}, {error, <<"too_large">>},
+ {reason, ?l2b(Reason)}]}
+ end, ExceedDocs),
+ {ExceedErrs, DocsArray}.
+
+exceed_doc_size(JsonBody, MaxSize) ->
+ size(term_to_binary(JsonBody)) > MaxSize.
+
+verify_doc_size(JsonBody) ->
+ SMaxSize = list_to_integer(config:get("couchdb",
+ "single_max_doc_size", "1048576")),
+ case exceed_doc_size(JsonBody, SMaxSize) of
+ true -> throw({single_doc_too_large, SMaxSize});
+ false -> ok
+ end.
parse_field(<<"id">>, undefined) ->
{ok, undefined};
[3/4] chttpd commit: updated refs/heads/2992-limit-doc-size to 5181ec8
Posted by to...@apache.org.
Merge remote branch 'github/pr/109'
This closes #109
Signed-off-by: ILYA Khlopotov <ii...@ca.ibm.com>
Project: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/repo
Commit: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/commit/be1e9595
Tree: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/tree/be1e9595
Diff: http://git-wip-us.apache.org/repos/asf/couchdb-chttpd/diff/be1e9595
Branch: refs/heads/2992-limit-doc-size
Commit: be1e959504ac7b0332110a9918365ff20937bc43
Parents: 02202d3 cd00955
Author: ILYA Khlopotov <ii...@ca.ibm.com>
Authored: Fri Apr 15 11:32:30 2016 -0700
Committer: ILYA Khlopotov <ii...@ca.ibm.com>
Committed: Fri Apr 15 11:32:30 2016 -0700
----------------------------------------------------------------------
src/chttpd.erl | 49 ++++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 48 insertions(+), 1 deletion(-)
----------------------------------------------------------------------